From 582a8e2c3d595bfc9766ded88e0e6f03e41e82fa Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 21 Jul 2026 01:50:09 -0400 Subject: [PATCH 01/22] Checkpoint 0.7.4 batch and GPU refactor WIP --- Cargo.lock | 16 +- Cargo.toml | 23 +- README.md | 48 +- crates/j2k-core/src/batch.rs | 13 + crates/j2k-core/src/batch/gpu_job_chunk.rs | 302 + .../src/batch/gpu_job_chunk/planner.rs | 202 + .../j2k-core/src/batch/gpu_job_chunk/tests.rs | 154 + crates/j2k-core/src/lib.rs | 10 +- crates/j2k-core/src/pixel.rs | 54 +- crates/j2k-core/src/sample.rs | 9 +- crates/j2k-cuda-runtime/build.rs | 15 + crates/j2k-cuda-runtime/src/build_flags.rs | 2 + crates/j2k-cuda-runtime/src/bytes.rs | 19 +- crates/j2k-cuda-runtime/src/bytes/abi.rs | 26 +- .../src/bytes/abi/native_store.rs | 85 + .../j2k-cuda-runtime/src/bytes/abi/tests.rs | 20 + crates/j2k-cuda-runtime/src/classic_decode.rs | 6 +- .../src/classic_decode/abi.rs | 20 + .../src/classic_decode/launch.rs | 123 +- .../src/classic_decode/queued.rs | 148 + .../src/classic_decode/tests.rs | 27 + crates/j2k-cuda-runtime/src/context.rs | 43 +- .../j2k-cuda-runtime/src/context/creation.rs | 6 +- .../src/context/diagnostics.rs | 320 + crates/j2k-cuda-runtime/src/context/inner.rs | 16 +- .../src/context/test_kernels.rs | 27 + .../cuda_oxide_htj2k_decode/simt/src/main.rs | 17 +- .../simt/src/abi.rs | 234 + .../simt/src/color.rs | 169 + .../simt/src/exports.rs | 347 ++ .../simt/src/layout.rs | 31 + .../simt/src/main.rs | 556 +- .../simt/src/memory.rs | 35 + .../simt/src/native_color.rs | 289 + .../simt/src/sample.rs | 91 + .../simt/src/transform.rs | 24 + crates/j2k-cuda-runtime/src/driver.rs | 13 + crates/j2k-cuda-runtime/src/error.rs | 77 + crates/j2k-cuda-runtime/src/execution.rs | 20 +- .../j2k-cuda-runtime/src/execution/events.rs | 41 +- .../src/execution/events/handles.rs | 97 +- .../src/execution/events/handles/stream.rs | 34 + .../src/execution/events/interop.rs | 183 + .../src/execution/memory_ops.rs | 26 + crates/j2k-cuda-runtime/src/htj2k_decode.rs | 13 +- .../j2k-cuda-runtime/src/htj2k_decode/api.rs | 3 +- .../src/htj2k_decode/completion.rs | 143 +- .../completion/cleanup_dequant_enqueue.rs | 215 + .../completion/cleanup_enqueue.rs | 210 + .../src/htj2k_decode/completion/dequant.rs | 13 +- .../htj2k_decode/completion/dequant/queued.rs | 69 + .../src/htj2k_decode/completion/tests.rs | 16 + .../src/htj2k_decode/launch.rs | 12 +- .../src/htj2k_decode/queued.rs | 70 +- .../src/htj2k_decode/queued/lifecycle.rs | 67 + .../src/htj2k_decode/status.rs | 37 +- .../src/htj2k_decode/status/tests.rs | 55 + .../src/htj2k_decode/status_group.rs | 241 + .../src/htj2k_decode/types.rs | 8 + crates/j2k-cuda-runtime/src/j2k_decode.rs | 14 +- .../j2k-cuda-runtime/src/j2k_decode/store.rs | 5 + .../src/j2k_decode/store/batch.rs | 40 +- .../src/j2k_decode/store/batch/external.rs | 151 + .../src/j2k_decode/store/batch/tests.rs | 15 + .../j2k_decode/store/color_native_batch.rs | 341 ++ .../store/color_native_batch/plan.rs | 236 + .../store/color_native_rgba_batch.rs | 295 + .../color_native_rgba_batch/validation.rs | 220 + .../j2k_decode/store/destination_groups.rs | 213 + .../src/j2k_decode/store/grayscale_batch.rs | 82 + .../j2k_decode/store/grayscale_batch/api.rs | 373 ++ .../store/grayscale_batch/enqueue.rs | 220 + .../j2k_decode/store/grayscale_batch/plan.rs | 197 + .../src/j2k_decode/store/tests.rs | 1 + .../j2k_decode/store/tests/color_native.rs | 120 + .../src/j2k_decode/store/validation.rs | 22 +- .../src/j2k_decode/store_launch.rs | 76 +- .../j2k_decode/store_launch/color_native.rs | 56 + .../store_launch/color_native_rgba.rs | 56 + .../src/j2k_decode/structure_tests.rs | 54 +- .../j2k-cuda-runtime/src/j2k_decode/types.rs | 71 + .../src/j2k_decode/types/color_native.rs | 90 + .../src/j2k_decode/types/color_native_rgba.rs | 103 + crates/j2k-cuda-runtime/src/kernels.rs | 18 + crates/j2k-cuda-runtime/src/kernels/j2k.rs | 9 + crates/j2k-cuda-runtime/src/kernels/tests.rs | 94 + crates/j2k-cuda-runtime/src/lib.rs | 22 +- crates/j2k-cuda-runtime/src/memory.rs | 15 +- crates/j2k-cuda-runtime/src/memory/pool.rs | 7 + crates/j2k-cuda-runtime/src/tests.rs | 166 +- .../src/tests/context_diagnostics.rs | 67 + .../src/tests/context_external.rs | 204 + .../src/tests/grayscale_external.rs | 161 + crates/j2k-cuda-runtime/src/tests/pipeline.rs | 176 +- .../src/tests/pipeline/forward_reference.rs | 179 + .../src/tests/pipeline/native_store.rs | 174 + crates/j2k-cuda/src/batch.rs | 194 + crates/j2k-cuda/src/batch/decoder.rs | 302 + crates/j2k-cuda/src/batch/external.rs | 170 + crates/j2k-cuda/src/batch/input.rs | 178 + .../j2k-cuda/src/batch/resident_submission.rs | 313 + crates/j2k-cuda/src/batch/types.rs | 318 + crates/j2k-cuda/src/codec.rs | 8 +- crates/j2k-cuda/src/decoder.rs | 46 +- crates/j2k-cuda/src/decoder/api.rs | 27 + crates/j2k-cuda/src/decoder/color_batch.rs | 685 +-- .../decoder/color_batch/batch_execution.rs | 368 ++ .../src/decoder/color_batch/finish.rs | 29 +- .../decoder/color_batch/finish/component.rs | 89 + .../src/decoder/color_batch/finish/surface.rs | 62 + .../src/decoder/color_batch/native_batch.rs | 280 + .../color_batch/native_batch/completion.rs | 92 + .../color_batch/native_batch/execution.rs | 226 + .../color_batch/native_batch/prepare.rs | 134 + .../decoder/color_batch/native_batch/store.rs | 205 + .../color_batch/native_batch/store/jobs.rs | 107 + .../native_batch/store/submission.rs | 214 + .../src/decoder/color_batch/single.rs | 164 + .../src/decoder/color_batch/store/batch.rs | 2 +- .../decoder/color_batch/store/validation.rs | 34 +- .../j2k-cuda/src/decoder/grayscale_batch.rs | 423 ++ .../src/decoder/grayscale_batch/completion.rs | 212 + .../src/decoder/grayscale_batch/execution.rs | 321 + .../decoder/grayscale_batch/preparation.rs | 211 + .../src/decoder/grayscale_batch/store.rs | 338 ++ .../src/decoder/pending_completion.rs | 246 + crates/j2k-cuda/src/decoder/plan.rs | 525 +- crates/j2k-cuda/src/decoder/plan/color.rs | 167 + .../src/decoder/plan/color_decoder.rs | 245 + .../j2k-cuda/src/decoder/plan/color_owners.rs | 7 + .../decoder/plan/color_owners/referenced.rs | 210 + .../src/decoder/plan/color_referenced.rs | 256 + crates/j2k-cuda/src/decoder/plan/grayscale.rs | 441 ++ crates/j2k-cuda/src/decoder/resident.rs | 11 + .../src/decoder/resident/chunked_cleanup.rs | 108 + .../resident/chunked_cleanup/enqueue.rs | 318 + .../resident/chunked_cleanup/planning.rs | 115 + .../decoder/resident/chunked_cleanup/tests.rs | 72 + .../src/decoder/resident/cleanup_dequant.rs | 14 +- .../resident/cleanup_dequant/classic.rs | 7 +- .../cleanup_dequant/classic/queued.rs | 156 + .../cleanup_dequant/classic/queued/tests.rs | 34 + .../resident/cleanup_dequant/enqueue.rs | 124 + .../j2k-cuda/src/decoder/resident/routing.rs | 10 + crates/j2k-cuda/src/direct_plan.rs | 136 +- crates/j2k-cuda/src/direct_plan/accessors.rs | 7 - crates/j2k-cuda/src/direct_plan/classic.rs | 22 +- .../src/direct_plan/classic/referenced.rs | 154 + crates/j2k-cuda/src/direct_plan/ht.rs | 122 +- crates/j2k-cuda/src/direct_plan/shared.rs | 66 +- crates/j2k-cuda/src/direct_plan/tests.rs | 67 + crates/j2k-cuda/src/error.rs | 60 +- crates/j2k-cuda/src/lib.rs | 11 + crates/j2k-cuda/src/session.rs | 292 +- crates/j2k-cuda/src/session/tests.rs | 84 + crates/j2k-cuda/src/surface.rs | 49 +- crates/j2k-cuda/tests/batch_decoder_api.rs | 35 + .../tests/batch_decoder_api/async_resident.rs | 129 + .../batch_decoder_api/basic_contracts.rs | 135 + .../tests/batch_decoder_api/classic_native.rs | 416 ++ .../tests/batch_decoder_api/exact_color.rs | 323 + .../batch_decoder_api/external_lifecycle.rs | 249 + .../tests/batch_decoder_api/multitile.rs | 120 + .../batch_decoder_api/refinement_overlap.rs | 76 + .../j2k-cuda/tests/batch_decoder_api/rgba.rs | 255 + .../tests/batch_decoder_api/session_soak.rs | 139 + .../tests/batch_decoder_api/signed_rgb.rs | 183 + .../tests/batch_decoder_api/support.rs | 31 + crates/j2k-cuda/tests/bench_harness.rs | 1 + crates/j2k-metal-support/src/lib.rs | 4 +- crates/j2k-metal-support/src/resident.rs | 90 +- .../src/resident/destination.rs | 343 ++ .../j2k-metal-support/src/tests/resident.rs | 90 +- crates/j2k-metal/Cargo.toml | 2 +- crates/j2k-metal/src/batch.rs | 784 +-- crates/j2k-metal/src/batch/execute.rs | 33 +- crates/j2k-metal/src/batch/request.rs | 129 + crates/j2k-metal/src/batch/routes.rs | 311 + crates/j2k-metal/src/batch/session.rs | 259 + crates/j2k-metal/src/batch/tests.rs | 277 + crates/j2k-metal/src/batch_decoder.rs | 288 + .../j2k-metal/src/batch_decoder/contracts.rs | 316 + crates/j2k-metal/src/batch_decoder/decoder.rs | 275 + .../src/batch_decoder/encoder_count_tests.rs | 232 + .../j2k-metal/src/batch_decoder/external.rs | 193 + .../j2k-metal/src/batch_decoder/plan_cache.rs | 252 + .../src/batch_decoder/queue_ordering_tests.rs | 545 ++ .../j2k-metal/src/batch_decoder/resident.rs | 80 + .../j2k-metal/src/batch_decoder/submission.rs | 370 ++ crates/j2k-metal/src/compute.rs | 243 +- crates/j2k-metal/src/compute/abi.rs | 20 + .../src/compute/buffer_validation.rs | 4 +- .../src/compute/classic_encode_pipeline.rs | 9 +- .../src/compute/classic_tier1_stats.rs | 3 +- .../j2k-metal/src/compute/decode_cleanup.rs | 13 +- .../j2k-metal/src/compute/decode_dispatch.rs | 79 +- .../decode_dispatch/classic_cleanup.rs | 257 +- .../classic_cleanup/distinct_batch.rs | 290 + .../distinct_metadata_tests.rs | 123 +- .../decode_dispatch/classic_subband.rs | 5 + .../decode_dispatch/classic_subband/tests.rs | 32 + .../src/compute/decode_dispatch/ht_chunks.rs | 418 ++ .../decode_dispatch/ht_chunks/execution.rs | 309 + .../decode_dispatch/ht_chunks/prepared.rs | 454 ++ .../ht_chunks/prepared/tests.rs | 262 + .../decode_dispatch/ht_chunks/tests.rs | 6 + .../decode_dispatch/ht_chunks/tests/cache.rs | 100 + .../ht_chunks/tests/cache/referenced.rs | 91 + .../decode_dispatch/ht_chunks/tests/parity.rs | 142 + .../ht_chunks/tests/planning.rs | 193 + .../ht_chunks/tests/planning/fixtures.rs | 116 + .../ht_chunks/tests/planning/referenced.rs | 123 + .../decode_dispatch/ht_chunks/tests/status.rs | 406 ++ .../compute/decode_dispatch/ht_distinct.rs | 259 +- .../src/compute/decode_dispatch/ht_subband.rs | 168 +- .../src/compute/decode_dispatch/idwt.rs | 91 + .../src/compute/decode_dispatch/store.rs | 41 +- .../decode_dispatch/store/destination.rs | 100 + .../j2k-metal/src/compute/direct_buffers.rs | 3 +- .../j2k-metal/src/compute/direct_commands.rs | 38 +- crates/j2k-metal/src/compute/direct_cpu.rs | 39 +- .../j2k-metal/src/compute/direct_execute.rs | 2 + .../j2k-metal/src/compute/direct_flattened.rs | 33 +- .../src/compute/direct_grayscale_execute.rs | 337 +- .../color_destination.rs | 95 + .../color_destination/encoder.rs | 203 + .../color_destination/store.rs | 156 + .../color_destination/validation.rs | 114 + .../component_plane.rs | 363 +- .../component_plane/execution.rs | 418 ++ .../component_plane/execution/final_plane.rs | 120 + .../direct_grayscale_execute/destination.rs | 77 + .../destination/group_encode.rs | 203 + .../destination/submission.rs | 261 + .../destination_index_validation.rs | 106 + .../grayscale_batch.rs | 162 + .../direct_grayscale_execute/single.rs | 368 +- .../single/execution.rs | 355 ++ .../src/compute/direct_plan_support.rs | 261 - .../src/compute/direct_plan_types.rs | 106 +- .../compute/direct_plan_types/allocation.rs | 42 +- .../src/compute/direct_plan_validation.rs | 18 + .../compute/direct_plan_validation/runtime.rs | 226 + .../compute/direct_plan_validation/shape.rs | 86 + .../src/compute/direct_plane_pack.rs | 24 +- .../j2k-metal/src/compute/direct_prepare.rs | 606 +- .../src/compute/direct_prepare/classic.rs | 472 ++ .../src/compute/direct_prepare/color.rs | 347 ++ .../src/compute/direct_prepare/grayscale.rs | 202 + .../src/compute/direct_prepare/ht.rs | 385 ++ .../src/compute/direct_prepare/referenced.rs | 141 + crates/j2k-metal/src/compute/direct_roi.rs | 103 +- .../j2k-metal/src/compute/direct_scratch.rs | 9 +- .../src/compute/direct_stacked_batch.rs | 40 +- .../command_submission.rs | 774 +-- .../command_submission/classic_tier1.rs | 345 ++ .../command_submission/final_store.rs | 107 + .../command_submission/ht_tier1.rs | 337 ++ .../command_submission/reconstruction.rs | 196 + .../repeated_grayscale/execution.rs | 407 +- .../execution/final_store.rs | 167 + .../execution/reconstruction.rs | 170 + .../repeated_grayscale/execution/tier1.rs | 177 + .../compute/direct_stacked_batch/resources.rs | 42 +- .../compute/direct_stacked_batch/result.rs | 22 - .../direct_stacked_batch/validation.rs | 188 + crates/j2k-metal/src/compute/direct_status.rs | 293 +- .../src/compute/direct_surface_pack.rs | 25 +- crates/j2k-metal/src/compute/direct_tier1.rs | 16 +- .../j2k-metal/src/compute/encode_capacity.rs | 6 +- .../src/compute/forward_transform.rs | 20 +- .../src/compute/ht_forward_reader_tests.rs | 96 + .../src/compute/ht_sigprop_context_tests.rs | 30 + .../j2k-metal/src/compute/lossless_prepare.rs | 41 +- .../src/compute/resident_codestream.rs | 110 +- .../compute/resident_codestream/ht_cleanup.rs | 226 +- .../resident_codestream/ht_cleanup/tests.rs | 29 + .../src/compute/resident_packet_plan.rs | 11 +- .../j2k-metal/src/compute/resident_tier1.rs | 54 +- crates/j2k-metal/src/compute/runtime.rs | 61 +- crates/j2k-metal/src/compute/shader_source.rs | 2 + .../j2k-metal/src/compute/symbol_inventory.rs | 219 - crates/j2k-metal/src/compute/test_counters.rs | 106 +- crates/j2k-metal/src/compute/tests.rs | 1721 +----- .../j2k-metal/src/compute/tests/capacity.rs | 196 + crates/j2k-metal/src/compute/tests/classic.rs | 297 + .../j2k-metal/src/compute/tests/grouping.rs | 298 + crates/j2k-metal/src/compute/tests/hybrid.rs | 280 + .../src/compute/tests/hybrid_support.rs | 98 + .../src/compute/tests/referenced_plan.rs | 59 + .../compute/tests/referenced_plan/prepared.rs | 93 + crates/j2k-metal/src/compute/tests/reuse.rs | 150 + crates/j2k-metal/src/compute/tests/roi.rs | 269 + crates/j2k-metal/src/compute/tests/runtime.rs | 217 + crates/j2k-metal/src/compute/tier1_encode.rs | 26 +- .../src/compute/tier1_encode/test_support.rs | 11 +- crates/j2k-metal/src/decoder.rs | 29 +- crates/j2k-metal/src/decoder/direct_paths.rs | 151 +- crates/j2k-metal/src/decoder/tests.rs | 54 +- crates/j2k-metal/src/error.rs | 39 + crates/j2k-metal/src/ht.rs | 2 +- crates/j2k-metal/src/ht_cleanup.metal | 343 +- crates/j2k-metal/src/hybrid.rs | 143 +- crates/j2k-metal/src/idwt.metal | 17 +- crates/j2k-metal/src/lib.rs | 13 + crates/j2k-metal/src/session.rs | 392 +- crates/j2k-metal/src/session/cache.rs | 10 +- crates/j2k-metal/src/session/cache/key.rs | 137 +- crates/j2k-metal/src/session/cache/tests.rs | 99 + .../src/session/direct_plan_cache.rs | 335 ++ .../src/session/direct_plan_cache/tests.rs | 53 + crates/j2k-metal/src/store.metal | 134 +- .../src/store_native_color_batch.metal | 214 + crates/j2k-metal/src/surface.rs | 14 + crates/j2k-metal/tests/batch_classic_color.rs | 271 + .../j2k-metal/tests/batch_decoder_contract.rs | 30 + crates/j2k-metal/tests/batch_rgba.rs | 383 ++ crates/j2k-metal/tests/device.rs | 2492 +------- .../j2k-metal/tests/device/auto_tile_batch.rs | 385 ++ .../j2k-metal/tests/device/batch_sessions.rs | 314 + crates/j2k-metal/tests/device/color_batch.rs | 377 ++ crates/j2k-metal/tests/device/decode.rs | 388 ++ .../tests/device/direct_gray_requests.rs | 340 ++ .../j2k-metal/tests/device/direct_repeated.rs | 269 + .../tests/device/direct_rgb_requests.rs | 278 + .../j2k-metal/tests/device/external_batch.rs | 352 ++ .../tests/device/grayscale_external.rs | 279 + crates/j2k-metal/tests/device/legacy_batch.rs | 406 ++ .../j2k-metal/tests/device/multitile_color.rs | 14 + .../device/multitile_color/batch_inputs.rs | 19 + .../tests/device/multitile_color/classic.rs | 65 + .../tests/device/multitile_color/gray12.rs | 284 + .../tests/device/multitile_color/rgb.rs | 167 + .../tests/device/multitile_color/signed.rs | 54 + .../j2k-metal/tests/device/resident_batch.rs | 230 + crates/j2k-metal/tests/device/tile_batch.rs | 315 + crates/j2k-metal/tests/shader_integrity.rs | 67 + crates/j2k-ml/Cargo.toml | 45 +- crates/j2k-ml/README.md | 24 +- crates/j2k-ml/benches/batch_decode.rs | 206 + crates/j2k-ml/benches/batch_decode_cuda.rs | 435 ++ crates/j2k-ml/benches/batch_decode_metal.rs | 499 ++ crates/j2k-ml/benches/cuda_telemetry.rs | 200 + crates/j2k-ml/benches/metal_telemetry.rs | 198 + crates/j2k-ml/benches/support/decode_case.rs | 64 + crates/j2k-ml/benches/support/fixture.rs | 109 + .../j2k-ml/benches/support/input_selection.rs | 38 + .../benches/support/metal_process_policy.rs | 46 + crates/j2k-ml/benches/support/mod.rs | 10 + .../j2k-ml/benches/support/process_policy.rs | 27 + crates/j2k-ml/benches/support/workload.rs | 139 + crates/j2k-ml/benches/tensor_decode.rs | 275 - crates/j2k-ml/src/batch_contract.rs | 37 + crates/j2k-ml/src/completion.rs | 153 + crates/j2k-ml/src/cpu.rs | 356 +- crates/j2k-ml/src/cpu/batch.rs | 139 + crates/j2k-ml/src/cpu/materialization.rs | 222 - crates/j2k-ml/src/cuda.rs | 369 +- crates/j2k-ml/src/cuda/batch.rs | 252 + crates/j2k-ml/src/cuda/config.rs | 70 - crates/j2k-ml/src/cuda/interop.rs | 222 +- crates/j2k-ml/src/lib.rs | 295 +- crates/j2k-ml/src/metal.rs | 408 +- crates/j2k-ml/src/metal/batch.rs | 331 ++ crates/j2k-ml/src/metal/interop.rs | 311 + crates/j2k-ml/tests/batch_sessions.rs | 188 + crates/j2k-ml/tests/benchmark_inputs.rs | 100 + crates/j2k-ml/tests/cpu.rs | 612 +- crates/j2k-ml/tests/cuda.rs | 545 +- crates/j2k-ml/tests/cuda_batch_sessions.rs | 229 + crates/j2k-ml/tests/cuda_rgba.rs | 256 + crates/j2k-ml/tests/metal.rs | 347 +- crates/j2k-ml/tests/metal/native_color.rs | 239 + crates/j2k-ml/tests/metal/requests.rs | 239 + crates/j2k-ml/tests/metal/sessions.rs | 171 + .../j2k-native/fixtures/htj2k/LICENSE.OpenJPH | 27 + crates/j2k-native/fixtures/htj2k/README.md | 32 +- .../j2k-native/fixtures/htj2k/gray_u12_53.j2c | Bin 0 -> 404 bytes .../fixtures/htj2k/gray_u12_53.oracle.raw | Bin 0 -> 494 bytes .../j2k-native/fixtures/htj2k/rgb_u12_53.j2c | Bin 0 -> 868 bytes .../fixtures/htj2k/rgb_u12_53.oracle.raw | Bin 0 -> 1482 bytes crates/j2k-native/src/direct_cpu.rs | 701 +-- .../j2k-native/src/direct_cpu/allocation.rs | 163 +- .../src/direct_cpu/allocation/referenced.rs | 143 + .../allocation/referenced/budget.rs | 95 + .../allocation/referenced/storage.rs | 120 + .../direct_cpu/allocation/referenced/view.rs | 243 + crates/j2k-native/src/direct_cpu/color.rs | 303 + crates/j2k-native/src/direct_cpu/component.rs | 352 ++ .../j2k-native/src/direct_cpu/referenced.rs | 140 + .../src/direct_cpu/referenced/component.rs | 145 + .../src/direct_cpu/referenced/output.rs | 181 + .../src/direct_cpu/referenced/payload.rs | 103 + .../src/direct_cpu/referenced_classic.rs | 464 ++ .../src/direct_cpu/referenced_staged.rs | 109 + .../direct_cpu/referenced_staged/entropy.rs | 208 + .../direct_cpu/referenced_staged/finish.rs | 150 + .../referenced_staged/plan_access.rs | 144 + .../direct_cpu/referenced_staged/prepare.rs | 147 + .../src/direct_cpu/referenced_staged/state.rs | 148 + crates/j2k-native/src/direct_plan.rs | 25 + .../j2k-native/src/direct_plan/allocation.rs | 150 +- .../src/direct_plan/referenced_classic.rs | 189 + .../src/direct_plan/referenced_ht.rs | 287 + crates/j2k-native/src/error.rs | 10 + crates/j2k-native/src/image.rs | 27 +- crates/j2k-native/src/image/direct_api.rs | 2 + .../src/image/direct_api/referenced.rs | 109 + .../j2k-native/src/j2c/arithmetic_decoder.rs | 4 + crates/j2k-native/src/j2c/bitplane/state.rs | 44 +- crates/j2k-native/src/j2c/decode.rs | 153 +- .../j2k-native/src/j2c/decode/direct_plan.rs | 662 +-- .../src/j2c/decode/direct_plan/classic.rs | 106 + .../src/j2c/decode/direct_plan/color.rs | 209 + .../decode/direct_plan/referenced_color.rs | 282 + .../direct_plan/referenced_grayscale.rs | 162 + .../src/j2c/decode/direct_plan/storage.rs | 319 + .../decode/direct_plan/storage/sub_band.rs | 403 ++ crates/j2k-native/src/j2c/decode/reuse.rs | 107 +- crates/j2k-native/src/j2c/decode/tier1.rs | 20 +- crates/j2k-native/src/j2c/decode/workspace.rs | 215 + .../src/j2c/ht_block_decode/readers.rs | 16 +- .../src/j2c/ht_block_decode/significance.rs | 5 +- .../src/j2c/ht_block_decode/state.rs | 16 +- .../src/j2c/ht_block_decode/tests.rs | 15 + crates/j2k-native/src/j2c/mod.rs | 14 +- crates/j2k-native/src/jp2/container.rs | 5 +- crates/j2k-native/src/lib.rs | 26 +- .../j2k-native/src/scalar/classic_decode.rs | 14 + crates/j2k-native/src/scalar/ht_decode.rs | 9 +- crates/j2k-native/src/tests.rs | 3 +- .../j2k-native/src/tests/workspace_reuse.rs | 255 + .../j2k-native/tests/referenced_multitile.rs | 280 + .../j2k-test-support/fixtures/htj2k/README.md | 41 + .../fixtures/htj2k/openhtj2k_hifi_ht1_02.j2k | Bin 0 -> 30665 bytes .../htj2k/openhtj2k_hifi_ht1_02.oracle.raw | Bin 0 -> 98304 bytes .../openhtj2k_sigprop_refinement_overlap.j2k | Bin 0 -> 4023 bytes ...gprop_refinement_overlap.openht.oracle.ppm | 3 + ...prop_refinement_overlap.openjph.oracle.raw | Bin 0 -> 98304 bytes .../htj2k/openjph_batch/LICENSE.OpenJPH | 27 + .../fixtures/htj2k/openjph_batch/README.md | 129 + .../htj2k/openjph_batch/encode_signed_rgb.cpp | 92 + .../fixtures/htj2k/openjph_batch/generate.rs | 428 ++ .../htj2k/openjph_batch/gray_s12_53.j2c | Bin 0 -> 404 bytes .../openjph_batch/gray_s12_53.oracle.raw | Bin 0 -> 494 bytes .../openjph_batch/gray_s12_53.source.raw | Bin 0 -> 494 bytes .../htj2k/openjph_batch/gray_s12_97.j2c | Bin 0 -> 504 bytes .../openjph_batch/gray_s12_97.oracle.raw | Bin 0 -> 494 bytes .../openjph_batch/gray_s12_97.source.raw | Bin 0 -> 494 bytes .../htj2k/openjph_batch/gray_s16_53.j2c | Bin 0 -> 429 bytes .../openjph_batch/gray_s16_53.oracle.raw | Bin 0 -> 494 bytes .../openjph_batch/gray_s16_53.source.raw | Bin 0 -> 494 bytes .../htj2k/openjph_batch/gray_s16_97.j2c | Bin 0 -> 507 bytes .../openjph_batch/gray_s16_97.oracle.raw | Bin 0 -> 494 bytes .../openjph_batch/gray_s16_97.source.raw | Bin 0 -> 494 bytes .../htj2k/openjph_batch/gray_s8_53.j2c | Bin 0 -> 665 bytes .../htj2k/openjph_batch/gray_s8_53.oracle.raw | 1 + .../htj2k/openjph_batch/gray_s8_53.source.raw | 1 + .../htj2k/openjph_batch/gray_u12_53.j2c | Bin 0 -> 404 bytes .../htj2k/openjph_batch/gray_u12_53.jph | Bin 0 -> 489 bytes .../openjph_batch/gray_u12_53.oracle.raw | Bin 0 -> 494 bytes .../openjph_batch/gray_u12_53.source.pgm | Bin 0 -> 508 bytes .../htj2k/openjph_batch/gray_u12_97.j2c | Bin 0 -> 523 bytes .../openjph_batch/gray_u12_97.oracle.raw | Bin 0 -> 494 bytes .../openjph_batch/gray_u12_97.source.pgm | Bin 0 -> 508 bytes .../htj2k/openjph_batch/gray_u16_53.j2c | Bin 0 -> 429 bytes .../openjph_batch/gray_u16_53.oracle.raw | Bin 0 -> 494 bytes .../openjph_batch/gray_u16_53.source.pgm | Bin 0 -> 509 bytes .../htj2k/openjph_batch/gray_u16_97.j2c | Bin 0 -> 536 bytes .../openjph_batch/gray_u16_97.oracle.raw | Bin 0 -> 494 bytes .../openjph_batch/gray_u16_97.source.pgm | Bin 0 -> 509 bytes .../htj2k/openjph_batch/gray_u8_53.j2c | Bin 0 -> 665 bytes .../htj2k/openjph_batch/gray_u8_53.oracle.raw | Bin 0 -> 247 bytes .../htj2k/openjph_batch/gray_u8_53.source.pgm | Bin 0 -> 260 bytes .../htj2k/openjph_batch/rgb_s12_53.j2c | Bin 0 -> 868 bytes .../htj2k/openjph_batch/rgb_s12_53.oracle.raw | Bin 0 -> 1482 bytes .../htj2k/openjph_batch/rgb_s12_53_single.j2c | Bin 0 -> 298 bytes .../htj2k/openjph_batch/rgb_s12_97.j2c | Bin 0 -> 1252 bytes .../htj2k/openjph_batch/rgb_s12_97.oracle.raw | Bin 0 -> 1482 bytes .../htj2k/openjph_batch/rgb_s16_53.j2c | Bin 0 -> 942 bytes .../htj2k/openjph_batch/rgb_s16_53.oracle.raw | Bin 0 -> 1482 bytes .../htj2k/openjph_batch/rgb_s16_53_single.j2c | Bin 0 -> 332 bytes .../htj2k/openjph_batch/rgb_s8_53.j2c | Bin 0 -> 1655 bytes .../htj2k/openjph_batch/rgb_s8_53.oracle.raw | Bin 0 -> 741 bytes .../htj2k/openjph_batch/rgb_s8_53_single.j2c | Bin 0 -> 1154 bytes .../htj2k/openjph_batch/rgb_u12_53.j2c | Bin 0 -> 868 bytes .../htj2k/openjph_batch/rgb_u12_53.oracle.raw | Bin 0 -> 1482 bytes .../htj2k/openjph_batch/rgb_u12_53.source.ppm | Bin 0 -> 1496 bytes .../htj2k/openjph_batch/rgb_u12_97.j2c | Bin 0 -> 1311 bytes .../htj2k/openjph_batch/rgb_u12_97.oracle.raw | Bin 0 -> 1482 bytes .../htj2k/openjph_batch/rgb_u12_97.source.ppm | Bin 0 -> 1496 bytes .../htj2k/openjph_batch/rgb_u16_53.j2c | Bin 0 -> 942 bytes .../htj2k/openjph_batch/rgb_u16_53.oracle.raw | Bin 0 -> 1482 bytes .../htj2k/openjph_batch/rgb_u16_53.source.ppm | Bin 0 -> 1497 bytes .../htj2k/openjph_batch/rgb_u8_53.j2c | Bin 0 -> 1655 bytes .../htj2k/openjph_batch/rgb_u8_53.oracle.raw | Bin 0 -> 741 bytes .../htj2k/openjph_batch/rgb_u8_53.source.ppm | Bin 0 -> 754 bytes .../htj2k/openjph_batch/rgb_u8_97.j2c | Bin 0 -> 1774 bytes .../htj2k/openjph_batch/rgb_u8_97.oracle.raw | Bin 0 -> 741 bytes .../htj2k/openjph_batch/rgb_u8_97.source.ppm | Bin 0 -> 754 bytes crates/j2k-test-support/src/fixtures.rs | 451 +- .../src/fixtures/generated_htj2k.rs | 262 + crates/j2k-test-support/src/fixtures/jp2.rs | 154 + crates/j2k-test-support/src/fixtures/jpeg.rs | 127 + .../j2k-test-support/src/fixtures/openjph.rs | 322 + crates/j2k-test-support/src/lib.rs | 18 +- crates/j2k-types/src/decode_payload.rs | 48 + crates/j2k-types/src/lib.rs | 4 + crates/j2k/Cargo.toml | 2 +- crates/j2k/src/adapter/device_plan.rs | 2 +- crates/j2k/src/backend.rs | 15 +- crates/j2k/src/batch.rs | 6 +- crates/j2k/src/batch/allocation.rs | 4 +- crates/j2k/src/batch/direct.rs | 3 +- crates/j2k/src/batch/direct/planning.rs | 12 +- crates/j2k/src/batch/scheduler.rs | 4 + crates/j2k/src/batch/scheduler/retained.rs | 70 + crates/j2k/src/batch/worker.rs | 16 +- crates/j2k/src/batch/worker/owned.rs | 208 + crates/j2k/src/decode.rs | 5 +- crates/j2k/src/decode/settings.rs | 74 + crates/j2k/src/lib.rs | 23 +- crates/j2k/src/owned_batch.rs | 88 + crates/j2k/src/owned_batch/contracts.rs | 358 ++ crates/j2k/src/owned_batch/cpu.rs | 377 ++ crates/j2k/src/owned_batch/cpu_diagnostics.rs | 194 + crates/j2k/src/owned_batch/cpu_execute.rs | 272 + crates/j2k/src/owned_batch/cpu_fast.rs | 493 ++ crates/j2k/src/owned_batch/cpu_fast/plan.rs | 273 + crates/j2k/src/owned_batch/cpu_materialize.rs | 243 + crates/j2k/src/owned_batch/cpu_prepared.rs | 282 + crates/j2k/src/owned_batch/cpu_staged.rs | 179 + .../j2k/src/owned_batch/cpu_staged_execute.rs | 407 ++ crates/j2k/src/owned_batch/errors.rs | 103 + crates/j2k/src/owned_batch/prepare.rs | 284 + crates/j2k/src/owned_batch/prepare/image.rs | 344 ++ crates/j2k/src/owned_batch/prepared.rs | 257 + crates/j2k/src/owned_batch/prepared_plan.rs | 220 + crates/j2k/src/view.rs | 24 +- crates/j2k/tests/device_plan.rs | 34 + crates/j2k/tests/owned_batch.rs | 23 + crates/j2k/tests/owned_batch/cpu_reuse.rs | 359 ++ .../tests/owned_batch/errors_and_requests.rs | 314 + crates/j2k/tests/owned_batch/fixtures.rs | 230 + crates/j2k/tests/owned_batch/grouping.rs | 24 + .../owned_batch/native_types_and_requests.rs | 94 + crates/j2k/tests/owned_batch/oracles.rs | 108 + crates/j2k/tests/owned_batch/payload_plan.rs | 147 + .../j2k/tests/owned_batch/payload_ranges.rs | 150 + crates/j2k/tests/owned_batch/reuse.rs | 250 + crates/j2k/tests/owned_batch/rgba.rs | 386 ++ crates/j2k/tests/owned_batch_fixtures.rs | 386 ++ .../j2k/tests/owned_batch_fixtures/classic.rs | 227 + .../tests/owned_batch_fixtures/ht_matrix.rs | 494 ++ .../owned_batch_fixtures/irreversible.rs | 260 + docs/architecture.md | 92 +- docs/benchmark-evidence.md | 116 + docs/env-vars.md | 2 + docs/j2k-ml.md | 275 +- ...able-api-1.0.implementation-public-api.txt | 372 +- docs/stable-api-1.0.public-api.txt | 483 ++ docs/unsafe-audit.md | 47 +- engineering/public-api-review-0.7.4.yml | 68 +- engineering/reviewed-public-api-diff-0.7.4.md | 536 +- .../cubecl-cuda-0.10.0-patched/.cargo-ok | 1 + .../.cargo_vcs_info.json | 6 + .../cubecl-cuda-0.10.0-patched/Cargo.lock | 2068 +++++++ .../cubecl-cuda-0.10.0-patched/Cargo.toml | 170 + .../Cargo.toml.orig | 73 + .../cubecl-cuda-0.10.0-patched/LICENSE-APACHE | 201 + .../cubecl-cuda-0.10.0-patched/LICENSE-MIT | 21 + .../PATCH_PROVENANCE.md | 12 + .../cubecl-cuda-0.10.0-patched/README.md | 13 + .../cubecl-cuda-0.10.0-patched/build.rs | 13 + .../cubecl-cuda-0.10.0-patched/cubecl.toml | 2 + .../src/compute/command.rs | 667 +++ .../src/compute/communication.rs | 106 + .../src/compute/context.rs | 344 ++ .../src/compute/io/controller.rs | 69 + .../src/compute/io/mod.rs | 1 + .../src/compute/mod.rs | 30 + .../src/compute/server.rs | 1167 ++++ .../src/compute/storage/cpu.rs | 132 + .../src/compute/storage/gpu.rs | 212 + .../src/compute/storage/mod.rs | 2 + .../src/compute/stream.rs | 107 + .../src/compute/sync/fence.rs | 87 + .../src/compute/sync/mod.rs | 3 + .../cubecl-cuda-0.10.0-patched/src/device.rs | 33 + .../cubecl-cuda-0.10.0-patched/src/lib.rs | 80 + .../cubecl-cuda-0.10.0-patched/src/runtime.rs | 392 ++ .../cubecl-runtime-0.10.0-patched/.cargo-ok | 1 + .../.cargo_vcs_info.json | 6 + .../cubecl-runtime-0.10.0-patched/Cargo.lock | 1897 ++++++ .../cubecl-runtime-0.10.0-patched/Cargo.toml | 224 + .../Cargo.toml.orig | 101 + .../LICENSE-APACHE | 201 + .../cubecl-runtime-0.10.0-patched/LICENSE-MIT | 21 + .../PATCH_PROVENANCE.md | 13 + .../cubecl-runtime-0.10.0-patched/README.md | 7 + .../benches/dynamic.rs | 36 + .../cubecl-runtime-0.10.0-patched/build.rs | 12 + .../src/allocator.rs | 146 + .../src/client.rs | 1076 ++++ .../src/compiler.rs | 91 + .../src/config/autotune.rs | 61 + .../src/config/base.rs | 219 + .../src/config/cache.rs | 61 + .../src/config/compilation.rs | 53 + .../src/config/logger.rs | 135 + .../src/config/memory.rs | 48 + .../src/config/mod.rs | 20 + .../src/config/profiling.rs | 36 + .../src/config/streaming.rs | 44 + .../cubecl-runtime-0.10.0-patched/src/id.rs | 266 + .../src/kernel.rs | 296 + .../cubecl-runtime-0.10.0-patched/src/lib.rs | 58 + .../src/logging/mod.rs | 6 + .../src/logging/profiling.rs | 159 + .../src/logging/server.rs | 216 + .../src/memory_management/base.rs | 103 + .../src/memory_management/drop_queue/mod.rs | 5 + .../memory_management/drop_queue/policy.rs | 112 + .../src/memory_management/drop_queue/queue.rs | 331 ++ .../src/memory_management/memory_manage.rs | 1101 ++++ .../src/memory_management/memory_pool/base.rs | 117 + .../memory_pool/exclusive_pool.rs | 259 + .../memory_management/memory_pool/handle.rs | 269 + .../memory_pool/memory_page.rs | 717 +++ .../src/memory_management/memory_pool/mod.rs | 14 + .../memory_pool/persistent_pool.rs | 275 + .../memory_pool/sliced_pool.rs | 177 + .../src/memory_management/mod.rs | 75 + .../src/runtime.rs | 52 + .../src/server/base.rs | 1159 ++++ .../src/server/handle.rs | 160 + .../src/server/mod.rs | 5 + .../src/storage/base.rs | 120 + .../src/storage/bytes_cpu.rs | 246 + .../src/storage/mod.rs | 8 + .../src/stream/base.rs | 103 + .../src/stream/event.rs | 529 ++ .../src/stream/mod.rs | 12 + .../src/stream/scheduler.rs | 266 + .../src/timestamp_profiler.rs | 59 + .../cubecl-runtime-0.10.0-patched/src/tma.rs | 130 + .../src/tune/base.rs | 592 ++ .../src/tune/input_generator.rs | 50 + .../src/tune/key_generator.rs | 24 + .../src/tune/local.rs | 190 + .../src/tune/mod.rs | 49 + .../src/tune/operation.rs | 138 + .../src/tune/tune_benchmark.rs | 126 + .../src/tune/tune_cache.rs | 265 + .../src/tune/tune_inputs.rs | 22 + .../src/tune/tuner.rs | 406 ++ .../src/tune/util.rs | 59 + .../src/validation.rs | 45 + .../tests/dummy/compute.rs | 136 + .../tests/dummy/kernel.rs | 36 + .../tests/dummy/mod.rs | 9 + .../tests/dummy/server.rs | 299 + .../tests/dummy/tune/autotune_operations.rs | 26 + .../tests/dummy/tune/kernels.rs | 70 + .../tests/dummy/tune/mod.rs | 8 + .../tests/dummy/tune/operation_sets.rs | 63 + .../tests/integration_test.rs | 102 + third_party/wgpu-29.0.4-patched/.cargo-ok | 1 + .../wgpu-29.0.4-patched/.cargo_vcs_info.json | 6 + third_party/wgpu-29.0.4-patched/Cargo.lock | 1280 ++++ third_party/wgpu-29.0.4-patched/Cargo.toml | 268 + .../wgpu-29.0.4-patched/Cargo.toml.orig | 271 + .../wgpu-29.0.4-patched/LICENSE.APACHE | 176 + third_party/wgpu-29.0.4-patched/LICENSE.MIT | 21 + .../wgpu-29.0.4-patched/PATCH_PROVENANCE.md | 19 + third_party/wgpu-29.0.4-patched/README.md | 168 + third_party/wgpu-29.0.4-patched/build.rs | 62 + .../wgpu-29.0.4-patched/src/api/adapter.rs | 222 + .../wgpu-29.0.4-patched/src/api/bind_group.rs | 167 + .../src/api/bind_group_layout.rs | 45 + .../wgpu-29.0.4-patched/src/api/blas.rs | 286 + .../wgpu-29.0.4-patched/src/api/buffer.rs | 1177 ++++ .../src/api/command_buffer.rs | 32 + .../src/api/command_buffer_actions.rs | 149 + .../src/api/command_encoder.rs | 448 ++ .../src/api/common_pipeline.rs | 58 + .../src/api/compute_pass.rs | 198 + .../src/api/compute_pipeline.rs | 87 + .../wgpu-29.0.4-patched/src/api/device.rs | 877 +++ .../src/api/external_texture.rs | 31 + .../wgpu-29.0.4-patched/src/api/instance.rs | 441 ++ .../wgpu-29.0.4-patched/src/api/mod.rs | 103 + .../src/api/pipeline_cache.rs | 96 + .../src/api/pipeline_layout.rs | 47 + .../wgpu-29.0.4-patched/src/api/query_set.rs | 50 + .../wgpu-29.0.4-patched/src/api/queue.rs | 358 ++ .../src/api/render_bundle.rs | 36 + .../src/api/render_bundle_encoder.rs | 212 + .../src/api/render_pass.rs | 667 +++ .../src/api/render_pipeline.rs | 334 ++ .../wgpu-29.0.4-patched/src/api/sampler.rs | 36 + .../src/api/shader_module.rs | 237 + .../wgpu-29.0.4-patched/src/api/surface.rs | 517 ++ .../src/api/surface_texture.rs | 106 + .../wgpu-29.0.4-patched/src/api/texture.rs | 188 + .../src/api/texture_view.rs | 92 + .../wgpu-29.0.4-patched/src/api/tlas.rs | 165 + .../wgpu-29.0.4-patched/src/backend/custom.rs | 99 + .../wgpu-29.0.4-patched/src/backend/mod.rs | 13 + .../wgpu-29.0.4-patched/src/backend/webgpu.rs | 4057 +++++++++++++ .../webgpu/defined_non_null_js_value.rs | 46 + .../src/backend/webgpu/ext_bindings.rs | 45 + .../src/backend/webgpu/webgpu_sys/gen_Gpu.rs | 87 + .../webgpu/webgpu_sys/gen_GpuAdapter.rs | 109 + .../webgpu/webgpu_sys/gen_GpuAdapterInfo.rs | 84 + .../webgpu/webgpu_sys/gen_GpuAddressMode.rs | 38 + .../webgpu_sys/gen_GpuAutoLayoutMode.rs | 36 + .../webgpu/webgpu_sys/gen_GpuBindGroup.rs | 62 + .../webgpu_sys/gen_GpuBindGroupDescriptor.rs | 126 + .../webgpu_sys/gen_GpuBindGroupEntry.rs | 102 + .../webgpu_sys/gen_GpuBindGroupLayout.rs | 62 + .../gen_GpuBindGroupLayoutDescriptor.rs | 101 + .../webgpu_sys/gen_GpuBindGroupLayoutEntry.rs | 232 + .../webgpu_sys/gen_GpuBlendComponent.rs | 130 + .../webgpu/webgpu_sys/gen_GpuBlendFactor.rs | 52 + .../webgpu_sys/gen_GpuBlendOperation.rs | 40 + .../webgpu/webgpu_sys/gen_GpuBlendState.rs | 102 + .../webgpu/webgpu_sys/gen_GpuBuffer.rs | 313 + .../webgpu/webgpu_sys/gen_GpuBufferBinding.rs | 125 + .../webgpu_sys/gen_GpuBufferBindingLayout.rs | 130 + .../webgpu_sys/gen_GpuBufferBindingType.rs | 38 + .../webgpu_sys/gen_GpuBufferDescriptor.rs | 150 + .../webgpu_sys/gen_GpuBufferMapState.rs | 38 + .../webgpu_sys/gen_GpuCanvasAlphaMode.rs | 37 + .../webgpu_sys/gen_GpuCanvasConfiguration.rs | 198 + .../webgpu/webgpu_sys/gen_GpuCanvasContext.rs | 98 + .../webgpu_sys/gen_GpuCanvasToneMapping.rs | 82 + .../gen_GpuCanvasToneMappingMode.rs | 37 + .../webgpu/webgpu_sys/gen_GpuColorDict.rs | 152 + .../webgpu_sys/gen_GpuColorTargetState.rs | 125 + .../webgpu/webgpu_sys/gen_GpuCommandBuffer.rs | 62 + .../gen_GpuCommandBufferDescriptor.rs | 82 + .../webgpu_sys/gen_GpuCommandEncoder.rs | 600 ++ .../gen_GpuCommandEncoderDescriptor.rs | 82 + .../webgpu_sys/gen_GpuCompareFunction.rs | 43 + .../webgpu_sys/gen_GpuCompilationInfo.rs | 51 + .../webgpu_sys/gen_GpuCompilationMessage.rs | 106 + .../gen_GpuCompilationMessageType.rs | 38 + .../gen_GpuComputePassDescriptor.rs | 111 + .../webgpu_sys/gen_GpuComputePassEncoder.rs | 292 + .../gen_GpuComputePassTimestampWrites.rs | 125 + .../webgpu_sys/gen_GpuComputePipeline.rs | 73 + .../gen_GpuComputePipelineDescriptor.rs | 126 + .../gen_GpuCopyExternalImageDestInfo.rs | 173 + .../gen_GpuCopyExternalImageSourceInfo.rs | 125 + .../webgpu/webgpu_sys/gen_GpuCullMode.rs | 38 + .../webgpu_sys/gen_GpuDepthStencilState.rs | 293 + .../webgpu/webgpu_sys/gen_GpuDevice.rs | 402 ++ .../webgpu_sys/gen_GpuDeviceDescriptor.rs | 154 + .../webgpu_sys/gen_GpuDeviceLostInfo.rs | 62 + .../webgpu_sys/gen_GpuDeviceLostReason.rs | 37 + .../backend/webgpu/webgpu_sys/gen_GpuError.rs | 51 + .../webgpu/webgpu_sys/gen_GpuErrorFilter.rs | 38 + .../webgpu/webgpu_sys/gen_GpuExtent3dDict.rs | 125 + .../webgpu_sys/gen_GpuExternalTexture.rs | 62 + .../gen_GpuExternalTextureBindingLayout.rs | 58 + .../gen_GpuExternalTextureDescriptor.rs | 101 + .../webgpu/webgpu_sys/gen_GpuFeatureName.rs | 51 + .../webgpu/webgpu_sys/gen_GpuFilterMode.rs | 37 + .../webgpu/webgpu_sys/gen_GpuFragmentState.rs | 150 + .../webgpu/webgpu_sys/gen_GpuFrontFace.rs | 37 + .../webgpu/webgpu_sys/gen_GpuIndexFormat.rs | 37 + .../webgpu/webgpu_sys/gen_GpuLoadOp.rs | 37 + .../webgpu_sys/gen_GpuMipmapFilterMode.rs | 37 + .../webgpu_sys/gen_GpuMultisampleState.rs | 130 + .../webgpu_sys/gen_GpuObjectDescriptorBase.rs | 82 + .../webgpu/webgpu_sys/gen_GpuOrigin2dDict.rs | 106 + .../webgpu/webgpu_sys/gen_GpuOrigin3dDict.rs | 130 + .../webgpu_sys/gen_GpuOutOfMemoryError.rs | 51 + .../gen_GpuPipelineDescriptorBase.rs | 101 + .../webgpu_sys/gen_GpuPipelineLayout.rs | 62 + .../gen_GpuPipelineLayoutDescriptor.rs | 104 + .../webgpu_sys/gen_GpuPowerPreference.rs | 37 + .../webgpu_sys/gen_GpuPrimitiveState.rs | 178 + .../webgpu_sys/gen_GpuPrimitiveTopology.rs | 40 + .../webgpu_sys/gen_GpuProgrammableStage.rs | 125 + .../webgpu/webgpu_sys/gen_GpuQuerySet.rs | 95 + .../webgpu_sys/gen_GpuQuerySetDescriptor.rs | 126 + .../webgpu/webgpu_sys/gen_GpuQueryType.rs | 37 + .../backend/webgpu/webgpu_sys/gen_GpuQueue.rs | 950 +++ .../webgpu_sys/gen_GpuQueueDescriptor.rs | 82 + .../webgpu/webgpu_sys/gen_GpuRenderBundle.rs | 62 + .../gen_GpuRenderBundleDescriptor.rs | 82 + .../webgpu_sys/gen_GpuRenderBundleEncoder.rs | 656 ++ .../gen_GpuRenderBundleEncoderDescriptor.rs | 202 + .../gen_GpuRenderPassColorAttachment.rs | 199 + ...gen_GpuRenderPassDepthStencilAttachment.rs | 269 + .../webgpu_sys/gen_GpuRenderPassDescriptor.rs | 207 + .../webgpu_sys/gen_GpuRenderPassEncoder.rs | 744 +++ .../gen_GpuRenderPassTimestampWrites.rs | 125 + .../webgpu_sys/gen_GpuRenderPipeline.rs | 73 + .../gen_GpuRenderPipelineDescriptor.rs | 222 + .../gen_GpuRequestAdapterOptions.rs | 154 + .../webgpu/webgpu_sys/gen_GpuSampler.rs | 62 + .../webgpu_sys/gen_GpuSamplerBindingLayout.rs | 82 + .../webgpu_sys/gen_GpuSamplerBindingType.rs | 38 + .../webgpu_sys/gen_GpuSamplerDescriptor.rs | 322 + .../webgpu/webgpu_sys/gen_GpuShaderModule.rs | 73 + .../gen_GpuShaderModuleDescriptor.rs | 125 + .../webgpu_sys/gen_GpuStencilFaceState.rs | 154 + .../webgpu_sys/gen_GpuStencilOperation.rs | 43 + .../webgpu_sys/gen_GpuStorageTextureAccess.rs | 38 + .../gen_GpuStorageTextureBindingLayout.rs | 127 + .../webgpu/webgpu_sys/gen_GpuStoreOp.rs | 37 + .../webgpu_sys/gen_GpuSupportedFeatures.rs | 109 + .../webgpu_sys/gen_GpuSupportedLimits.rs | 381 ++ .../webgpu_sys/gen_GpuTexelCopyBufferInfo.rs | 149 + .../gen_GpuTexelCopyBufferLayout.rs | 130 + .../webgpu_sys/gen_GpuTexelCopyTextureInfo.rs | 149 + .../webgpu/webgpu_sys/gen_GpuTexture.rs | 186 + .../webgpu/webgpu_sys/gen_GpuTextureAspect.rs | 38 + .../webgpu_sys/gen_GpuTextureBindingLayout.rs | 130 + .../webgpu_sys/gen_GpuTextureDescriptor.rs | 247 + .../webgpu_sys/gen_GpuTextureDimension.rs | 38 + .../webgpu/webgpu_sys/gen_GpuTextureFormat.rs | 130 + .../webgpu_sys/gen_GpuTextureSampleType.rs | 40 + .../webgpu/webgpu_sys/gen_GpuTextureView.rs | 62 + .../gen_GpuTextureViewDescriptor.rs | 274 + .../webgpu_sys/gen_GpuTextureViewDimension.rs | 41 + .../webgpu_sys/gen_GpuUncapturedErrorEvent.rs | 65 + .../gen_GpuUncapturedErrorEventInit.rs | 149 + .../webgpu_sys/gen_GpuValidationError.rs | 51 + .../webgpu_sys/gen_GpuVertexAttribute.rs | 127 + .../webgpu_sys/gen_GpuVertexBufferLayout.rs | 126 + .../webgpu/webgpu_sys/gen_GpuVertexFormat.rs | 76 + .../webgpu/webgpu_sys/gen_GpuVertexState.rs | 149 + .../webgpu_sys/gen_GpuVertexStepMode.rs | 37 + .../webgpu_sys/gen_WgslLanguageFeatures.rs | 109 + .../webgpu/webgpu_sys/gen_gpu_map_mode.rs | 47 + .../src/backend/webgpu/webgpu_sys/mod.rs | 281 + .../src/backend/wgpu_core.rs | 4036 +++++++++++++ .../src/backend/wgpu_core/thread_id.rs | 30 + third_party/wgpu-29.0.4-patched/src/cmp.rs | 109 + .../wgpu-29.0.4-patched/src/dispatch.rs | 985 +++ third_party/wgpu-29.0.4-patched/src/lib.rs | 190 + .../src/macros/be-aligned.spv | 1 + .../src/macros/le-aligned.spv | 1 + .../wgpu-29.0.4-patched/src/macros/mod.rs | 259 + .../wgpu-29.0.4-patched/src/util/belt.rs | 344 ++ .../wgpu-29.0.4-patched/src/util/blit.wgsl | 30 + .../wgpu-29.0.4-patched/src/util/device.rs | 167 + .../wgpu-29.0.4-patched/src/util/encoder.rs | 197 + .../wgpu-29.0.4-patched/src/util/init.rs | 128 + .../wgpu-29.0.4-patched/src/util/mod.rs | 197 + .../wgpu-29.0.4-patched/src/util/mutex.rs | 60 + .../wgpu-29.0.4-patched/src/util/panicking.rs | 9 + .../wgpu-29.0.4-patched/src/util/spirv.rs | 221 + .../src/util/texture_blitter.rs | 218 + .../wgpu-core-29.0.4-patched/.cargo-ok | 1 + .../.cargo_vcs_info.json | 6 + .../wgpu-core-29.0.4-patched/Cargo.lock | 1165 ++++ .../wgpu-core-29.0.4-patched/Cargo.toml | 230 + .../wgpu-core-29.0.4-patched/Cargo.toml.orig | 198 + .../wgpu-core-29.0.4-patched/LICENSE.APACHE | 176 + .../wgpu-core-29.0.4-patched/LICENSE.MIT | 21 + .../PATCH_PROVENANCE.md | 15 + third_party/wgpu-core-29.0.4-patched/build.rs | 27 + .../wgpu-core-29.0.4-patched/src/as_hal.rs | 413 ++ .../src/binding_model.rs | 1362 +++++ .../src/command/allocator.rs | 52 + .../src/command/bind.rs | 534 ++ .../src/command/bundle.rs | 1735 ++++++ .../src/command/clear.rs | 547 ++ .../src/command/compute.rs | 1319 +++++ .../src/command/compute_command.rs | 70 + .../src/command/draw.rs | 164 + .../src/command/encoder.rs | 53 + .../src/command/encoder_command.rs | 186 + .../src/command/ffi.rs | 9 + .../src/command/memory_init.rs | 338 ++ .../src/command/mod.rs | 2076 +++++++ .../src/command/pass.rs | 370 ++ .../src/command/query.rs | 525 ++ .../src/command/ray_tracing.rs | 1017 ++++ .../src/command/render.rs | 3859 ++++++++++++ .../src/command/render_command.rs | 143 + .../src/command/timestamp_writes.rs | 18 + .../src/command/transfer.rs | 1536 +++++ .../src/command/transition_resources.rs | 118 + .../wgpu-core-29.0.4-patched/src/conv.rs | 272 + .../src/device/bgl.rs | 125 + .../src/device/global.rs | 2165 +++++++ .../src/device/life.rs | 410 ++ .../src/device/mod.rs | 383 ++ .../src/device/queue.rs | 1878 ++++++ .../src/device/ray_tracing.rs | 405 ++ .../src/device/resource.rs | 5273 +++++++++++++++++ .../src/device/trace.rs | 287 + .../src/device/trace/record.rs | 952 +++ .../src/device/trace/replay.rs | 51 + .../wgpu-core-29.0.4-patched/src/error.rs | 64 + .../wgpu-core-29.0.4-patched/src/global.rs | 104 + .../src/hash_utils.rs | 14 + .../wgpu-core-29.0.4-patched/src/hub.rs | 261 + .../wgpu-core-29.0.4-patched/src/id.rs | 355 ++ .../wgpu-core-29.0.4-patched/src/identity.rs | 162 + .../src/indirect_validation/dispatch.rs | 408 ++ .../src/indirect_validation/draw.rs | 1032 ++++ .../src/indirect_validation/mod.rs | 109 + .../src/indirect_validation/utils.rs | 84 + .../indirect_validation/validate_draw.wgsl | 98 + .../src/init_tracker/buffer.rs | 40 + .../src/init_tracker/mod.rs | 426 ++ .../src/init_tracker/texture.rs | 107 + .../wgpu-core-29.0.4-patched/src/instance.rs | 1248 ++++ .../wgpu-core-29.0.4-patched/src/lib.rs | 249 + .../wgpu-core-29.0.4-patched/src/lock/mod.rs | 55 + .../src/lock/observing.rs | 503 ++ .../wgpu-core-29.0.4-patched/src/lock/rank.rs | 163 + .../src/lock/ranked.rs | 411 ++ .../src/lock/vanilla.rs | 151 + .../wgpu-core-29.0.4-patched/src/pipeline.rs | 853 +++ .../src/pipeline_cache.rs | 541 ++ .../wgpu-core-29.0.4-patched/src/pool.rs | 309 + .../wgpu-core-29.0.4-patched/src/present.rs | 423 ++ .../src/ray_tracing.rs | 448 ++ .../wgpu-core-29.0.4-patched/src/registry.rs | 152 + .../wgpu-core-29.0.4-patched/src/resource.rs | 2427 ++++++++ .../wgpu-core-29.0.4-patched/src/scratch.rs | 45 + .../wgpu-core-29.0.4-patched/src/snatch.rs | 200 + .../wgpu-core-29.0.4-patched/src/storage.rs | 144 + .../src/timestamp_normalization/common.wgsl | 142 + .../src/timestamp_normalization/mod.rs | 427 ++ .../timestamp_normalization.wgsl | 41 + .../src/track/blas.rs | 59 + .../src/track/buffer.rs | 815 +++ .../src/track/metadata.rs | 208 + .../wgpu-core-29.0.4-patched/src/track/mod.rs | 705 +++ .../src/track/range.rs | 206 + .../src/track/stateless.rs | 36 + .../src/track/texture.rs | 1547 +++++ .../src/validation.rs | 1748 ++++++ .../src/validation/shader_io_deductions.rs | 176 + .../wgpu-core-29.0.4-patched/src/weak_vec.rs | 66 + third_party/wgpu-hal-29.0.4-patched/.cargo-ok | 1 + .../.cargo_vcs_info.json | 6 + .../wgpu-hal-29.0.4-patched/Cargo.lock | 2512 ++++++++ .../wgpu-hal-29.0.4-patched/Cargo.toml | 520 ++ .../wgpu-hal-29.0.4-patched/Cargo.toml.orig | 355 ++ .../wgpu-hal-29.0.4-patched/LICENSE.APACHE | 176 + .../wgpu-hal-29.0.4-patched/LICENSE.MIT | 21 + .../PATCH_PROVENANCE.md | 20 + third_party/wgpu-hal-29.0.4-patched/README.md | 120 + third_party/wgpu-hal-29.0.4-patched/build.rs | 32 + .../examples/halmark/main.rs | 918 +++ .../examples/halmark/shader.wgsl | 50 + .../examples/raw-gles.em.html | 16 + .../examples/raw-gles.rs | 396 ++ .../examples/ray-traced-triangle/main.rs | 1223 ++++ .../examples/ray-traced-triangle/shader.wgsl | 39 + .../src/auxil/dxgi/conv.rs | 295 + .../src/auxil/dxgi/exception.rs | 97 + .../src/auxil/dxgi/factory.rs | 181 + .../src/auxil/dxgi/mod.rs | 6 + .../src/auxil/dxgi/name.rs | 24 + .../src/auxil/dxgi/result.rs | 28 + .../src/auxil/dxgi/time.rs | 97 + .../wgpu-hal-29.0.4-patched/src/auxil/mod.rs | 231 + .../src/auxil/renderdoc.rs | 141 + .../src/dx12/adapter.rs | 1472 +++++ .../src/dx12/command.rs | 1859 ++++++ .../wgpu-hal-29.0.4-patched/src/dx12/conv.rs | 449 ++ .../wgpu-hal-29.0.4-patched/src/dx12/dcomp.rs | 120 + .../src/dx12/descriptor.rs | 329 + .../src/dx12/device.rs | 2636 ++++++++ .../src/dx12/device_creation.rs | 176 + .../src/dx12/instance.rs | 188 + .../wgpu-hal-29.0.4-patched/src/dx12/mod.rs | 1698 ++++++ .../src/dx12/pipeline_desc.rs | 344 ++ .../src/dx12/sampler.rs | 252 + .../src/dx12/shader_compilation.rs | 447 ++ .../src/dx12/suballocation.rs | 603 ++ .../wgpu-hal-29.0.4-patched/src/dx12/types.rs | 39 + .../wgpu-hal-29.0.4-patched/src/dx12/view.rs | 340 ++ .../src/dynamic/adapter.rs | 81 + .../src/dynamic/command.rs | 761 +++ .../src/dynamic/device.rs | 558 ++ .../src/dynamic/instance.rs | 70 + .../src/dynamic/mod.rs | 221 + .../src/dynamic/queue.rs | 56 + .../src/dynamic/surface.rs | 72 + .../src/gles/adapter.rs | 1366 +++++ .../src/gles/command.rs | 1297 ++++ .../wgpu-hal-29.0.4-patched/src/gles/conv.rs | 430 ++ .../src/gles/device.rs | 1677 ++++++ .../wgpu-hal-29.0.4-patched/src/gles/egl.rs | 1460 +++++ .../src/gles/emscripten.rs | 26 + .../wgpu-hal-29.0.4-patched/src/gles/fence.rs | 168 + .../wgpu-hal-29.0.4-patched/src/gles/mod.rs | 1140 ++++ .../wgpu-hal-29.0.4-patched/src/gles/queue.rs | 1963 ++++++ .../src/gles/shaders/clear.frag | 7 + .../src/gles/shaders/clear.vert | 9 + .../src/gles/shaders/srgb_present.frag | 16 + .../src/gles/shaders/srgb_present.vert | 18 + .../wgpu-hal-29.0.4-patched/src/gles/web.rs | 453 ++ .../wgpu-hal-29.0.4-patched/src/gles/wgl.rs | 900 +++ .../wgpu-hal-29.0.4-patched/src/lib.rs | 2852 +++++++++ .../src/metal/adapter.rs | 1578 +++++ .../src/metal/command.rs | 1916 ++++++ .../wgpu-hal-29.0.4-patched/src/metal/conv.rs | 487 ++ .../src/metal/device.rs | 2028 +++++++ .../src/metal/library_from_metallib.rs | 48 + .../wgpu-hal-29.0.4-patched/src/metal/mod.rs | 1216 ++++ .../src/metal/surface.rs | 195 + .../wgpu-hal-29.0.4-patched/src/metal/time.rs | 38 + .../src/noop/buffer.rs | 88 + .../src/noop/command.rs | 336 ++ .../wgpu-hal-29.0.4-patched/src/noop/mod.rs | 507 ++ .../src/validation_canary.rs | 39 + .../src/vulkan/adapter.rs | 3387 +++++++++++ .../src/vulkan/command.rs | 1402 +++++ .../src/vulkan/conv.rs | 1037 ++++ .../src/vulkan/device.rs | 2826 +++++++++ .../wgpu-hal-29.0.4-patched/src/vulkan/drm.rs | 166 + .../src/vulkan/instance.rs | 1044 ++++ .../wgpu-hal-29.0.4-patched/src/vulkan/mod.rs | 1557 +++++ .../src/vulkan/sampler.rs | 177 + .../src/vulkan/semaphore_list.rs | 195 + .../src/vulkan/swapchain/mod.rs | 107 + .../src/vulkan/swapchain/native.rs | 903 +++ typos.toml | 5 + xtask/src/benchmark_commands.rs | 46 +- xtask/src/benchmark_commands/tests.rs | 31 +- xtask/src/cuda.rs | 18 +- xtask/src/cuda/tests.rs | 2 +- xtask/src/main.rs | 8 +- xtask/src/metal.rs | 8 +- xtask/src/metal/tests.rs | 11 +- .../repo_lint_support/dependency_policy.rs | 10 +- .../decoder_fixture_policy.rs | 10 +- .../decoder_fixture_policy/shared_codec.rs | 34 + .../duplication_policy.rs | 22 +- .../repo_lint_support/gpu_adapter_policy.rs | 142 +- .../cuda_batch_api_structure_policy.rs | 83 + .../gpu_adapter_policy/cuda_decoder_policy.rs | 165 +- .../cuda_decoder_policy/allocations.rs | 79 + .../cuda_decoder_policy/architecture.rs | 48 +- .../cuda_decoder_policy/color_runtime.rs | 96 +- .../color_runtime/async_cleanup.rs | 23 + .../color_runtime/owner_completion.rs | 41 + .../resident_leaf_structure/classic.rs | 10 + .../cuda_htj2k_runtime_structure_policy.rs | 45 +- .../cuda_runtime_safety_policy/abi.rs | 2 +- .../allocation/phase_contract/sources.rs | 6 + .../cuda_runtime_safety_policy/lifecycle.rs | 4 + .../lifecycle/queued.rs | 16 +- .../validation/focus.rs | 1 + .../validation/htj2k_output.rs | 3 +- .../validation/htj2k_output/planning.rs | 16 +- .../validation/queued.rs | 34 +- .../metal_batch_allocation_policy.rs | 14 +- .../metal_batch_structure_policy.rs | 209 + .../metal_plan_cache_policy.rs | 14 +- .../optional_outcomes.rs | 9 +- .../metal_plan_cache_policy/source.rs | 1 + .../metal_typed_error_policy.rs | 3 +- .../gpu_device_structure_policy.rs | 88 + .../tests/repo_lint_support/j2k_ml_policy.rs | 296 +- .../benchmark_support_structure.rs | 137 + .../metal_compute_structure_policy.rs | 140 +- .../batch_execution.rs | 241 + .../compute_tests.rs | 85 + .../direct_plan_ratchets.rs | 35 + .../metal_compute_symbol_policy.rs | 22 + .../metal_direct_plan_structure_policy.rs | 40 + xtask/tests/repo_lint_support/mod.rs | 3 + .../native_decode_allocation_policy.rs | 84 +- .../downstream.rs | 92 + .../native_direct_plan_structure_policy.rs | 70 + .../repo_lint_support/public_docs_policy.rs | 28 + .../tests/repo_lint_support/shader_policy.rs | 50 + .../tests/repo_lint_support/source_policy.rs | 216 + 1082 files changed, 232133 insertions(+), 17104 deletions(-) create mode 100644 crates/j2k-core/src/batch/gpu_job_chunk.rs create mode 100644 crates/j2k-core/src/batch/gpu_job_chunk/planner.rs create mode 100644 crates/j2k-core/src/batch/gpu_job_chunk/tests.rs create mode 100644 crates/j2k-cuda-runtime/src/bytes/abi/native_store.rs create mode 100644 crates/j2k-cuda-runtime/src/classic_decode/queued.rs create mode 100644 crates/j2k-cuda-runtime/src/context/diagnostics.rs create mode 100644 crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/abi.rs create mode 100644 crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/color.rs create mode 100644 crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/exports.rs create mode 100644 crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/layout.rs create mode 100644 crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/memory.rs create mode 100644 crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/native_color.rs create mode 100644 crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/sample.rs create mode 100644 crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/transform.rs create mode 100644 crates/j2k-cuda-runtime/src/execution/events/handles/stream.rs create mode 100644 crates/j2k-cuda-runtime/src/execution/events/interop.rs create mode 100644 crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_dequant_enqueue.rs create mode 100644 crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_enqueue.rs create mode 100644 crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant/queued.rs create mode 100644 crates/j2k-cuda-runtime/src/htj2k_decode/completion/tests.rs create mode 100644 crates/j2k-cuda-runtime/src/htj2k_decode/queued/lifecycle.rs create mode 100644 crates/j2k-cuda-runtime/src/htj2k_decode/status_group.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/batch/external.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/batch/tests.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_batch.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_batch/plan.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_rgba_batch.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_rgba_batch/validation.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/destination_groups.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/api.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/enqueue.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/plan.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store/tests/color_native.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native_rgba.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/types/color_native.rs create mode 100644 crates/j2k-cuda-runtime/src/j2k_decode/types/color_native_rgba.rs create mode 100644 crates/j2k-cuda-runtime/src/tests/context_diagnostics.rs create mode 100644 crates/j2k-cuda-runtime/src/tests/context_external.rs create mode 100644 crates/j2k-cuda-runtime/src/tests/grayscale_external.rs create mode 100644 crates/j2k-cuda-runtime/src/tests/pipeline/forward_reference.rs create mode 100644 crates/j2k-cuda-runtime/src/tests/pipeline/native_store.rs create mode 100644 crates/j2k-cuda/src/batch.rs create mode 100644 crates/j2k-cuda/src/batch/decoder.rs create mode 100644 crates/j2k-cuda/src/batch/external.rs create mode 100644 crates/j2k-cuda/src/batch/input.rs create mode 100644 crates/j2k-cuda/src/batch/resident_submission.rs create mode 100644 crates/j2k-cuda/src/batch/types.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/batch_execution.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/finish/component.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/finish/surface.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/native_batch.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/native_batch/completion.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/native_batch/execution.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/native_batch/prepare.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/native_batch/store.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/native_batch/store/jobs.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/native_batch/store/submission.rs create mode 100644 crates/j2k-cuda/src/decoder/color_batch/single.rs create mode 100644 crates/j2k-cuda/src/decoder/grayscale_batch.rs create mode 100644 crates/j2k-cuda/src/decoder/grayscale_batch/completion.rs create mode 100644 crates/j2k-cuda/src/decoder/grayscale_batch/execution.rs create mode 100644 crates/j2k-cuda/src/decoder/grayscale_batch/preparation.rs create mode 100644 crates/j2k-cuda/src/decoder/grayscale_batch/store.rs create mode 100644 crates/j2k-cuda/src/decoder/pending_completion.rs create mode 100644 crates/j2k-cuda/src/decoder/plan/color.rs create mode 100644 crates/j2k-cuda/src/decoder/plan/color_decoder.rs create mode 100644 crates/j2k-cuda/src/decoder/plan/color_owners/referenced.rs create mode 100644 crates/j2k-cuda/src/decoder/plan/color_referenced.rs create mode 100644 crates/j2k-cuda/src/decoder/plan/grayscale.rs create mode 100644 crates/j2k-cuda/src/decoder/resident/chunked_cleanup.rs create mode 100644 crates/j2k-cuda/src/decoder/resident/chunked_cleanup/enqueue.rs create mode 100644 crates/j2k-cuda/src/decoder/resident/chunked_cleanup/planning.rs create mode 100644 crates/j2k-cuda/src/decoder/resident/chunked_cleanup/tests.rs create mode 100644 crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued.rs create mode 100644 crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued/tests.rs create mode 100644 crates/j2k-cuda/src/decoder/resident/cleanup_dequant/enqueue.rs create mode 100644 crates/j2k-cuda/src/direct_plan/classic/referenced.rs create mode 100644 crates/j2k-cuda/src/session/tests.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/async_resident.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/basic_contracts.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/classic_native.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/exact_color.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/external_lifecycle.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/multitile.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/refinement_overlap.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/rgba.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/session_soak.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/signed_rgb.rs create mode 100644 crates/j2k-cuda/tests/batch_decoder_api/support.rs create mode 100644 crates/j2k-metal-support/src/resident/destination.rs create mode 100644 crates/j2k-metal/src/batch/request.rs create mode 100644 crates/j2k-metal/src/batch/routes.rs create mode 100644 crates/j2k-metal/src/batch/session.rs create mode 100644 crates/j2k-metal/src/batch/tests.rs create mode 100644 crates/j2k-metal/src/batch_decoder.rs create mode 100644 crates/j2k-metal/src/batch_decoder/contracts.rs create mode 100644 crates/j2k-metal/src/batch_decoder/decoder.rs create mode 100644 crates/j2k-metal/src/batch_decoder/encoder_count_tests.rs create mode 100644 crates/j2k-metal/src/batch_decoder/external.rs create mode 100644 crates/j2k-metal/src/batch_decoder/plan_cache.rs create mode 100644 crates/j2k-metal/src/batch_decoder/queue_ordering_tests.rs create mode 100644 crates/j2k-metal/src/batch_decoder/resident.rs create mode 100644 crates/j2k-metal/src/batch_decoder/submission.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_batch.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/classic_subband/tests.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/execution.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/prepared.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/prepared/tests.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/cache.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/cache/referenced.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/parity.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning/fixtures.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning/referenced.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/status.rs create mode 100644 crates/j2k-metal/src/compute/decode_dispatch/store/destination.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/encoder.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/store.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/validation.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution/final_plane.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/destination.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/destination/group_encode.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/destination/submission.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/destination_index_validation.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/grayscale_batch.rs create mode 100644 crates/j2k-metal/src/compute/direct_grayscale_execute/single/execution.rs delete mode 100644 crates/j2k-metal/src/compute/direct_plan_support.rs create mode 100644 crates/j2k-metal/src/compute/direct_plan_validation.rs create mode 100644 crates/j2k-metal/src/compute/direct_plan_validation/runtime.rs create mode 100644 crates/j2k-metal/src/compute/direct_plan_validation/shape.rs create mode 100644 crates/j2k-metal/src/compute/direct_prepare/classic.rs create mode 100644 crates/j2k-metal/src/compute/direct_prepare/color.rs create mode 100644 crates/j2k-metal/src/compute/direct_prepare/grayscale.rs create mode 100644 crates/j2k-metal/src/compute/direct_prepare/ht.rs create mode 100644 crates/j2k-metal/src/compute/direct_prepare/referenced.rs create mode 100644 crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/classic_tier1.rs create mode 100644 crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/final_store.rs create mode 100644 crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/ht_tier1.rs create mode 100644 crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/reconstruction.rs create mode 100644 crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/final_store.rs create mode 100644 crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/reconstruction.rs create mode 100644 crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/tier1.rs delete mode 100644 crates/j2k-metal/src/compute/direct_stacked_batch/result.rs create mode 100644 crates/j2k-metal/src/compute/ht_forward_reader_tests.rs create mode 100644 crates/j2k-metal/src/compute/ht_sigprop_context_tests.rs create mode 100644 crates/j2k-metal/src/compute/resident_codestream/ht_cleanup/tests.rs delete mode 100644 crates/j2k-metal/src/compute/symbol_inventory.rs create mode 100644 crates/j2k-metal/src/compute/tests/capacity.rs create mode 100644 crates/j2k-metal/src/compute/tests/classic.rs create mode 100644 crates/j2k-metal/src/compute/tests/grouping.rs create mode 100644 crates/j2k-metal/src/compute/tests/hybrid.rs create mode 100644 crates/j2k-metal/src/compute/tests/hybrid_support.rs create mode 100644 crates/j2k-metal/src/compute/tests/referenced_plan.rs create mode 100644 crates/j2k-metal/src/compute/tests/referenced_plan/prepared.rs create mode 100644 crates/j2k-metal/src/compute/tests/reuse.rs create mode 100644 crates/j2k-metal/src/compute/tests/roi.rs create mode 100644 crates/j2k-metal/src/compute/tests/runtime.rs create mode 100644 crates/j2k-metal/src/session/direct_plan_cache.rs create mode 100644 crates/j2k-metal/src/session/direct_plan_cache/tests.rs create mode 100644 crates/j2k-metal/src/store_native_color_batch.metal create mode 100644 crates/j2k-metal/tests/batch_classic_color.rs create mode 100644 crates/j2k-metal/tests/batch_decoder_contract.rs create mode 100644 crates/j2k-metal/tests/batch_rgba.rs create mode 100644 crates/j2k-metal/tests/device/auto_tile_batch.rs create mode 100644 crates/j2k-metal/tests/device/batch_sessions.rs create mode 100644 crates/j2k-metal/tests/device/color_batch.rs create mode 100644 crates/j2k-metal/tests/device/decode.rs create mode 100644 crates/j2k-metal/tests/device/direct_gray_requests.rs create mode 100644 crates/j2k-metal/tests/device/direct_repeated.rs create mode 100644 crates/j2k-metal/tests/device/direct_rgb_requests.rs create mode 100644 crates/j2k-metal/tests/device/external_batch.rs create mode 100644 crates/j2k-metal/tests/device/grayscale_external.rs create mode 100644 crates/j2k-metal/tests/device/legacy_batch.rs create mode 100644 crates/j2k-metal/tests/device/multitile_color.rs create mode 100644 crates/j2k-metal/tests/device/multitile_color/batch_inputs.rs create mode 100644 crates/j2k-metal/tests/device/multitile_color/classic.rs create mode 100644 crates/j2k-metal/tests/device/multitile_color/gray12.rs create mode 100644 crates/j2k-metal/tests/device/multitile_color/rgb.rs create mode 100644 crates/j2k-metal/tests/device/multitile_color/signed.rs create mode 100644 crates/j2k-metal/tests/device/resident_batch.rs create mode 100644 crates/j2k-metal/tests/device/tile_batch.rs create mode 100644 crates/j2k-ml/benches/batch_decode.rs create mode 100644 crates/j2k-ml/benches/batch_decode_cuda.rs create mode 100644 crates/j2k-ml/benches/batch_decode_metal.rs create mode 100644 crates/j2k-ml/benches/cuda_telemetry.rs create mode 100644 crates/j2k-ml/benches/metal_telemetry.rs create mode 100644 crates/j2k-ml/benches/support/decode_case.rs create mode 100644 crates/j2k-ml/benches/support/fixture.rs create mode 100644 crates/j2k-ml/benches/support/input_selection.rs create mode 100644 crates/j2k-ml/benches/support/metal_process_policy.rs create mode 100644 crates/j2k-ml/benches/support/mod.rs create mode 100644 crates/j2k-ml/benches/support/process_policy.rs create mode 100644 crates/j2k-ml/benches/support/workload.rs delete mode 100644 crates/j2k-ml/benches/tensor_decode.rs create mode 100644 crates/j2k-ml/src/batch_contract.rs create mode 100644 crates/j2k-ml/src/completion.rs create mode 100644 crates/j2k-ml/src/cpu/batch.rs delete mode 100644 crates/j2k-ml/src/cpu/materialization.rs create mode 100644 crates/j2k-ml/src/cuda/batch.rs delete mode 100644 crates/j2k-ml/src/cuda/config.rs create mode 100644 crates/j2k-ml/src/metal/batch.rs create mode 100644 crates/j2k-ml/src/metal/interop.rs create mode 100644 crates/j2k-ml/tests/batch_sessions.rs create mode 100644 crates/j2k-ml/tests/benchmark_inputs.rs create mode 100644 crates/j2k-ml/tests/cuda_batch_sessions.rs create mode 100644 crates/j2k-ml/tests/cuda_rgba.rs create mode 100644 crates/j2k-ml/tests/metal/native_color.rs create mode 100644 crates/j2k-ml/tests/metal/requests.rs create mode 100644 crates/j2k-ml/tests/metal/sessions.rs create mode 100644 crates/j2k-native/fixtures/htj2k/LICENSE.OpenJPH create mode 100644 crates/j2k-native/fixtures/htj2k/gray_u12_53.j2c create mode 100644 crates/j2k-native/fixtures/htj2k/gray_u12_53.oracle.raw create mode 100644 crates/j2k-native/fixtures/htj2k/rgb_u12_53.j2c create mode 100644 crates/j2k-native/fixtures/htj2k/rgb_u12_53.oracle.raw create mode 100644 crates/j2k-native/src/direct_cpu/allocation/referenced.rs create mode 100644 crates/j2k-native/src/direct_cpu/allocation/referenced/budget.rs create mode 100644 crates/j2k-native/src/direct_cpu/allocation/referenced/storage.rs create mode 100644 crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs create mode 100644 crates/j2k-native/src/direct_cpu/color.rs create mode 100644 crates/j2k-native/src/direct_cpu/component.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced/component.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced/output.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced/payload.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced_classic.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced_staged.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced_staged/entropy.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced_staged/finish.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced_staged/plan_access.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced_staged/prepare.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced_staged/state.rs create mode 100644 crates/j2k-native/src/direct_plan/referenced_classic.rs create mode 100644 crates/j2k-native/src/direct_plan/referenced_ht.rs create mode 100644 crates/j2k-native/src/image/direct_api/referenced.rs create mode 100644 crates/j2k-native/src/j2c/decode/direct_plan/color.rs create mode 100644 crates/j2k-native/src/j2c/decode/direct_plan/referenced_color.rs create mode 100644 crates/j2k-native/src/j2c/decode/direct_plan/referenced_grayscale.rs create mode 100644 crates/j2k-native/src/j2c/decode/direct_plan/storage.rs create mode 100644 crates/j2k-native/src/j2c/decode/direct_plan/storage/sub_band.rs create mode 100644 crates/j2k-native/src/j2c/decode/workspace.rs create mode 100644 crates/j2k-native/src/tests/workspace_reuse.rs create mode 100644 crates/j2k-native/tests/referenced_multitile.rs create mode 100644 crates/j2k-test-support/fixtures/htj2k/openhtj2k_hifi_ht1_02.j2k create mode 100644 crates/j2k-test-support/fixtures/htj2k/openhtj2k_hifi_ht1_02.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openhtj2k_sigprop_refinement_overlap.j2k create mode 100644 crates/j2k-test-support/fixtures/htj2k/openhtj2k_sigprop_refinement_overlap.openht.oracle.ppm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openhtj2k_sigprop_refinement_overlap.openjph.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/LICENSE.OpenJPH create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/README.md create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/encode_signed_rgb.cpp create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/generate.rs create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_53.source.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_97.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_97.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_97.source.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_53.source.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_97.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_97.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_97.source.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s8_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s8_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s8_53.source.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.jph create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.source.pgm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_97.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_97.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_97.source.pgm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_53.source.pgm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_97.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_97.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_97.source.pgm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u8_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u8_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u8_53.source.pgm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s12_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s12_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s12_53_single.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s12_97.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s12_97.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s16_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s16_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s16_53_single.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s8_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s8_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s8_53_single.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u12_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u12_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u12_53.source.ppm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u12_97.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u12_97.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u12_97.source.ppm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u16_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u16_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u16_53.source.ppm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_53.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_53.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_53.source.ppm create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_97.j2c create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_97.oracle.raw create mode 100644 crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_97.source.ppm create mode 100644 crates/j2k-test-support/src/fixtures/generated_htj2k.rs create mode 100644 crates/j2k-test-support/src/fixtures/jp2.rs create mode 100644 crates/j2k-test-support/src/fixtures/jpeg.rs create mode 100644 crates/j2k-test-support/src/fixtures/openjph.rs create mode 100644 crates/j2k-types/src/decode_payload.rs create mode 100644 crates/j2k/src/batch/scheduler/retained.rs create mode 100644 crates/j2k/src/batch/worker/owned.rs create mode 100644 crates/j2k/src/decode/settings.rs create mode 100644 crates/j2k/src/owned_batch.rs create mode 100644 crates/j2k/src/owned_batch/contracts.rs create mode 100644 crates/j2k/src/owned_batch/cpu.rs create mode 100644 crates/j2k/src/owned_batch/cpu_diagnostics.rs create mode 100644 crates/j2k/src/owned_batch/cpu_execute.rs create mode 100644 crates/j2k/src/owned_batch/cpu_fast.rs create mode 100644 crates/j2k/src/owned_batch/cpu_fast/plan.rs create mode 100644 crates/j2k/src/owned_batch/cpu_materialize.rs create mode 100644 crates/j2k/src/owned_batch/cpu_prepared.rs create mode 100644 crates/j2k/src/owned_batch/cpu_staged.rs create mode 100644 crates/j2k/src/owned_batch/cpu_staged_execute.rs create mode 100644 crates/j2k/src/owned_batch/errors.rs create mode 100644 crates/j2k/src/owned_batch/prepare.rs create mode 100644 crates/j2k/src/owned_batch/prepare/image.rs create mode 100644 crates/j2k/src/owned_batch/prepared.rs create mode 100644 crates/j2k/src/owned_batch/prepared_plan.rs create mode 100644 crates/j2k/tests/owned_batch.rs create mode 100644 crates/j2k/tests/owned_batch/cpu_reuse.rs create mode 100644 crates/j2k/tests/owned_batch/errors_and_requests.rs create mode 100644 crates/j2k/tests/owned_batch/fixtures.rs create mode 100644 crates/j2k/tests/owned_batch/grouping.rs create mode 100644 crates/j2k/tests/owned_batch/native_types_and_requests.rs create mode 100644 crates/j2k/tests/owned_batch/oracles.rs create mode 100644 crates/j2k/tests/owned_batch/payload_plan.rs create mode 100644 crates/j2k/tests/owned_batch/payload_ranges.rs create mode 100644 crates/j2k/tests/owned_batch/reuse.rs create mode 100644 crates/j2k/tests/owned_batch/rgba.rs create mode 100644 crates/j2k/tests/owned_batch_fixtures.rs create mode 100644 crates/j2k/tests/owned_batch_fixtures/classic.rs create mode 100644 crates/j2k/tests/owned_batch_fixtures/ht_matrix.rs create mode 100644 crates/j2k/tests/owned_batch_fixtures/irreversible.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/.cargo-ok create mode 100644 third_party/cubecl-cuda-0.10.0-patched/.cargo_vcs_info.json create mode 100644 third_party/cubecl-cuda-0.10.0-patched/Cargo.lock create mode 100644 third_party/cubecl-cuda-0.10.0-patched/Cargo.toml create mode 100644 third_party/cubecl-cuda-0.10.0-patched/Cargo.toml.orig create mode 100644 third_party/cubecl-cuda-0.10.0-patched/LICENSE-APACHE create mode 100644 third_party/cubecl-cuda-0.10.0-patched/LICENSE-MIT create mode 100644 third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md create mode 100644 third_party/cubecl-cuda-0.10.0-patched/README.md create mode 100644 third_party/cubecl-cuda-0.10.0-patched/build.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/cubecl.toml create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/command.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/communication.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/context.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/io/controller.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/io/mod.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/mod.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/server.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/cpu.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/gpu.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/mod.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/stream.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/sync/fence.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/compute/sync/mod.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/device.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/lib.rs create mode 100644 third_party/cubecl-cuda-0.10.0-patched/src/runtime.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/.cargo-ok create mode 100644 third_party/cubecl-runtime-0.10.0-patched/.cargo_vcs_info.json create mode 100644 third_party/cubecl-runtime-0.10.0-patched/Cargo.lock create mode 100644 third_party/cubecl-runtime-0.10.0-patched/Cargo.toml create mode 100644 third_party/cubecl-runtime-0.10.0-patched/Cargo.toml.orig create mode 100644 third_party/cubecl-runtime-0.10.0-patched/LICENSE-APACHE create mode 100644 third_party/cubecl-runtime-0.10.0-patched/LICENSE-MIT create mode 100644 third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md create mode 100644 third_party/cubecl-runtime-0.10.0-patched/README.md create mode 100644 third_party/cubecl-runtime-0.10.0-patched/benches/dynamic.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/build.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/allocator.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/client.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/compiler.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/config/autotune.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/config/base.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/config/cache.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/config/compilation.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/config/logger.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/config/memory.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/config/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/config/profiling.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/config/streaming.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/id.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/kernel.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/lib.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/logging/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/logging/profiling.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/logging/server.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/base.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/policy.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/queue.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_manage.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/base.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/exclusive_pool.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/handle.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/memory_page.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/persistent_pool.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/sliced_pool.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/memory_management/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/runtime.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/server/base.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/server/handle.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/server/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/storage/base.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/storage/bytes_cpu.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/storage/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/stream/base.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/stream/event.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/stream/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/stream/scheduler.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/timestamp_profiler.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tma.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/base.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/input_generator.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/key_generator.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/local.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/operation.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_benchmark.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_cache.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_inputs.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/tuner.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/tune/util.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/src/validation.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/tests/dummy/compute.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/tests/dummy/kernel.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/tests/dummy/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/tests/dummy/server.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/autotune_operations.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/kernels.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/mod.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/operation_sets.rs create mode 100644 third_party/cubecl-runtime-0.10.0-patched/tests/integration_test.rs create mode 100644 third_party/wgpu-29.0.4-patched/.cargo-ok create mode 100644 third_party/wgpu-29.0.4-patched/.cargo_vcs_info.json create mode 100644 third_party/wgpu-29.0.4-patched/Cargo.lock create mode 100644 third_party/wgpu-29.0.4-patched/Cargo.toml create mode 100644 third_party/wgpu-29.0.4-patched/Cargo.toml.orig create mode 100644 third_party/wgpu-29.0.4-patched/LICENSE.APACHE create mode 100644 third_party/wgpu-29.0.4-patched/LICENSE.MIT create mode 100644 third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md create mode 100644 third_party/wgpu-29.0.4-patched/README.md create mode 100644 third_party/wgpu-29.0.4-patched/build.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/adapter.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/bind_group.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/bind_group_layout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/blas.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/buffer.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/command_buffer.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/command_buffer_actions.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/command_encoder.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/common_pipeline.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/compute_pass.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/compute_pipeline.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/device.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/external_texture.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/instance.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/mod.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/pipeline_cache.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/pipeline_layout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/query_set.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/queue.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/render_bundle.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/render_bundle_encoder.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/render_pass.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/render_pipeline.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/sampler.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/shader_module.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/surface.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/surface_texture.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/texture.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/texture_view.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/api/tlas.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/custom.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/mod.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/defined_non_null_js_value.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/ext_bindings.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_Gpu.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAdapter.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAdapterInfo.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAddressMode.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAutoLayoutMode.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroup.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupEntry.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayoutDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayoutEntry.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendComponent.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendFactor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendOperation.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendState.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBuffer.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBinding.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBindingLayout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBindingType.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferMapState.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasAlphaMode.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasConfiguration.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasContext.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasToneMapping.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasToneMappingMode.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuColorDict.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuColorTargetState.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandBuffer.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandBufferDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandEncoder.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandEncoderDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompareFunction.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationInfo.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationMessage.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationMessageType.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassEncoder.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassTimestampWrites.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePipeline.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePipelineDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCopyExternalImageDestInfo.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCopyExternalImageSourceInfo.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCullMode.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDepthStencilState.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDevice.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceLostInfo.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceLostReason.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuError.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuErrorFilter.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExtent3dDict.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTexture.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTextureBindingLayout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTextureDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFeatureName.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFilterMode.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFragmentState.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFrontFace.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuIndexFormat.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuLoadOp.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuMipmapFilterMode.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuMultisampleState.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuObjectDescriptorBase.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOrigin2dDict.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOrigin3dDict.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOutOfMemoryError.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineDescriptorBase.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineLayout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineLayoutDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPowerPreference.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPrimitiveState.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPrimitiveTopology.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuProgrammableStage.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQuerySet.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQuerySetDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueryType.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueue.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueueDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundle.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleEncoder.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleEncoderDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassColorAttachment.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassDepthStencilAttachment.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassEncoder.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassTimestampWrites.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPipeline.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPipelineDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRequestAdapterOptions.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSampler.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerBindingLayout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerBindingType.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuShaderModule.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuShaderModuleDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStencilFaceState.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStencilOperation.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStorageTextureAccess.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStorageTextureBindingLayout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStoreOp.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSupportedFeatures.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSupportedLimits.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyBufferInfo.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyBufferLayout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyTextureInfo.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexture.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureAspect.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureBindingLayout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureDimension.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureFormat.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureSampleType.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureView.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureViewDescriptor.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureViewDimension.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuUncapturedErrorEvent.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuUncapturedErrorEventInit.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuValidationError.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexAttribute.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexBufferLayout.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexFormat.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexState.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexStepMode.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_WgslLanguageFeatures.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_gpu_map_mode.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/mod.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/wgpu_core.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/backend/wgpu_core/thread_id.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/cmp.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/dispatch.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/lib.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/macros/be-aligned.spv create mode 100644 third_party/wgpu-29.0.4-patched/src/macros/le-aligned.spv create mode 100644 third_party/wgpu-29.0.4-patched/src/macros/mod.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/util/belt.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/util/blit.wgsl create mode 100644 third_party/wgpu-29.0.4-patched/src/util/device.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/util/encoder.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/util/init.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/util/mod.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/util/mutex.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/util/panicking.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/util/spirv.rs create mode 100644 third_party/wgpu-29.0.4-patched/src/util/texture_blitter.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/.cargo-ok create mode 100644 third_party/wgpu-core-29.0.4-patched/.cargo_vcs_info.json create mode 100644 third_party/wgpu-core-29.0.4-patched/Cargo.lock create mode 100644 third_party/wgpu-core-29.0.4-patched/Cargo.toml create mode 100644 third_party/wgpu-core-29.0.4-patched/Cargo.toml.orig create mode 100644 third_party/wgpu-core-29.0.4-patched/LICENSE.APACHE create mode 100644 third_party/wgpu-core-29.0.4-patched/LICENSE.MIT create mode 100644 third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md create mode 100644 third_party/wgpu-core-29.0.4-patched/build.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/as_hal.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/binding_model.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/allocator.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/bind.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/bundle.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/clear.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/compute.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/compute_command.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/draw.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/encoder.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/encoder_command.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/ffi.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/memory_init.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/mod.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/pass.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/query.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/ray_tracing.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/render.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/render_command.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/timestamp_writes.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/transfer.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/command/transition_resources.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/conv.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/bgl.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/global.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/life.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/mod.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/queue.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/ray_tracing.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/resource.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/trace.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/trace/record.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/device/trace/replay.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/error.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/global.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/hash_utils.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/hub.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/id.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/identity.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/indirect_validation/dispatch.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/indirect_validation/draw.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/indirect_validation/mod.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/indirect_validation/utils.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/indirect_validation/validate_draw.wgsl create mode 100644 third_party/wgpu-core-29.0.4-patched/src/init_tracker/buffer.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/init_tracker/mod.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/init_tracker/texture.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/instance.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/lib.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/lock/mod.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/lock/observing.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/lock/rank.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/lock/ranked.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/lock/vanilla.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/pipeline.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/pipeline_cache.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/pool.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/present.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/ray_tracing.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/registry.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/resource.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/scratch.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/snatch.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/storage.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/common.wgsl create mode 100644 third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/mod.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/timestamp_normalization.wgsl create mode 100644 third_party/wgpu-core-29.0.4-patched/src/track/blas.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/track/buffer.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/track/metadata.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/track/mod.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/track/range.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/track/stateless.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/track/texture.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/validation.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/validation/shader_io_deductions.rs create mode 100644 third_party/wgpu-core-29.0.4-patched/src/weak_vec.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/.cargo-ok create mode 100644 third_party/wgpu-hal-29.0.4-patched/.cargo_vcs_info.json create mode 100644 third_party/wgpu-hal-29.0.4-patched/Cargo.lock create mode 100644 third_party/wgpu-hal-29.0.4-patched/Cargo.toml create mode 100644 third_party/wgpu-hal-29.0.4-patched/Cargo.toml.orig create mode 100644 third_party/wgpu-hal-29.0.4-patched/LICENSE.APACHE create mode 100644 third_party/wgpu-hal-29.0.4-patched/LICENSE.MIT create mode 100644 third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md create mode 100644 third_party/wgpu-hal-29.0.4-patched/README.md create mode 100644 third_party/wgpu-hal-29.0.4-patched/build.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/examples/halmark/main.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/examples/halmark/shader.wgsl create mode 100644 third_party/wgpu-hal-29.0.4-patched/examples/raw-gles.em.html create mode 100644 third_party/wgpu-hal-29.0.4-patched/examples/raw-gles.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/examples/ray-traced-triangle/main.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/examples/ray-traced-triangle/shader.wgsl create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/conv.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/exception.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/factory.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/mod.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/name.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/result.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/time.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/auxil/mod.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/auxil/renderdoc.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/adapter.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/command.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/conv.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/dcomp.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/descriptor.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/device.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/device_creation.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/instance.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/mod.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/pipeline_desc.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/sampler.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/shader_compilation.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/suballocation.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/types.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dx12/view.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dynamic/adapter.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dynamic/command.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dynamic/device.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dynamic/instance.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dynamic/mod.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dynamic/queue.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/dynamic/surface.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/adapter.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/command.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/conv.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/device.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/egl.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/emscripten.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/fence.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/mod.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/queue.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/clear.frag create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/clear.vert create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/srgb_present.frag create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/srgb_present.vert create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/web.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/gles/wgl.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/lib.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/metal/adapter.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/metal/command.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/metal/conv.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/metal/device.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/metal/library_from_metallib.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/metal/mod.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/metal/surface.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/metal/time.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/noop/buffer.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/noop/command.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/noop/mod.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/validation_canary.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/adapter.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/command.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/conv.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/device.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/drm.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/instance.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/mod.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/sampler.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/semaphore_list.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/swapchain/mod.rs create mode 100644 third_party/wgpu-hal-29.0.4-patched/src/vulkan/swapchain/native.rs create mode 100644 xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy/shared_codec.rs create mode 100644 xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_batch_api_structure_policy.rs create mode 100644 xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy/allocations.rs create mode 100644 xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy/color_runtime/async_cleanup.rs create mode 100644 xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy/color_runtime/owner_completion.rs create mode 100644 xtask/tests/repo_lint_support/gpu_adapter_policy/metal_batch_structure_policy.rs create mode 100644 xtask/tests/repo_lint_support/j2k_ml_policy/benchmark_support_structure.rs create mode 100644 xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution.rs create mode 100644 xtask/tests/repo_lint_support/metal_compute_structure_policy/compute_tests.rs create mode 100644 xtask/tests/repo_lint_support/metal_compute_structure_policy/direct_plan_ratchets.rs create mode 100644 xtask/tests/repo_lint_support/metal_compute_symbol_policy.rs create mode 100644 xtask/tests/repo_lint_support/metal_direct_plan_structure_policy.rs create mode 100644 xtask/tests/repo_lint_support/native_decode_allocation_policy/downstream.rs create mode 100644 xtask/tests/repo_lint_support/native_direct_plan_structure_policy.rs diff --git a/Cargo.lock b/Cargo.lock index c70cee0f..2fd49519 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -255,7 +255,6 @@ dependencies = [ "hashbrown 0.16.1", "log", "num-traits", - "parking_lot", "portable-atomic", "spin", ] @@ -1038,8 +1037,6 @@ dependencies = [ [[package]] name = "cubecl-cuda" version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b0a69ff45688d322ad8e92c8bf645167b9ca490fa8fa087fc6adac8c5e46be" dependencies = [ "bytemuck", "cubecl-common", @@ -1156,8 +1153,6 @@ dependencies = [ [[package]] name = "cubecl-runtime" version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b68491bf5b3e997ae36bdc4e63b4ccd6d2f0e86b3b596a5d7a48d2b9e92622a0" dependencies = [ "ahash", "async-channel", @@ -2705,7 +2700,6 @@ dependencies = [ name = "j2k-ml" version = "0.7.4" dependencies = [ - "burn-autodiff", "burn-core", "burn-cubecl", "burn-cuda", @@ -2716,11 +2710,15 @@ dependencies = [ "burn-wgpu", "criterion", "j2k", + "j2k-core", "j2k-cuda", "j2k-cuda-runtime", "j2k-metal", + "j2k-native", "j2k-test-support", + "metal", "thiserror 2.0.18", + "wgpu-hal", ] [[package]] @@ -5150,8 +5148,6 @@ checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" [[package]] name = "wgpu" version = "29.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76e8840e1ba2881d4cbb18d2147627a56af426ff064c0401eb0c8410c6325d07" dependencies = [ "arrayvec", "bitflags", @@ -5180,8 +5176,6 @@ dependencies = [ [[package]] name = "wgpu-core" version = "29.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7" dependencies = [ "arrayvec", "bit-set 0.9.1", @@ -5240,8 +5234,6 @@ dependencies = [ [[package]] name = "wgpu-hal" version = "29.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d" dependencies = [ "android_system_properties", "arrayvec", diff --git a/Cargo.toml b/Cargo.toml index eb93089a..e25acf78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,14 @@ members = [ "crates/j2k-cli", "xtask", ] -exclude = ["third_party/block-0.1.6-patched"] +exclude = [ + "third_party/block-0.1.6-patched", + "third_party/cubecl-cuda-0.10.0-patched", + "third_party/cubecl-runtime-0.10.0-patched", + "third_party/wgpu-29.0.4-patched", + "third_party/wgpu-core-29.0.4-patched", + "third_party/wgpu-hal-29.0.4-patched", +] [workspace.package] version = "0.7.4" @@ -80,6 +87,20 @@ burn-ir = { version = "0.21.0", default-features = false, features = ["std"] } # Rust hard error. Keep this patch minimal and remove it when `metal` publishes # a fixed dependency path. block = { path = "third_party/block-0.1.6-patched" } +# Burn 0.21 / CubeCL 0.10 does not expose the owning CUDA stream required to +# register event dependencies around an external codec write. These patches +# add one hidden, binding-resolving stream hook; the raw token remains confined +# to j2k-ml's guarded interop module. +cubecl-cuda = { path = "third_party/cubecl-cuda-0.10.0-patched" } +cubecl-runtime = { path = "third_party/cubecl-runtime-0.10.0-patched" } +# Burn 0.21 uses wgpu 29, whose Metal HAL does not expose an ownership-safe +# MTLBuffer/MTLDevice bridge and whose lazy buffer initialization tracker +# cannot otherwise learn about completed external GPU writes. These local +# patches add retained raw-handle accessors plus one guarded, range-scoped +# initialization handoff used inside j2k-ml's external-write boundary. +wgpu = { path = "third_party/wgpu-29.0.4-patched" } +wgpu-core = { path = "third_party/wgpu-core-29.0.4-patched" } +wgpu-hal = { path = "third_party/wgpu-hal-29.0.4-patched" } # Shared strict-lint policy for safe crates. Crates with genuinely different # needs (SIMD/FFI crates that use `unsafe`, crates with documented temporary diff --git a/README.md b/README.md index 14c0c11f..914e7d98 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,47 @@ CUDA paths use J2K-owned CUDA Oxide device kernels through `cuda-runtime`. NVIDIA performance claims require self-hosted benchmark evidence; hosted CI is not treated as NVIDIA performance evidence. +## High-throughput owned batches + +The additive owned-batch API accepts `EncodedImage` values containing an +`Arc<[u8]>` and one of `Full`, `Region`, `Reduced`, or `RegionReduced`. It +prepares inputs concurrently, keeps unlike output shapes in separate groups +without padding, and returns source indices and indexed failures. Representable +Gray, RGB, and RGBA groups use exact native `U8`, `U16`, or `I16` storage in +NCHW or NHWC order; float conversion and normalization are deliberately not +codec operations. + +Preparation can retain either a `PreparedHtj2kPlan` or a +`PreparedClassicPlan` with per-tile packet, code-block, and destination +geometry. Both plans reference compressed payload ranges inside the original +`Arc<[u8]>`; neither duplicates the codestream. `CpuBatchDecoder` consumes +single- and multi-tile plans without reparsing. Inputs outside the retained-plan +boundary can remain metadata-only and use the broader CPU codec when that path +supports them. + +`j2k-cuda` and `j2k-metal` own persistent accelerator sessions, resident +output, and validated caller-owned destinations. Their direct final stores +produce the requested native dtype and layout in the destination allocation, +so decoded pixels do not make a GPU-to-CPU-to-GPU round trip. The validated +Metal fast-batch path covers the single-tile classic JPEG 2000 and HTJ2K +Gray/RGB/RGBA `U8`/`U16`/`I16` matrix for all four requests and both layouts. +Metal hardware tests additionally cover exact multi-tile HT Gray12 and RGB8 +for all four requests, plus classic RGB8 full decode. CUDA has +RTX 4070 hardware validation for the classic and HT Gray/RGB/RGBA +`U8`/`U16`/`I16` fast-batch matrix, all four requests, both layouts, and +codec-resident, external-destination, and Burn-direct output. The release lane +also covers classic and HT multi-tile regressions, asynchronous drop and +session reuse, and long-running reuse soaks. Reversible output is bit-exact; +irreversible 9/7 output agrees with the CPU oracle within one integer LSB. +Codestream RGN/maxshift reconstruction still fails closed on both GPU prepared +routes; it never triggers CPU staging. + +`j2k-ml` is only the thin Burn allocation and synchronization adapter over +these codec sessions. Readers such as `wsi-rs` remain responsible for finding +and supplying encoded image bytes. The new HT-dominant publication throughput +matrix has not yet been run, so these API and correctness guarantees are not a +published speedup claim. + ## Which crate should I use? Use `cargo add j2k` for JPEG 2000 / HTJ2K application code. Lower-level @@ -139,7 +180,7 @@ Use lower-level crates only when you need a specific integration point: | JPEG-to-HTJ2K coefficient-domain transcode | `j2k-transcode` | | CUDA adapters | `j2k-jpeg-cuda`, `j2k-cuda`, `j2k-transcode-cuda` | | Metal adapters | `j2k-jpeg-metal`, `j2k-metal`, `j2k-transcode-metal` | -| Experimental Burn 0.21 tensor decode integration | `j2k-ml` (unpublished) | +| Experimental Burn 0.21 native integer batch adapter | `j2k-ml` (unpublished) | | Tile compression codecs | `j2k-tilecodec` | | Command-line inspection and JPEG-to-HTJ2K smoke transcode | `j2k-cli` | @@ -153,6 +194,7 @@ The names `statumen` and `wsi-dicom` are not current package names. | JPEG 2000 Part 1 decode | Full-frame, ROI, scaled, row, tile-batch, and component-plane API surfaces | CPU is the portable correctness baseline. | | JPEG 2000 Part 1 encode | Native Rust encode APIs for codestream and JP2 output, including component-plane metadata | Stable public API is centered on `j2k`; adapter SPI remains experimental. | | HTJ2K Part 15 inspect/decode/encode | Raw HT codestreams and JPH still-image files, including cleanup and refinement paths | HT requests beyond the Part 15 coded-bitplane limit reject explicitly. | +| Owned batch decode | Prepared or one-shot Gray/RGB/RGBA groups in NCHW/NHWC with exact `U8`, `U16`, or `I16` samples | Full, ROI, reduced, and ROI-plus-reduced requests are supported. CPU consumes retained single- and multi-tile plans. Metal has hardware validation for the single-tile matrix and selected exact multi-tile paths. CUDA has RTX 4070 validation for the full fast-batch dtype/request/layout matrix, resident and external destinations, Burn-direct output, and multi-tile regressions. GPU RGN/maxshift inputs fail closed. | | JP2/JPH metadata | IHDR/COLR/BPCC/PCLR/CMAP/CDEF/ICC still-image metadata paths covered by repo-local tests | Broader external JP2/JPH metadata parity remains publication evidence. | | Recode | J2K-to-HTJ2K coefficient recode where valid, pixel-preserving fallback otherwise | Palette/component-mapped fallbacks intentionally drop mapping metadata after resolving pixels. | | JPEG input | JPEG inspect/decode through `j2k-jpeg` | Used by transcode and fixture workflows. | @@ -231,8 +273,8 @@ Reference files: environment variables - [docs/public-support.md](docs/public-support.md) - exact J2K Part 1, HTJ2K Part 15, JP2/JPH, and out-of-scope support boundary -- [docs/j2k-ml.md](docs/j2k-ml.md) - Burn tensor layouts, normalization, - batching, and accelerator route guarantees +- [docs/j2k-ml.md](docs/j2k-ml.md) - Burn native integer batch groups, + prepared reuse, and direct accelerator destination guarantees - [docs/release.md](docs/release.md) - release and package validation policy - [docs/stable-api-1.0.md](docs/stable-api-1.0.md) - stable API snapshot policy - [CHANGELOG.md](CHANGELOG.md) - current release notes diff --git a/crates/j2k-core/src/batch.rs b/crates/j2k-core/src/batch.rs index de3182ab..d6a5bb25 100644 --- a/crates/j2k-core/src/batch.rs +++ b/crates/j2k-core/src/batch.rs @@ -6,6 +6,7 @@ use crate::{backend::BackendRequest, pixel::PixelFormat, scale::Downscale, types mod allocation; mod collection; +mod gpu_job_chunk; #[doc(hidden)] pub use allocation::{ @@ -15,6 +16,12 @@ pub use allocation::{ pub use collection::{ try_collect_indexed_batch_results, try_collect_ordered_batch_results_with_limits, }; +#[doc(hidden)] +pub use gpu_job_chunk::{ + plan_ht_gpu_job_chunks, HtGpuJobChunk, HtGpuJobChunkEntry, HtGpuJobChunkLimit, + HtGpuJobChunkLimits, HtGpuJobChunkPlan, HtGpuJobChunkPlanError, HtGpuJobChunkRequest, + HtGpuJobPassBucket, +}; /// Worker configuration for CPU tile batches. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -132,6 +139,12 @@ pub enum BatchInfrastructureError { /// An internal planning boundary was invoked without any submitted jobs. #[error("batch plan requires at least one job")] EmptyBatchPlan, + /// A newer shared batch contract cannot be represented by this consumer. + #[error("unsupported batch contract: {what}")] + UnsupportedContract { + /// Contract element that the current consumer cannot represent. + what: &'static str, + }, /// Batch-owned metadata would exceed the operation's host-memory budget. #[error("{what} is too large: requested {requested} bytes, cap {cap}")] AllocationTooLarge { diff --git a/crates/j2k-core/src/batch/gpu_job_chunk.rs b/crates/j2k-core/src/batch/gpu_job_chunk.rs new file mode 100644 index 00000000..4ae69128 --- /dev/null +++ b/crates/j2k-core/src/batch/gpu_job_chunk.rs @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Bounded, backend-neutral HTJ2K GPU job chunk planning. + +use alloc::vec::Vec; +use core::num::NonZeroUsize; + +use super::BatchInfrastructureError; + +mod planner; +pub use self::planner::plan_ht_gpu_job_chunks; + +/// HTJ2K coding-pass family used to keep cleanup-heavy jobs on a fused path. +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum HtGpuJobPassBucket { + /// One cleanup pass and no refinement payload. + CleanupOnly, + /// Cleanup followed by one `SigProp` pass. + SigProp, + /// Cleanup, `SigProp`, and `MagRef`, including any future pass count above three. + MagRef, +} + +/// One flattened HTJ2K job submitted to the shared chunk planner. +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct HtGpuJobChunkRequest { + source_index: usize, + coding_passes: u8, + payload_bytes: usize, + descriptor_bytes: usize, +} + +impl HtGpuJobChunkRequest { + /// Describe one job without borrowing backend-specific payload or descriptor types. + #[doc(hidden)] + #[must_use] + pub const fn new( + source_index: usize, + coding_passes: u8, + payload_bytes: usize, + descriptor_bytes: usize, + ) -> Self { + Self { + source_index, + coding_passes, + payload_bytes, + descriptor_bytes, + } + } + + /// Caller input index that owns this job. + #[doc(hidden)] + #[must_use] + pub const fn source_index(self) -> usize { + self.source_index + } + + /// Number of HT coding passes represented by the job. + #[doc(hidden)] + #[must_use] + pub const fn coding_passes(self) -> u8 { + self.coding_passes + } + + /// Compressed cleanup and refinement payload bytes. + #[doc(hidden)] + #[must_use] + pub const fn payload_bytes(self) -> usize { + self.payload_bytes + } + + /// Backend descriptor-arena bytes required by the job. + #[doc(hidden)] + #[must_use] + pub const fn descriptor_bytes(self) -> usize { + self.descriptor_bytes + } +} + +/// Per-chunk limits enforced independently for every pass bucket. +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct HtGpuJobChunkLimits { + jobs: NonZeroUsize, + payload_bytes: usize, + descriptor_bytes: usize, +} + +impl HtGpuJobChunkLimits { + /// Construct explicit job, compressed-payload, and descriptor-arena limits. + #[doc(hidden)] + #[must_use] + pub const fn new( + max_jobs: NonZeroUsize, + max_payload_bytes: usize, + max_descriptor_bytes: usize, + ) -> Self { + Self { + jobs: max_jobs, + payload_bytes: max_payload_bytes, + descriptor_bytes: max_descriptor_bytes, + } + } + + /// Maximum jobs in one chunk. + #[doc(hidden)] + #[must_use] + pub const fn max_jobs(self) -> NonZeroUsize { + self.jobs + } + + /// Maximum compressed payload bytes in one chunk. + #[doc(hidden)] + #[must_use] + pub const fn max_payload_bytes(self) -> usize { + self.payload_bytes + } + + /// Maximum backend descriptor bytes in one chunk. + #[doc(hidden)] + #[must_use] + pub const fn max_descriptor_bytes(self) -> usize { + self.descriptor_bytes + } +} + +/// Limit dimension exceeded by one job before any chunk is constructed. +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum HtGpuJobChunkLimit { + /// Compressed cleanup and refinement payload bytes. + PayloadBytes, + /// Backend descriptor-arena bytes. + DescriptorBytes, +} + +/// Failure returned by bounded HTJ2K GPU chunk planning. +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum HtGpuJobChunkPlanError { + /// A submitted job does not contain the mandatory cleanup pass. + #[error( + "HTJ2K source {source_index} job {original_job_index} has invalid coding-pass count {coding_passes}" + )] + InvalidCodingPassCount { + /// Caller input index that owns the invalid job. + source_index: usize, + /// Position of the invalid job in the submitted flattened job slice. + original_job_index: usize, + /// Invalid coding-pass count. + coding_passes: u8, + }, + /// One job cannot fit an otherwise valid empty chunk. + #[error( + "HTJ2K source {source_index} job {original_job_index} requires {requested} {limit:?}, cap {cap}" + )] + SingleJobTooLarge { + /// Caller input index that owns the oversized job. + source_index: usize, + /// Position of the oversized job in the submitted flattened job slice. + original_job_index: usize, + /// Limit dimension exceeded by the job. + limit: HtGpuJobChunkLimit, + /// Bytes required by the single job. + requested: usize, + /// Configured per-chunk byte cap. + cap: usize, + }, + /// Host planning metadata could not be represented or allocated safely. + #[error(transparent)] + BatchInfrastructure(#[from] BatchInfrastructureError), +} + +/// Stable identity retained for one job after pass-family bucketing. +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct HtGpuJobChunkEntry { + original_job_index: usize, + source_index: usize, +} + +impl HtGpuJobChunkEntry { + pub(super) const fn new(original_job_index: usize, source_index: usize) -> Self { + Self { + original_job_index, + source_index, + } + } + + /// Position in the caller's original flattened job slice. + #[doc(hidden)] + #[must_use] + pub const fn original_job_index(self) -> usize { + self.original_job_index + } + + /// Caller input index that owns this job. + #[doc(hidden)] + #[must_use] + pub const fn source_index(self) -> usize { + self.source_index + } +} + +/// One bounded, pass-homogeneous range in [`HtGpuJobChunkPlan::entries`]. +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct HtGpuJobChunk { + bucket: HtGpuJobPassBucket, + entry_start: usize, + entry_end: usize, + payload_bytes: usize, + descriptor_bytes: usize, +} + +impl HtGpuJobChunk { + pub(super) const fn new( + bucket: HtGpuJobPassBucket, + entry_start: usize, + entry_end: usize, + payload_bytes: usize, + descriptor_bytes: usize, + ) -> Self { + Self { + bucket, + entry_start, + entry_end, + payload_bytes, + descriptor_bytes, + } + } + + /// Cleanup-only, SigProp, or MagRef pass family for every entry. + #[doc(hidden)] + #[must_use] + pub const fn bucket(self) -> HtGpuJobPassBucket { + self.bucket + } + + /// Number of jobs in the chunk. + #[doc(hidden)] + #[must_use] + pub const fn job_count(self) -> usize { + self.entry_end - self.entry_start + } + + /// Aggregate compressed payload bytes in the chunk. + #[doc(hidden)] + #[must_use] + pub const fn payload_bytes(self) -> usize { + self.payload_bytes + } + + /// Aggregate backend descriptor bytes in the chunk. + #[doc(hidden)] + #[must_use] + pub const fn descriptor_bytes(self) -> usize { + self.descriptor_bytes + } +} + +/// Complete bounded plan with flat stable-identity storage shared by all chunks. +#[doc(hidden)] +#[derive(Debug, Default, PartialEq, Eq)] +pub struct HtGpuJobChunkPlan { + chunks: Vec, + entries: Vec, +} + +impl HtGpuJobChunkPlan { + pub(super) const fn new(chunks: Vec, entries: Vec) -> Self { + Self { chunks, entries } + } + + /// Chunks ordered cleanup-only, SigProp, then MagRef; stable within each bucket. + #[doc(hidden)] + #[must_use] + pub fn chunks(&self) -> &[HtGpuJobChunk] { + &self.chunks + } + + /// Flat stable-identity entries referenced by the chunk descriptors. + #[doc(hidden)] + #[must_use] + pub fn entries(&self) -> &[HtGpuJobChunkEntry] { + &self.entries + } + + /// Entries belonging to one chunk index, or `None` when the index is invalid. + #[doc(hidden)] + #[must_use] + pub fn chunk_entries(&self, chunk_index: usize) -> Option<&[HtGpuJobChunkEntry]> { + let chunk = self.chunks.get(chunk_index)?; + self.entries.get(chunk.entry_start..chunk.entry_end) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/j2k-core/src/batch/gpu_job_chunk/planner.rs b/crates/j2k-core/src/batch/gpu_job_chunk/planner.rs new file mode 100644 index 00000000..60a1c0e2 --- /dev/null +++ b/crates/j2k-core/src/batch/gpu_job_chunk/planner.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use alloc::vec::Vec; + +use super::{ + HtGpuJobChunk, HtGpuJobChunkEntry, HtGpuJobChunkLimit, HtGpuJobChunkLimits, HtGpuJobChunkPlan, + HtGpuJobChunkPlanError, HtGpuJobChunkRequest, HtGpuJobPassBucket, +}; +use crate::{ + try_batch_reserve_for_push, BatchAllocationBudget, BatchAllocationRequest, + BatchInfrastructureError, +}; + +const BUCKET_ORDER: [HtGpuJobPassBucket; 3] = [ + HtGpuJobPassBucket::CleanupOnly, + HtGpuJobPassBucket::SigProp, + HtGpuJobPassBucket::MagRef, +]; + +/// Build bounded pass-homogeneous chunks while retaining caller job identity. +#[doc(hidden)] +pub fn plan_ht_gpu_job_chunks( + jobs: &[HtGpuJobChunkRequest], + limits: HtGpuJobChunkLimits, +) -> Result { + preflight_jobs(jobs, limits)?; + if jobs.is_empty() { + return Ok(HtGpuJobChunkPlan::default()); + } + + let mut budget = BatchAllocationBudget::new("HTJ2K GPU chunk planning metadata"); + budget.preflight(&[ + BatchAllocationRequest::of::(jobs.len()), + BatchAllocationRequest::of::(jobs.len()), + ])?; + let mut entries = budget.try_vec(jobs.len(), "HTJ2K GPU chunk job identities")?; + let mut chunks = budget.try_vec(jobs.len(), "HTJ2K GPU chunk descriptors")?; + + for bucket in BUCKET_ORDER { + plan_bucket(jobs, limits, bucket, &mut entries, &mut chunks)?; + } + Ok(HtGpuJobChunkPlan::new(chunks, entries)) +} + +fn preflight_jobs( + jobs: &[HtGpuJobChunkRequest], + limits: HtGpuJobChunkLimits, +) -> Result<(), HtGpuJobChunkPlanError> { + for (original_job_index, job) in jobs.iter().copied().enumerate() { + bucket_for(job, original_job_index)?; + ensure_single_job_fits( + job, + original_job_index, + HtGpuJobChunkLimit::PayloadBytes, + job.payload_bytes(), + limits.max_payload_bytes(), + )?; + ensure_single_job_fits( + job, + original_job_index, + HtGpuJobChunkLimit::DescriptorBytes, + job.descriptor_bytes(), + limits.max_descriptor_bytes(), + )?; + } + Ok(()) +} + +fn ensure_single_job_fits( + job: HtGpuJobChunkRequest, + original_job_index: usize, + limit: HtGpuJobChunkLimit, + requested: usize, + cap: usize, +) -> Result<(), HtGpuJobChunkPlanError> { + if requested > cap { + return Err(HtGpuJobChunkPlanError::SingleJobTooLarge { + source_index: job.source_index(), + original_job_index, + limit, + requested, + cap, + }); + } + Ok(()) +} + +fn plan_bucket( + jobs: &[HtGpuJobChunkRequest], + limits: HtGpuJobChunkLimits, + bucket: HtGpuJobPassBucket, + entries: &mut Vec, + chunks: &mut Vec, +) -> Result<(), HtGpuJobChunkPlanError> { + let mut entry_start = entries.len(); + let mut payload_bytes = 0usize; + let mut descriptor_bytes = 0usize; + + for (original_job_index, job) in jobs.iter().copied().enumerate() { + if bucket_for(job, original_job_index)? != bucket { + continue; + } + let job_count = entries.len() - entry_start; + if job_count == limits.max_jobs().get() + || exceeds( + payload_bytes, + job.payload_bytes(), + limits.max_payload_bytes(), + ) + || exceeds( + descriptor_bytes, + job.descriptor_bytes(), + limits.max_descriptor_bytes(), + ) + { + finish_chunk( + chunks, + bucket, + entry_start, + entries.len(), + payload_bytes, + descriptor_bytes, + )?; + entry_start = entries.len(); + payload_bytes = 0; + descriptor_bytes = 0; + } + + try_batch_reserve_for_push(entries, "HTJ2K GPU chunk job identities")?; + entries.push(HtGpuJobChunkEntry::new( + original_job_index, + job.source_index(), + )); + payload_bytes = payload_bytes.checked_add(job.payload_bytes()).ok_or( + BatchInfrastructureError::AllocationTooLarge { + what: "HTJ2K GPU chunk payload bytes", + requested: usize::MAX, + cap: limits.max_payload_bytes(), + }, + )?; + descriptor_bytes = descriptor_bytes.checked_add(job.descriptor_bytes()).ok_or( + BatchInfrastructureError::AllocationTooLarge { + what: "HTJ2K GPU chunk descriptor bytes", + requested: usize::MAX, + cap: limits.max_descriptor_bytes(), + }, + )?; + } + + finish_chunk( + chunks, + bucket, + entry_start, + entries.len(), + payload_bytes, + descriptor_bytes, + ) +} + +fn finish_chunk( + chunks: &mut Vec, + bucket: HtGpuJobPassBucket, + entry_start: usize, + entry_end: usize, + payload_bytes: usize, + descriptor_bytes: usize, +) -> Result<(), HtGpuJobChunkPlanError> { + if entry_start == entry_end { + return Ok(()); + } + try_batch_reserve_for_push(chunks, "HTJ2K GPU chunk descriptors")?; + chunks.push(HtGpuJobChunk::new( + bucket, + entry_start, + entry_end, + payload_bytes, + descriptor_bytes, + )); + Ok(()) +} + +fn bucket_for( + job: HtGpuJobChunkRequest, + original_job_index: usize, +) -> Result { + match job.coding_passes() { + 0 => Err(HtGpuJobChunkPlanError::InvalidCodingPassCount { + source_index: job.source_index(), + original_job_index, + coding_passes: 0, + }), + 1 => Ok(HtGpuJobPassBucket::CleanupOnly), + 2 => Ok(HtGpuJobPassBucket::SigProp), + _ => Ok(HtGpuJobPassBucket::MagRef), + } +} + +fn exceeds(current: usize, additional: usize, cap: usize) -> bool { + current + .checked_add(additional) + .is_none_or(|requested| requested > cap) +} diff --git a/crates/j2k-core/src/batch/gpu_job_chunk/tests.rs b/crates/j2k-core/src/batch/gpu_job_chunk/tests.rs new file mode 100644 index 00000000..6c73b542 --- /dev/null +++ b/crates/j2k-core/src/batch/gpu_job_chunk/tests.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::num::NonZeroUsize; + +use alloc::vec::Vec; + +use super::{ + plan_ht_gpu_job_chunks, HtGpuJobChunkLimit, HtGpuJobChunkLimits, HtGpuJobChunkPlanError, + HtGpuJobChunkRequest, HtGpuJobPassBucket, +}; + +fn limits(jobs: usize, payload: usize, descriptors: usize) -> HtGpuJobChunkLimits { + HtGpuJobChunkLimits::new( + NonZeroUsize::new(jobs).expect("non-zero test job limit"), + payload, + descriptors, + ) +} + +fn job( + source_index: usize, + coding_passes: u8, + payload_bytes: usize, + descriptor_bytes: usize, +) -> HtGpuJobChunkRequest { + HtGpuJobChunkRequest::new(source_index, coding_passes, payload_bytes, descriptor_bytes) +} + +fn identities(plan: &super::HtGpuJobChunkPlan, chunk_index: usize) -> Vec<(usize, usize)> { + plan.chunk_entries(chunk_index) + .expect("planned chunk entries") + .iter() + .map(|entry| (entry.original_job_index(), entry.source_index())) + .collect() +} + +#[test] +fn interleaved_jobs_are_bucketed_in_stable_original_order() { + let jobs = [ + job(7, 3, 2, 1), + job(2, 1, 2, 1), + job(8, 2, 2, 1), + job(2, 1, 2, 1), + job(7, 4, 2, 1), + ]; + let plan = plan_ht_gpu_job_chunks(&jobs, limits(8, 32, 32)).expect("chunk plan"); + + assert_eq!( + plan.chunks() + .iter() + .map(|chunk| chunk.bucket()) + .collect::>(), + [ + HtGpuJobPassBucket::CleanupOnly, + HtGpuJobPassBucket::SigProp, + HtGpuJobPassBucket::MagRef, + ] + ); + assert_eq!(identities(&plan, 0), [(1, 2), (3, 2)]); + assert_eq!(identities(&plan, 1), [(2, 8)]); + assert_eq!(identities(&plan, 2), [(0, 7), (4, 7)]); +} + +#[test] +fn exact_job_payload_and_descriptor_boundaries_share_one_chunk() { + let jobs = [job(0, 1, 2, 1), job(1, 1, 3, 2)]; + let plan = plan_ht_gpu_job_chunks(&jobs, limits(2, 5, 3)).expect("exact boundaries"); + + assert_eq!(plan.chunks().len(), 1); + let chunk = &plan.chunks()[0]; + assert_eq!(chunk.job_count(), 2); + assert_eq!(chunk.payload_bytes(), 5); + assert_eq!(chunk.descriptor_bytes(), 3); + assert_eq!(identities(&plan, 0), [(0, 0), (1, 1)]); +} + +#[test] +fn tiny_caps_split_without_losing_job_or_source_order() { + let jobs = [ + job(4, 2, 1, 1), + job(3, 2, 1, 1), + job(4, 2, 1, 1), + job(3, 2, 1, 1), + ]; + let plan = plan_ht_gpu_job_chunks(&jobs, limits(2, 2, 2)).expect("tiny chunk caps"); + + assert_eq!(plan.chunks().len(), 2); + assert!(plan + .chunks() + .iter() + .all(|chunk| chunk.bucket() == HtGpuJobPassBucket::SigProp)); + assert_eq!(identities(&plan, 0), [(0, 4), (1, 3)]); + assert_eq!(identities(&plan, 1), [(2, 4), (3, 3)]); +} + +#[test] +fn payload_and_descriptor_caps_each_force_a_boundary() { + let payload_jobs = [job(0, 1, 2, 1), job(1, 1, 2, 1)]; + let payload_plan = + plan_ht_gpu_job_chunks(&payload_jobs, limits(8, 3, 8)).expect("payload split"); + assert_eq!(payload_plan.chunks().len(), 2); + + let descriptor_jobs = [job(0, 3, 1, 2), job(1, 3, 1, 2)]; + let descriptor_plan = + plan_ht_gpu_job_chunks(&descriptor_jobs, limits(8, 8, 3)).expect("descriptor split"); + assert_eq!(descriptor_plan.chunks().len(), 2); +} + +#[test] +fn single_payload_and_descriptor_oversize_errors_are_source_indexed() { + let jobs = [job(3, 1, 1, 1), job(44, 1, 6, 2)]; + assert_eq!( + plan_ht_gpu_job_chunks(&jobs, limits(4, 5, 5)).expect_err("payload too large"), + HtGpuJobChunkPlanError::SingleJobTooLarge { + source_index: 44, + original_job_index: 1, + limit: HtGpuJobChunkLimit::PayloadBytes, + requested: 6, + cap: 5, + } + ); + + let jobs = [job(9, 3, 1, 7)]; + assert_eq!( + plan_ht_gpu_job_chunks(&jobs, limits(4, 5, 6)).expect_err("descriptor too large"), + HtGpuJobChunkPlanError::SingleJobTooLarge { + source_index: 9, + original_job_index: 0, + limit: HtGpuJobChunkLimit::DescriptorBytes, + requested: 7, + cap: 6, + } + ); +} + +#[test] +fn zero_pass_job_is_rejected_with_original_identity() { + let jobs = [job(12, 0, 0, 0)]; + assert_eq!( + plan_ht_gpu_job_chunks(&jobs, limits(1, 0, 0)).expect_err("zero pass job"), + HtGpuJobChunkPlanError::InvalidCodingPassCount { + source_index: 12, + original_job_index: 0, + coding_passes: 0, + } + ); +} + +#[test] +fn empty_input_produces_an_allocation_free_empty_plan() { + let plan = plan_ht_gpu_job_chunks(&[], limits(1, 0, 0)).expect("empty plan"); + assert!(plan.chunks().is_empty()); + assert!(plan.entries().is_empty()); +} diff --git a/crates/j2k-core/src/lib.rs b/crates/j2k-core/src/lib.rs index 29e5a6d1..33613b4b 100644 --- a/crates/j2k-core/src/lib.rs +++ b/crates/j2k-core/src/lib.rs @@ -70,13 +70,17 @@ pub use accelerator::{ AcceleratorSession, DeviceMemoryRange, ExecutionStats, SurfaceMetadata, SurfaceResidency, }; pub use backend::{BackendCapabilities, BackendKind, BackendRequest, CpuFeatures}; +#[doc(hidden)] +pub use batch::plan_ht_gpu_job_chunks; pub use batch::{ checked_batch_count_product, checked_batch_count_sum, tile_batch_worker_count, try_batch_reserve_for_push, try_batch_reserve_to, try_collect_indexed_batch_results, try_collect_ordered_batch_results_with_limits, BatchAllocationBudget, BatchAllocationRequest, - BatchDecodeError, BatchInfrastructureError, BatchResultSlot, IndexedBatchResult, - TileBatchError, TileBatchOptions, TileDecodeJob, TileRegionDecodeJob, - TileRegionScaledDecodeJob, TileRegionScaledDeviceDecodeRequest, TileScaledDecodeJob, + BatchDecodeError, BatchInfrastructureError, BatchResultSlot, HtGpuJobChunk, HtGpuJobChunkEntry, + HtGpuJobChunkLimit, HtGpuJobChunkLimits, HtGpuJobChunkPlan, HtGpuJobChunkPlanError, + HtGpuJobChunkRequest, HtGpuJobPassBucket, IndexedBatchResult, TileBatchError, TileBatchOptions, + TileDecodeJob, TileRegionDecodeJob, TileRegionScaledDecodeJob, + TileRegionScaledDeviceDecodeRequest, TileScaledDecodeJob, }; pub use buffer::{ checked_surface_len, copy_tight_pixels_to_strided_output, ensure_allocation_within_cap, diff --git a/crates/j2k-core/src/pixel.rs b/crates/j2k-core/src/pixel.rs index 83233c3b..7b7cc9e3 100644 --- a/crates/j2k-core/src/pixel.rs +++ b/crates/j2k-core/src/pixel.rs @@ -14,6 +14,18 @@ pub enum PixelLayout { Gray, } +impl PixelLayout { + /// Return the number of channels in this layout. + #[must_use] + pub const fn channels(self) -> usize { + match self { + Self::Rgb => 3, + Self::Rgba => 4, + Self::Gray => 1, + } + } +} + /// Concrete interleaved pixel format. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[non_exhaustive] @@ -30,6 +42,12 @@ pub enum PixelFormat { Rgba16, /// 16-bit grayscale. Gray16, + /// Interleaved signed 16-bit RGB. + RgbI16, + /// Interleaved signed 16-bit RGBA. + RgbaI16, + /// Signed 16-bit grayscale. + GrayI16, } impl PixelFormat { @@ -37,9 +55,9 @@ impl PixelFormat { #[must_use] pub const fn layout(self) -> PixelLayout { match self { - Self::Rgb8 | Self::Rgb16 => PixelLayout::Rgb, - Self::Rgba8 | Self::Rgba16 => PixelLayout::Rgba, - Self::Gray8 | Self::Gray16 => PixelLayout::Gray, + Self::Rgb8 | Self::Rgb16 | Self::RgbI16 => PixelLayout::Rgb, + Self::Rgba8 | Self::Rgba16 | Self::RgbaI16 => PixelLayout::Rgba, + Self::Gray8 | Self::Gray16 | Self::GrayI16 => PixelLayout::Gray, } } @@ -49,17 +67,14 @@ impl PixelFormat { match self { Self::Rgb8 | Self::Rgba8 | Self::Gray8 => SampleType::U8, Self::Rgb16 | Self::Rgba16 | Self::Gray16 => SampleType::U16, + Self::RgbI16 | Self::RgbaI16 | Self::GrayI16 => SampleType::I16, } } /// Return the number of channels per pixel. #[must_use] pub const fn channels(self) -> usize { - match self.layout() { - PixelLayout::Rgb => 3, - PixelLayout::Rgba => 4, - PixelLayout::Gray => 1, - } + self.layout().channels() } /// Return the number of bytes in one channel sample. @@ -67,7 +82,7 @@ impl PixelFormat { pub const fn bytes_per_sample(self) -> usize { match self.sample() { SampleType::U8 => 1, - SampleType::U16 => 2, + SampleType::U16 | SampleType::I16 => 2, } } @@ -77,3 +92,24 @@ impl PixelFormat { self.channels() * self.bytes_per_sample() } } + +#[cfg(test)] +mod tests { + use super::{PixelFormat, PixelLayout}; + use crate::SampleType; + + #[test] + fn signed_sixteen_bit_formats_preserve_layout_and_size() { + for (format, layout, channels) in [ + (PixelFormat::RgbI16, PixelLayout::Rgb, 3), + (PixelFormat::RgbaI16, PixelLayout::Rgba, 4), + (PixelFormat::GrayI16, PixelLayout::Gray, 1), + ] { + assert_eq!(format.layout(), layout); + assert_eq!(format.sample(), SampleType::I16); + assert_eq!(format.channels(), channels); + assert_eq!(format.bytes_per_sample(), 2); + assert_eq!(format.bytes_per_pixel(), channels * 2); + } + } +} diff --git a/crates/j2k-core/src/sample.rs b/crates/j2k-core/src/sample.rs index c3160075..1e1193db 100644 --- a/crates/j2k-core/src/sample.rs +++ b/crates/j2k-core/src/sample.rs @@ -1,13 +1,15 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 /// Integer sample width used by a pixel format or row sink. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum SampleType { /// Unsigned 8-bit samples. U8, /// Unsigned 16-bit samples. U16, + /// Signed 16-bit samples. + I16, } /// Supported integer sample type for row-oriented APIs. @@ -27,3 +29,8 @@ impl Sample for u16 { const TYPE: SampleType = SampleType::U16; const BITS: u8 = 16; } + +impl Sample for i16 { + const TYPE: SampleType = SampleType::I16; + const BITS: u8 = 16; +} diff --git a/crates/j2k-cuda-runtime/build.rs b/crates/j2k-cuda-runtime/build.rs index c385e5e7..8d918b33 100644 --- a/crates/j2k-cuda-runtime/build.rs +++ b/crates/j2k-cuda-runtime/build.rs @@ -33,6 +33,16 @@ const CUDA_OXIDE_J2K_ENCODE_EXTRA_SOURCES: &[&str] = &[ "simt/src/quantization.rs", "simt/src/tag_tree.rs", ]; +const CUDA_OXIDE_J2K_DECODE_STORE_EXTRA_SOURCES: &[&str] = &[ + "simt/src/abi.rs", + "simt/src/color.rs", + "simt/src/exports.rs", + "simt/src/layout.rs", + "simt/src/memory.rs", + "simt/src/native_color.rs", + "simt/src/sample.rs", + "simt/src/transform.rs", +]; const CUDA_OXIDE_TRANSCODE_EXTRA_SOURCES: &[&str] = &[ "simt/src/abi.rs", "simt/src/constants.rs", @@ -117,6 +127,9 @@ fn emit_build_script_metadata(codec_math_crate_path: &Path) { println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_decode_store/src/main.rs"); println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_decode_store/simt/Cargo.toml.in"); println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_decode_store/simt/src/main.rs"); + for relative in CUDA_OXIDE_J2K_DECODE_STORE_EXTRA_SOURCES { + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_decode_store/{relative}"); + } println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_classic_decode/Cargo.toml.in"); println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_classic_decode/rust-toolchain.toml"); println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_classic_decode/src/main.rs"); @@ -598,6 +611,8 @@ fn copy_cuda_oxide_project(source_dir: &Path, project_dir: &Path, codec_math_cra fn cuda_oxide_extra_sources(source_dir: &Path) -> &'static [&'static str] { if source_dir == Path::new("src/cuda_oxide_j2k_encode") { CUDA_OXIDE_J2K_ENCODE_EXTRA_SOURCES + } else if source_dir == Path::new("src/cuda_oxide_j2k_decode_store") { + CUDA_OXIDE_J2K_DECODE_STORE_EXTRA_SOURCES } else if source_dir == Path::new("src/cuda_oxide_transcode") { CUDA_OXIDE_TRANSCODE_EXTRA_SOURCES } else if source_dir == Path::new("src/cuda_oxide_jpeg_decode") { diff --git a/crates/j2k-cuda-runtime/src/build_flags.rs b/crates/j2k-cuda-runtime/src/build_flags.rs index 91e546d9..f58131c2 100644 --- a/crates/j2k-cuda-runtime/src/build_flags.rs +++ b/crates/j2k-cuda-runtime/src/build_flags.rs @@ -5,6 +5,8 @@ use std::{os::raw::c_uint, sync::OnceLock}; pub(crate) const CUDA_SUCCESS: CuResult = 0; +pub(crate) const CUDA_ERROR_NOT_READY: CuResult = 600; + pub(crate) const PINNED_POOLED_I16_UPLOAD_MAX_BYTES: usize = 4 * 1024 * 1024; pub(crate) const DWT97_ROW_LIFT_MAX_WIDTH: i32 = 1024; diff --git a/crates/j2k-cuda-runtime/src/bytes.rs b/crates/j2k-cuda-runtime/src/bytes.rs index 0b2f4e77..63645c45 100644 --- a/crates/j2k-cuda-runtime/src/bytes.rs +++ b/crates/j2k-cuda-runtime/src/bytes.rs @@ -17,9 +17,11 @@ use crate::{ CudaHtj2kPacketizationSubbandTagState, CudaHtj2kPacketizationTagNodeState, }, j2k_decode::{ - CudaJ2kIdwtJob, CudaJ2kIdwtMultiKernelJob, CudaJ2kInverseMctJob, CudaJ2kStoreGray16Job, - CudaJ2kStoreGray8Job, CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, - CudaJ2kStoreRgb8MctBatchJob, + CudaJ2kIdwtJob, CudaJ2kIdwtMultiKernelJob, CudaJ2kInverseMctJob, + CudaJ2kStoreGray16BatchJob, CudaJ2kStoreGray16Job, CudaJ2kStoreGray8BatchJob, + CudaJ2kStoreGray8Job, CudaJ2kStoreGrayI16BatchJob, CudaJ2kStoreRgb16Job, + CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, CudaJ2kStoreRgb8MctBatchJob, + CudaJ2kStoreRgbNativeBatchJob, CudaJ2kStoreRgbaNativeBatchJob, }, jpeg::{ CudaJpegBaselineEncodeHuffmanTable, CudaJpegBaselineEncodeParams, @@ -103,6 +105,11 @@ gpu_slice_bytes! { #[cfg_attr(not(feature = "cuda-oxide-jpeg-encode"), expect(dead_code, reason = "helper is used only by the JPEG encode kernel feature"))] cuda_jpeg_baseline_encode_statuses_as_bytes: CudaJpegBaselineEncodeStatus; store_rgb8_mct_batch_jobs_as_bytes: CudaJ2kStoreRgb8MctBatchJob; + store_rgb_native_batch_jobs_as_bytes: CudaJ2kStoreRgbNativeBatchJob; + store_rgba_native_batch_jobs_as_bytes: CudaJ2kStoreRgbaNativeBatchJob; + store_gray8_batch_jobs_as_bytes: CudaJ2kStoreGray8BatchJob; + store_gray16_batch_jobs_as_bytes: CudaJ2kStoreGray16BatchJob; + store_grayi16_batch_jobs_as_bytes: CudaJ2kStoreGrayI16BatchJob; htj2k_encode_jobs_as_bytes: CudaHtj2kEncodeKernelJob; htj2k_encode_multi_input_jobs_as_bytes: CudaHtj2kEncodeMultiInputKernelJob; htj2k_encode_compact_jobs_as_bytes: CudaHtj2kEncodeCompactJob; @@ -148,3 +155,9 @@ pub(crate) fn htj2k_statuses_byte_len(count: usize) -> Result .checked_mul(std::mem::size_of::()) .ok_or(CudaError::LengthTooLarge { len: count }) } + +pub(crate) fn classic_statuses_byte_len(count: usize) -> Result { + count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count }) +} diff --git a/crates/j2k-cuda-runtime/src/bytes/abi.rs b/crates/j2k-cuda-runtime/src/bytes/abi.rs index a5bca56e..c735b4ed 100644 --- a/crates/j2k-cuda-runtime/src/bytes/abi.rs +++ b/crates/j2k-cuda-runtime/src/bytes/abi.rs @@ -19,8 +19,10 @@ use crate::{ }, j2k_decode::{ CudaJ2kIdwtJob, CudaJ2kIdwtMultiKernelJob, CudaJ2kInverseMctJob, CudaJ2kRect, - CudaJ2kStoreGray16Job, CudaJ2kStoreGray8Job, CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb16MctJob, - CudaJ2kStoreRgb8Job, CudaJ2kStoreRgb8MctBatchJob, CudaJ2kStoreRgb8MctJob, + CudaJ2kStoreGray16BatchJob, CudaJ2kStoreGray16Job, CudaJ2kStoreGray8BatchJob, + CudaJ2kStoreGray8Job, CudaJ2kStoreGrayI16BatchJob, CudaJ2kStoreRgb16Job, + CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, CudaJ2kStoreRgb8MctBatchJob, + CudaJ2kStoreRgb8MctJob, }, jpeg::{ CudaJpegBaselineEncodeHuffmanTable, CudaJpegBaselineEncodeParams, @@ -379,6 +381,24 @@ impl_cuda_gpu_abi! { addend: f32, bit_depth: u32, }, + CudaJ2kStoreGray8BatchJob { + input_ptr: u64, + output_ptr: u64, + job: CudaJ2kStoreGray8Job, + reserved_tail: u32, + }, + CudaJ2kStoreGray16BatchJob { + input_ptr: u64, + output_ptr: u64, + job: CudaJ2kStoreGray16Job, + reserved_tail: u32, + }, + CudaJ2kStoreGrayI16BatchJob { + input_ptr: u64, + output_ptr: u64, + job: CudaJ2kStoreGray16Job, + reserved_tail: u32, + }, CudaJ2kInverseMctJob { len: u32, irreversible97: u32, @@ -452,5 +472,7 @@ impl_cuda_gpu_abi! { }, } +mod native_store; + #[cfg(test)] mod tests; diff --git a/crates/j2k-cuda-runtime/src/bytes/abi/native_store.rs b/crates/j2k-cuda-runtime/src/bytes/abi/native_store.rs new file mode 100644 index 00000000..eed811e0 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/bytes/abi/native_store.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_core::accelerator::GpuAbi; +use std::mem::{offset_of, size_of}; + +use crate::j2k_decode::{ + CudaJ2kStoreRgbNativeBatchJob, CudaJ2kStoreRgbNativeJob, CudaJ2kStoreRgbaNativeBatchJob, + CudaJ2kStoreRgbaNativeJob, +}; + +impl_cuda_gpu_abi! { + CudaJ2kStoreRgbNativeJob { + input_width0: u32, + input_width1: u32, + input_width2: u32, + source_x0: u32, + source_y0: u32, + source_x1: u32, + source_y1: u32, + source_x2: u32, + source_y2: u32, + copy_width: u32, + copy_height: u32, + output_width: u32, + output_height: u32, + output_x: u32, + output_y: u32, + addend0: f32, + addend1: f32, + addend2: f32, + bit_depth0: u32, + bit_depth1: u32, + bit_depth2: u32, + layout: u32, + transform: u32, + reserved: u32, + }, + CudaJ2kStoreRgbNativeBatchJob { + plane0_ptr: u64, + plane1_ptr: u64, + plane2_ptr: u64, + output_ptr: u64, + job: CudaJ2kStoreRgbNativeJob, + }, + CudaJ2kStoreRgbaNativeJob { + input_width0: u32, + input_width1: u32, + input_width2: u32, + input_width3: u32, + source_x0: u32, + source_y0: u32, + source_x1: u32, + source_y1: u32, + source_x2: u32, + source_y2: u32, + source_x3: u32, + source_y3: u32, + copy_width: u32, + copy_height: u32, + output_width: u32, + output_height: u32, + output_x: u32, + output_y: u32, + addend0: f32, + addend1: f32, + addend2: f32, + addend3: f32, + bit_depth0: u32, + bit_depth1: u32, + bit_depth2: u32, + bit_depth3: u32, + layout: u32, + transform: u32, + reserved: u32, + }, + CudaJ2kStoreRgbaNativeBatchJob { + plane0_ptr: u64, + plane1_ptr: u64, + plane2_ptr: u64, + plane3_ptr: u64, + output_ptr: u64, + job: CudaJ2kStoreRgbaNativeJob, + reserved_tail: u32, + }, +} diff --git a/crates/j2k-cuda-runtime/src/bytes/abi/tests.rs b/crates/j2k-cuda-runtime/src/bytes/abi/tests.rs index 78efaef3..d05ace87 100644 --- a/crates/j2k-cuda-runtime/src/bytes/abi/tests.rs +++ b/crates/j2k-cuda-runtime/src/bytes/abi/tests.rs @@ -1,6 +1,10 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::*; +use crate::j2k_decode::{ + CudaJ2kStoreRgbNativeBatchJob, CudaJ2kStoreRgbNativeJob, CudaJ2kStoreRgbaNativeBatchJob, + CudaJ2kStoreRgbaNativeJob, +}; #[test] fn explicit_tail_fields_preserve_cuda_host_abi_sizes_and_offsets() { @@ -30,6 +34,22 @@ fn explicit_tail_fields_preserve_cuda_host_abi_sizes_and_offsets() { assert_eq!(offset_of!(CudaJ2kIdwtMultiKernelJob, reserved_tail), 124); assert_eq!(size_of::(), 128); assert_eq!(offset_of!(CudaJ2kStoreRgb8MctBatchJob, reserved_tail), 124); + assert_eq!(size_of::(), 96); + assert_eq!(size_of::(), 128); + assert_eq!(offset_of!(CudaJ2kStoreRgbNativeBatchJob, job), 32); + assert_eq!(size_of::(), 116); + assert_eq!(size_of::(), 160); + assert_eq!(offset_of!(CudaJ2kStoreRgbaNativeBatchJob, job), 40); + assert_eq!( + offset_of!(CudaJ2kStoreRgbaNativeBatchJob, reserved_tail), + 156 + ); + assert_eq!(size_of::(), 64); + assert_eq!(offset_of!(CudaJ2kStoreGray8BatchJob, reserved_tail), 60); + assert_eq!(size_of::(), 64); + assert_eq!(offset_of!(CudaJ2kStoreGray16BatchJob, reserved_tail), 60); + assert_eq!(size_of::(), 64); + assert_eq!(offset_of!(CudaJ2kStoreGrayI16BatchJob, reserved_tail), 60); } #[test] diff --git a/crates/j2k-cuda-runtime/src/classic_decode.rs b/crates/j2k-cuda-runtime/src/classic_decode.rs index eea3e71b..f736b102 100644 --- a/crates/j2k-cuda-runtime/src/classic_decode.rs +++ b/crates/j2k-cuda-runtime/src/classic_decode.rs @@ -3,11 +3,13 @@ mod abi; mod launch; mod prepare; +mod queued; #[cfg(test)] mod tests; pub use abi::{ - CudaClassicCodeBlockJob, CudaClassicDecodeStageTimings, CudaClassicDecodeTarget, - CudaClassicSegment, CudaClassicStatus, + CudaClassicCodeBlockJob, CudaClassicDecodeStageTimings, CudaClassicDecodeTableResources, + CudaClassicDecodeTarget, CudaClassicSegment, CudaClassicStatus, }; pub(crate) use abi::{CudaClassicKernelJob, CudaClassicKernelSegment, CudaClassicKernelTables}; +pub use queued::CudaQueuedClassicDecode; diff --git a/crates/j2k-cuda-runtime/src/classic_decode/abi.rs b/crates/j2k-cuda-runtime/src/classic_decode/abi.rs index e59e0b65..08c4e800 100644 --- a/crates/j2k-cuda-runtime/src/classic_decode/abi.rs +++ b/crates/j2k-cuda-runtime/src/classic_decode/abi.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use std::sync::Arc; + use crate::memory::CudaDeviceBuffer; use j2k_codec_math::classic::{ MQ_QE_VALUES, PACKED_MQ_TRANSITION_VALUES, PACKED_SIGN_CONTEXT_LOOKUP, ZERO_CTX_HH_LOOKUP, @@ -131,6 +133,24 @@ pub struct CudaClassicDecodeStageTimings { pub status_d2h_us: u128, } +/// Device-resident static lookup tables for classic JPEG 2000 Tier-1 decode. +#[doc(hidden)] +#[derive(Clone, Debug)] +pub struct CudaClassicDecodeTableResources { + pub(crate) inner: Arc, +} + +#[derive(Debug)] +pub(crate) struct CudaClassicDecodeTableResourceInner { + pub(crate) tables: CudaDeviceBuffer, +} + +impl CudaClassicDecodeTableResources { + pub(crate) fn is_owned_by(&self, context: &crate::CudaContext) -> bool { + self.inner.tables.is_owned_by(context) + } +} + #[repr(C)] #[derive(Clone, Copy)] pub(crate) struct CudaClassicKernelTables { diff --git a/crates/j2k-cuda-runtime/src/classic_decode/launch.rs b/crates/j2k-cuda-runtime/src/classic_decode/launch.rs index 4d63ed12..3dc9ba4e 100644 --- a/crates/j2k-cuda-runtime/src/classic_decode/launch.rs +++ b/crates/j2k-cuda-runtime/src/classic_decode/launch.rs @@ -1,19 +1,21 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::time::Instant; +use std::{sync::Arc, time::Instant}; use super::abi::{ - CudaClassicDecodeStageTimings, CudaClassicDecodeTarget, CudaClassicStatus, + CudaClassicDecodeStageTimings, CudaClassicDecodeTableResourceInner, + CudaClassicDecodeTableResources, CudaClassicDecodeTarget, CudaClassicStatus, CLASSIC_KERNEL_TABLES, }; use super::prepare::{ checked_bytes, invalid, prepare_classic_decode, validate_classic_launch_owners, }; +use super::queued::CudaQueuedClassicDecode; use crate::{ allocation::HostPhaseBudget, bytes::{ classic_jobs_as_bytes, classic_segments_as_bytes, classic_statuses_as_bytes_mut, - classic_tables_as_bytes, + classic_statuses_byte_len, classic_tables_as_bytes, }, context::CudaContext, error::{select_resource_release_error, CudaError}, @@ -26,6 +28,19 @@ use crate::{ const CLASSIC_KERNEL_NAME: &str = "j2k_decode_classic_codeblocks_multi"; impl CudaContext { + /// Upload static classic Tier-1 lookup tables once for session reuse. + #[doc(hidden)] + pub fn upload_classic_decode_table_resources( + &self, + ) -> Result { + self.inner.set_current()?; + Ok(CudaClassicDecodeTableResources { + inner: Arc::new(CudaClassicDecodeTableResourceInner { + tables: self.upload(classic_tables_as_bytes(&CLASSIC_KERNEL_TABLES))?, + }), + }) + } + /// Allocate and clear one classic Tier-1 coefficient plane. #[doc(hidden)] pub fn allocate_classic_coefficients_with_pool( @@ -40,8 +55,7 @@ impl CudaContext { } let bytes = checked_bytes::(output_words)?; let output = pool.take(bytes)?; - self.memset_d32(pooled_device_buffer(&output)?, 0, output_words)?; - self.synchronize()?; + self.memset_d32_async(pooled_device_buffer(&output)?, 0, output_words)?; Ok(output) } @@ -64,6 +78,105 @@ impl CudaContext { .map(|(statuses, _)| statuses) } + /// Enqueue classic Tier-1 decoding and defer its single status transfer. + /// + /// # Safety + /// + /// Payload, table, coefficient, and pool owners must remain live and + /// unmodified until the returned guard is finished or dropped. Targets + /// must remain pairwise disjoint and confined to this context's default + /// stream until completion. + #[doc(hidden)] + pub unsafe fn decode_classic_codeblocks_multi_enqueue_with_resources_and_pool( + &self, + resources: &CudaHtj2kDecodeResources, + tables: &CudaClassicDecodeTableResources, + targets: &[CudaClassicDecodeTarget<'_>], + pool: &CudaBufferPool, + live_host_bytes: usize, + ) -> Result { + validate_classic_launch_owners(self, resources, targets, pool)?; + if !tables.is_owned_by(self) { + return Err(CudaError::InvalidArgument { + message: "classic Tier-1 tables must belong to the decode context".to_string(), + }); + } + let mut host_budget = + HostPhaseBudget::with_live_bytes("CUDA queued classic Tier-1 owners", live_host_bytes)?; + let prepared = prepare_classic_decode(resources.payload_len, targets, &mut host_budget)?; + if prepared.jobs.is_empty() { + return Ok(CudaQueuedClassicDecode { + context: self.clone(), + resources: Vec::new(), + status_buffer: None, + status_count: 0, + execution: crate::execution::CudaExecutionStats::default(), + timings: CudaClassicDecodeStageTimings::default(), + pool_reuse_guard: None, + finish_host_live_bytes: 0, + }); + } + let payload = resources.payload.buffer()?; + let jobs = pool.upload_pinned(classic_jobs_as_bytes(&prepared.jobs))?; + let segments = pool.upload_pinned(classic_segments_as_bytes(&prepared.segments))?; + let statuses = pool.take(classic_statuses_byte_len(prepared.jobs.len())?)?; + let scratch = pool.take(checked_bytes::(prepared.scratch_words)?)?; + let mut queued_resources = host_budget.try_vec_with_capacity(3)?; + queued_resources.push(jobs); + queued_resources.push(segments); + queued_resources.push(scratch); + let mut finish_budget = HostPhaseBudget::with_live_bytes( + "CUDA queued classic Tier-1 retained metadata", + live_host_bytes, + )?; + finish_budget.account_vec(&queued_resources)?; + + let mut payload_ptr = payload.device_ptr(); + let mut jobs_ptr = pooled_device_buffer(&queued_resources[0])?.device_ptr(); + let mut segments_ptr = pooled_device_buffer(&queued_resources[1])?.device_ptr(); + let mut tables_ptr = tables.inner.tables.device_ptr(); + let mut statuses_ptr = pooled_device_buffer(&statuses)?.device_ptr(); + let mut scratch_ptr = pooled_device_buffer(&queued_resources[2])?.device_ptr(); + let mut params = cuda_kernel_params!( + payload_ptr, + jobs_ptr, + segments_ptr, + tables_ptr, + statuses_ptr, + scratch_ptr + ); + let geometry = j2k_classic_codeblock_launch_geometry(prepared.jobs.len()).ok_or( + CudaError::LengthTooLarge { + len: prepared.jobs.len(), + }, + )?; + let function = self.inner.cuda_oxide_j2k_classic_decode_kernel_function( + CudaKernel::J2kClassicDecodeCodeblocksMulti, + )?; + let pool_reuse_guard = pool.defer_reuse()?; + let launch_result = self.with_nvtx_range("j2k.classic.decode.tier1.batch", || { + self.launch_kernel_async(function, geometry, &mut params) + }); + if let Err(error) = launch_result { + return pool_reuse_guard.synchronize_then_error(error); + } + Ok(CudaQueuedClassicDecode { + context: self.clone(), + resources: queued_resources, + status_buffer: Some(statuses), + status_count: prepared.jobs.len(), + execution: crate::execution::CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + timings: CudaClassicDecodeStageTimings::default(), + pool_reuse_guard: Some(pool_reuse_guard), + finish_host_live_bytes: finish_budget.live_bytes(), + }) + } + /// Decode classic Tier-1 code-blocks and return optional stage timings. #[doc(hidden)] pub fn decode_classic_codeblocks_multi_with_resources_and_pool_timed( diff --git a/crates/j2k-cuda-runtime/src/classic_decode/queued.rs b/crates/j2k-cuda-runtime/src/classic_decode/queued.rs new file mode 100644 index 00000000..246d2353 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/classic_decode/queued.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + allocation::HostPhaseBudget, + bytes::classic_statuses_as_bytes_mut, + context::CudaContext, + error::{select_resource_release_error, select_uncertain_completion_error, CudaError}, + execution::CudaExecutionStats, + memory::{CudaBufferPoolReuseGuard, CudaPooledDeviceBuffer}, +}; + +use super::{CudaClassicDecodeStageTimings, CudaClassicStatus}; + +const CLASSIC_KERNEL_NAME: &str = "j2k_decode_classic_codeblocks_multi"; + +/// Enqueued classic Tier-1 work retained until one deferred status readback. +#[doc(hidden)] +#[derive(Debug)] +#[must_use = "queued classic decode must be finished or retained until Drop synchronizes it"] +pub struct CudaQueuedClassicDecode { + pub(crate) context: CudaContext, + pub(crate) resources: Vec, + pub(crate) status_buffer: Option, + pub(crate) status_count: usize, + pub(crate) execution: CudaExecutionStats, + pub(crate) timings: CudaClassicDecodeStageTimings, + pub(crate) pool_reuse_guard: Option, + pub(crate) finish_host_live_bytes: usize, +} + +impl CudaQueuedClassicDecode { + /// Number of descriptor statuses downloaded by completion. + #[doc(hidden)] + pub const fn status_count(&self) -> usize { + self.status_count + } + + /// CUDA execution counters for the enqueued classic work. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Finish the ordered graph with one status transfer and validate every job. + pub fn finish( + mut self, + ) -> Result<(CudaExecutionStats, CudaClassicDecodeStageTimings), CudaError> { + if self.status_count == 0 { + self.release_after_stream_completion()?; + return Ok((self.execution, self.timings)); + } + let statuses_result = HostPhaseBudget::with_live_bytes( + "CUDA queued classic Tier-1 status readback", + self.finish_host_live_bytes, + ) + .and_then(|mut budget| { + budget.try_vec_filled(self.status_count, CudaClassicStatus::default()) + }); + let mut statuses = match statuses_result { + Ok(statuses) => statuses, + Err(error) => return Err(self.synchronize_release_after_error(error)), + }; + let copy_result = self.status_buffer.as_ref().map_or_else( + || { + Err(CudaError::StatePoisoned { + message: "queued classic status buffer disappeared before readback".to_string(), + }) + }, + |buffer| buffer.copy_to_host(classic_statuses_as_bytes_mut(&mut statuses)), + ); + if let Err(error) = copy_result { + return Err(self.release_after_recoverable_operation_error(error)); + } + self.context.record_status_device_to_host_copy( + self.status_count + .saturating_mul(core::mem::size_of::()), + ); + let status_error = statuses + .iter() + .copied() + .enumerate() + .find(|(_, status)| status.code != 0) + .map(|(job_index, status)| CudaError::KernelJobStatus { + kernel: CLASSIC_KERNEL_NAME, + job_index, + code: status.code, + detail: status.detail, + }); + let release_result = self.release_after_stream_completion(); + match (status_error, release_result) { + (Some(primary), Err(release)) => Err(select_resource_release_error(primary, release)), + (Some(error), Ok(())) | (None, Err(error)) => Err(error), + (None, Ok(())) => Ok((self.execution, self.timings)), + } + } + + fn release_after_stream_completion(&mut self) -> Result<(), CudaError> { + self.status_buffer.take(); + self.resources.clear(); + if let Some(guard) = self.pool_reuse_guard.take() { + guard.release()?; + } + Ok(()) + } + + fn abandon_resources(&mut self) { + self.status_buffer.take(); + self.resources.clear(); + if let Some(guard) = self.pool_reuse_guard.take() { + guard.abandon(); + } + } + + fn release_after_recoverable_operation_error(&mut self, primary: CudaError) -> CudaError { + if self.context.inner.resource_lifetimes_poisoned() { + self.abandon_resources(); + return primary; + } + match self.release_after_stream_completion() { + Ok(()) => primary, + Err(release) => select_resource_release_error(primary, release), + } + } + + fn synchronize_release_after_error(&mut self, primary: CudaError) -> CudaError { + let outcome = self.context.synchronize_for_resource_release(); + if let Err(completion) = outcome.into_result() { + self.abandon_resources(); + return select_uncertain_completion_error(primary, Some(completion)); + } + match self.release_after_stream_completion() { + Ok(()) => primary, + Err(release) => select_resource_release_error(primary, release), + } + } +} + +impl Drop for CudaQueuedClassicDecode { + fn drop(&mut self) { + if self.pool_reuse_guard.is_some() { + let outcome = self.context.synchronize_for_resource_release(); + if outcome.completion_established() { + let _ = self.release_after_stream_completion(); + } else { + self.abandon_resources(); + } + } + } +} diff --git a/crates/j2k-cuda-runtime/src/classic_decode/tests.rs b/crates/j2k-cuda-runtime/src/classic_decode/tests.rs index c8519b43..add1a5e0 100644 --- a/crates/j2k-cuda-runtime/src/classic_decode/tests.rs +++ b/crates/j2k-cuda-runtime/src/classic_decode/tests.rs @@ -137,3 +137,30 @@ fn classic_runtime_validates_empty_work_and_times_only_status_copy() { .expect("pool release"); assert!(status_copy < status_timing && status_timing < pool_release); } + +#[test] +fn queued_classic_launch_defers_its_only_status_copy_to_guard_completion() { + let launch = include_str!("launch.rs"); + let enqueue = launch + .split("pub unsafe fn decode_classic_codeblocks_multi_enqueue_with_resources_and_pool") + .nth(1) + .expect("queued classic decode method") + .split("/// Decode classic Tier-1 code-blocks and return optional stage timings.") + .next() + .expect("queued classic decode body"); + assert!(enqueue.contains("launch_kernel_async")); + assert!(enqueue.contains("CudaQueuedClassicDecode")); + assert!(!enqueue.contains("copy_to_host")); + + let completion = include_str!("queued.rs") + .split("pub fn finish(") + .nth(1) + .expect("queued classic completion"); + let status_copy = completion + .find("copy_to_host") + .expect("deferred status copy"); + let release = completion + .find("let release_result = self.release_after_stream_completion()") + .expect("release after status completion"); + assert!(status_copy < release); +} diff --git a/crates/j2k-cuda-runtime/src/context.rs b/crates/j2k-cuda-runtime/src/context.rs index 408b3622..81f96b46 100644 --- a/crates/j2k-cuda-runtime/src/context.rs +++ b/crates/j2k-cuda-runtime/src/context.rs @@ -4,18 +4,18 @@ use std::sync::Arc; use crate::{ error::CudaError, - execution::{CudaExecutionStats, CudaLaunchMode}, + execution::CudaExecutionStats, htj2k_decode::{ htj2k_decode_needs_zero_fill, CudaHtj2kCodeBlockJob, CudaHtj2kDecodeOutput, - CudaHtj2kDecodeStageTimings, CudaQueuedHtj2kCleanup, + CudaHtj2kDecodeStageTimings, }, - memory::pooled_device_buffer, }; mod band_transfer; mod compact; mod creation; mod device; +mod diagnostics; mod host_budget; mod inner; mod kernel_cache; @@ -29,6 +29,7 @@ mod resource_creation; mod test_kernels; pub use self::compact::{CudaHtj2kCompactEncodedCodeBlock, CudaHtj2kCompactEncodedCodeBlocks}; +pub use self::diagnostics::CudaContextDiagnostics; #[doc(hidden)] pub use self::host_budget::{CudaExternalHostOwner, CudaExternalHostReservation}; #[cfg(test)] @@ -38,7 +39,7 @@ pub(crate) use self::test_kernels::{CudaKernelModule, CudaKernelName}; pub(crate) use self::{ band_transfer::cuda_idwt_trace_enabled, compact::HTJ2K_UVLC_ENCODE_TABLE_BYTES, - inner::ContextInner, + inner::{ContextInner, ContextOwnership}, kernel_cache::{CompiledKernel, CompiledKernelKey}, lifecycle::ContextResourceLifecycle, operations::ensure_context_ownership, @@ -67,40 +68,6 @@ impl CudaContext { self.inner.device_ordinal } - /// Dequantize HTJ2K cleanup outputs using the metadata buffer already held - /// live by a queued cleanup launch. - #[doc(hidden)] - pub fn j2k_dequantize_queued_htj2k_cleanup_with_pool( - &self, - cleanup: &CudaQueuedHtj2kCleanup, - ) -> Result { - if !self.is_same_context(&cleanup.context) { - return Err(CudaError::InvalidArgument { - message: "queued HTJ2K cleanup belongs to a different CUDA context".to_string(), - }); - } - self.inner.set_current()?; - if cleanup.status_count == 0 { - return Ok(CudaExecutionStats::default()); - } - let Some(jobs_buffer) = cleanup.resources.first() else { - return Err(CudaError::InvalidArgument { - message: "queued HTJ2K cleanup has no metadata buffer".to_string(), - }); - }; - self.launch_j2k_dequantize_htj2k_cleanup_jobs_multi( - pooled_device_buffer(jobs_buffer)?, - cleanup.status_count, - CudaLaunchMode::Sync, - )?; - Ok(CudaExecutionStats { - kernel_dispatches: 1, - copy_kernel_dispatches: 0, - decode_kernel_dispatches: 1, - hardware_decode: false, - }) - } - pub(crate) fn decode_empty_htj2k_codeblocks( &self, jobs: &[CudaHtj2kCodeBlockJob], diff --git a/crates/j2k-cuda-runtime/src/context/creation.rs b/crates/j2k-cuda-runtime/src/context/creation.rs index 3d91f42c..09bcee2a 100644 --- a/crates/j2k-cuda-runtime/src/context/creation.rs +++ b/crates/j2k-cuda-runtime/src/context/creation.rs @@ -8,6 +8,7 @@ use std::{ use crate::{driver::Driver, error::CudaError, memory::PinnedUploadStagingPool}; use super::{ + diagnostics::{CudaContextDiagnosticsState, CudaEventPoolState}, host_budget::SharedCudaHostBudget, inner::{ContextInner, ContextOwnership}, validate_resource_handle, ContextResourceLifecycle, CudaContext, @@ -68,8 +69,7 @@ pub(super) fn create_context( ) { match ownership { ContextOwnership::Owned => { - // SAFETY: a successful context-create call is balanced on the - // validation failure path before ownership can escape. + // SAFETY: balance successful creation before ownership can escape. let _ = unsafe { (driver.cu_ctx_destroy)(context) }; } ContextOwnership::RetainedPrimary { device } => { @@ -88,6 +88,8 @@ pub(super) fn create_context( ownership, device_ordinal, modules: Mutex::new(HashMap::new()), + event_pool: Mutex::new(CudaEventPoolState::default()), + diagnostics: CudaContextDiagnosticsState::default(), pinned_upload_operation: Mutex::new(()), pinned_upload_staging: Mutex::new(PinnedUploadStagingPool::new()), host_budget: SharedCudaHostBudget::new(), diff --git a/crates/j2k-cuda-runtime/src/context/diagnostics.rs b/crates/j2k-cuda-runtime/src/context/diagnostics.rs new file mode 100644 index 00000000..07c3b224 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/context/diagnostics.rs @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::{driver::CuEvent, error::CudaError}; + +use super::CudaContext; + +#[derive(Debug, Default)] +pub(crate) struct CudaEventPoolState { + cached: Vec, + driver_allocations: u64, + reuses: u64, +} + +#[derive(Debug, Default)] +pub(crate) struct CudaContextDiagnosticsState { + host_to_device_operations: AtomicU64, + host_to_device_bytes: AtomicU64, + device_to_host_operations: AtomicU64, + device_to_host_bytes: AtomicU64, + status_device_to_host_operations: AtomicU64, + status_device_to_host_bytes: AtomicU64, + kernel_launches: AtomicU64, + device_allocation_operations: AtomicU64, + device_allocation_bytes: AtomicU64, + live_device_allocations: AtomicU64, + live_device_bytes: AtomicU64, + peak_live_device_allocations: AtomicU64, + peak_live_device_bytes: AtomicU64, + event_host_synchronizations: AtomicU64, + context_host_synchronizations: AtomicU64, +} + +impl CudaContextDiagnosticsState { + fn record_host_to_device_copy(&self, byte_len: usize) { + atomic_saturating_add(&self.host_to_device_operations, 1); + atomic_saturating_add(&self.host_to_device_bytes, usize_as_u64(byte_len)); + } + + fn record_kernel_launch(&self) { + atomic_saturating_add(&self.kernel_launches, 1); + } + + fn record_device_allocation(&self, byte_len: usize) { + atomic_saturating_add(&self.device_allocation_operations, 1); + atomic_saturating_add(&self.device_allocation_bytes, usize_as_u64(byte_len)); + let live_allocations = atomic_saturating_add(&self.live_device_allocations, 1); + let live_bytes = atomic_saturating_add(&self.live_device_bytes, usize_as_u64(byte_len)); + self.peak_live_device_allocations + .fetch_max(live_allocations, Ordering::Relaxed); + self.peak_live_device_bytes + .fetch_max(live_bytes, Ordering::Relaxed); + } + + fn record_device_free(&self, byte_len: usize) { + atomic_saturating_sub(&self.live_device_allocations, 1); + atomic_saturating_sub(&self.live_device_bytes, usize_as_u64(byte_len)); + } +} + +impl CudaEventPoolState { + pub(crate) fn take(&mut self) -> Option { + let event = self.cached.pop()?; + self.reuses = self.reuses.saturating_add(1); + Some(event as CuEvent) + } + + pub(crate) fn record_driver_allocation(&mut self) { + self.driver_allocations = self.driver_allocations.saturating_add(1); + } + + pub(crate) fn recycle(&mut self, event: CuEvent) -> Result<(), ()> { + self.cached.try_reserve(1).map_err(|_| ())?; + self.cached.push(event as usize); + Ok(()) + } + + pub(crate) fn drain(&mut self) -> impl Iterator + '_ { + self.cached.drain(..).map(|event| event as CuEvent) + } +} + +/// Monotonic runtime-work counters and the current event-cache size for one CUDA context. +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaContextDiagnostics { + /// Successful host-to-device copy operations submitted through this runtime. + pub host_to_device_operations: u64, + /// Successful host-to-device bytes copied through this runtime. + pub host_to_device_bytes: u64, + /// Successful device-to-host copy operations submitted through this runtime. + pub device_to_host_operations: u64, + /// Successful device-to-host bytes copied through this runtime. + pub device_to_host_bytes: u64, + /// Device-to-host operations used specifically to validate codec statuses. + pub status_device_to_host_operations: u64, + /// Device-to-host bytes used specifically to validate codec statuses. + pub status_device_to_host_bytes: u64, + /// Successful kernel submissions made through this runtime. + pub kernel_launches: u64, + /// Successful runtime-owned CUDA device allocation operations. + pub device_allocation_operations: u64, + /// Cumulative bytes requested by successful runtime-owned device allocations. + pub device_allocation_bytes: u64, + /// Runtime-owned CUDA device allocations that have not been successfully freed. + pub live_device_allocations: u64, + /// Runtime-owned CUDA device bytes that have not been successfully freed. + pub live_device_bytes: u64, + /// Highest observed number of simultaneous runtime-owned device allocations. + pub peak_live_device_allocations: u64, + /// Highest observed runtime-owned device byte total. + pub peak_live_device_bytes: u64, + /// CUDA event handles created by the driver for this context. + pub event_driver_allocations: u64, + /// Event checkouts satisfied from this context's re-recordable event cache. + pub event_reuses: u64, + /// Event handles currently retained for reuse by this context. + pub cached_events: usize, + /// Successful host waits on individual CUDA events. + pub event_host_synchronizations: u64, + /// Successful context-wide host synchronization operations. + pub context_host_synchronizations: u64, +} + +impl CudaContext { + /// Snapshot runtime-work diagnostics without synchronizing CUDA work. + #[doc(hidden)] + pub fn diagnostics(&self) -> Result { + self.inner.ensure_resource_lifetime_available()?; + let events = self + .inner + .event_pool + .lock() + .map_err(|error| CudaError::StatePoisoned { + message: error.to_string(), + })?; + Ok(CudaContextDiagnostics { + host_to_device_operations: self + .inner + .diagnostics + .host_to_device_operations + .load(Ordering::Relaxed), + host_to_device_bytes: self + .inner + .diagnostics + .host_to_device_bytes + .load(Ordering::Relaxed), + device_to_host_operations: self + .inner + .diagnostics + .device_to_host_operations + .load(Ordering::Relaxed), + device_to_host_bytes: self + .inner + .diagnostics + .device_to_host_bytes + .load(Ordering::Relaxed), + status_device_to_host_operations: self + .inner + .diagnostics + .status_device_to_host_operations + .load(Ordering::Relaxed), + status_device_to_host_bytes: self + .inner + .diagnostics + .status_device_to_host_bytes + .load(Ordering::Relaxed), + kernel_launches: self + .inner + .diagnostics + .kernel_launches + .load(Ordering::Relaxed), + device_allocation_operations: self + .inner + .diagnostics + .device_allocation_operations + .load(Ordering::Relaxed), + device_allocation_bytes: self + .inner + .diagnostics + .device_allocation_bytes + .load(Ordering::Relaxed), + live_device_allocations: self + .inner + .diagnostics + .live_device_allocations + .load(Ordering::Relaxed), + live_device_bytes: self + .inner + .diagnostics + .live_device_bytes + .load(Ordering::Relaxed), + peak_live_device_allocations: self + .inner + .diagnostics + .peak_live_device_allocations + .load(Ordering::Relaxed), + peak_live_device_bytes: self + .inner + .diagnostics + .peak_live_device_bytes + .load(Ordering::Relaxed), + event_driver_allocations: events.driver_allocations, + event_reuses: events.reuses, + cached_events: events.cached.len(), + event_host_synchronizations: self + .inner + .diagnostics + .event_host_synchronizations + .load(Ordering::Relaxed), + context_host_synchronizations: self + .inner + .diagnostics + .context_host_synchronizations + .load(Ordering::Relaxed), + }) + } + + pub(crate) fn record_device_to_host_copy(&self, byte_len: usize) { + atomic_saturating_add(&self.inner.diagnostics.device_to_host_operations, 1); + atomic_saturating_add( + &self.inner.diagnostics.device_to_host_bytes, + u64::try_from(byte_len).unwrap_or(u64::MAX), + ); + } + + pub(crate) fn record_host_to_device_copy(&self, byte_len: usize) { + self.inner.diagnostics.record_host_to_device_copy(byte_len); + } + + pub(crate) fn record_kernel_launch(&self) { + self.inner.diagnostics.record_kernel_launch(); + } + + pub(crate) fn record_device_allocation(&self, byte_len: usize) { + self.inner.diagnostics.record_device_allocation(byte_len); + } + + pub(crate) fn record_device_free(&self, byte_len: usize) { + self.inner.diagnostics.record_device_free(byte_len); + } + + pub(crate) fn record_status_device_to_host_copy(&self, byte_len: usize) { + atomic_saturating_add(&self.inner.diagnostics.status_device_to_host_operations, 1); + atomic_saturating_add( + &self.inner.diagnostics.status_device_to_host_bytes, + u64::try_from(byte_len).unwrap_or(u64::MAX), + ); + } + + pub(crate) fn record_event_host_synchronization(&self) { + atomic_saturating_add(&self.inner.diagnostics.event_host_synchronizations, 1); + } + + pub(crate) fn record_context_host_synchronization(&self) { + atomic_saturating_add(&self.inner.diagnostics.context_host_synchronizations, 1); + } +} + +fn usize_as_u64(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +fn atomic_saturating_add(counter: &AtomicU64, increment: u64) -> u64 { + counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_add(increment)) + }) + .map_or_else( + |current| current, + |previous| previous.saturating_add(increment), + ) +} + +fn atomic_saturating_sub(counter: &AtomicU64, decrement: u64) -> u64 { + counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_sub(decrement)) + }) + .map_or_else( + |current| current, + |previous| previous.saturating_sub(decrement), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn diagnostics_state_tracks_successful_runtime_work_and_live_allocation_peaks() { + let state = CudaContextDiagnosticsState::default(); + + state.record_host_to_device_copy(17); + state.record_host_to_device_copy(5); + state.record_kernel_launch(); + state.record_kernel_launch(); + state.record_device_allocation(64); + state.record_device_allocation(96); + state.record_device_free(64); + state.record_device_allocation(32); + + assert_eq!(state.host_to_device_operations.load(Ordering::Relaxed), 2); + assert_eq!(state.host_to_device_bytes.load(Ordering::Relaxed), 22); + assert_eq!(state.kernel_launches.load(Ordering::Relaxed), 2); + assert_eq!( + state.device_allocation_operations.load(Ordering::Relaxed), + 3 + ); + assert_eq!(state.device_allocation_bytes.load(Ordering::Relaxed), 192); + assert_eq!(state.live_device_allocations.load(Ordering::Relaxed), 2); + assert_eq!(state.live_device_bytes.load(Ordering::Relaxed), 128); + assert_eq!( + state.peak_live_device_allocations.load(Ordering::Relaxed), + 2 + ); + assert_eq!(state.peak_live_device_bytes.load(Ordering::Relaxed), 160); + } +} diff --git a/crates/j2k-cuda-runtime/src/context/inner.rs b/crates/j2k-cuda-runtime/src/context/inner.rs index a8dac092..b0c442f0 100644 --- a/crates/j2k-cuda-runtime/src/context/inner.rs +++ b/crates/j2k-cuda-runtime/src/context/inner.rs @@ -8,7 +8,10 @@ use crate::{ }; use super::host_budget::SharedCudaHostBudget; -use super::{CompiledKernel, CompiledKernelKey, ContextResourceLifecycle}; +use super::{ + diagnostics::{CudaContextDiagnosticsState, CudaEventPoolState}, + CompiledKernel, CompiledKernelKey, ContextResourceLifecycle, +}; #[derive(Debug, Clone, Copy)] pub(crate) enum ContextOwnership { @@ -22,6 +25,8 @@ pub(crate) struct ContextInner { pub(crate) ownership: ContextOwnership, pub(crate) device_ordinal: usize, pub(crate) modules: Mutex>, + pub(crate) event_pool: Mutex, + pub(crate) diagnostics: CudaContextDiagnosticsState, pub(crate) pinned_upload_operation: Mutex<()>, pub(crate) pinned_upload_staging: Mutex, pub(crate) host_budget: std::sync::Arc, @@ -53,6 +58,15 @@ impl Drop for ContextInner { // cannot surface errors, so cleanup failures are ignored. let _ = unsafe { (self.driver.cu_module_unload)(compiled.module) }; } + let event_pool = match self.event_pool.get_mut() { + Ok(event_pool) => event_pool, + Err(poisoned) => poisoned.into_inner(), + }; + for event in event_pool.drain() { + // SAFETY: cached event handles were created by this CUDA + // context and are released before the context itself. + let _ = unsafe { (self.driver.cu_event_destroy)(event) }; + } } match self.ownership { ContextOwnership::Owned => { diff --git a/crates/j2k-cuda-runtime/src/context/test_kernels.rs b/crates/j2k-cuda-runtime/src/context/test_kernels.rs index aedd42ef..04ef4c47 100644 --- a/crates/j2k-cuda-runtime/src/context/test_kernels.rs +++ b/crates/j2k-cuda-runtime/src/context/test_kernels.rs @@ -23,9 +23,18 @@ pub(crate) enum CudaKernelName { J2kIdwtVertical97, J2kInverseMct, J2kStoreGray8, + J2kStoreGray8Batch, J2kStoreGray16, + J2kStoreGray16Batch, + J2kStoreGrayI16Batch, J2kStoreRgb8, J2kStoreRgb8MctBatch, + J2kStoreRgb8NativeBatch, + J2kStoreRgb16NativeBatch, + J2kStoreRgbI16NativeBatch, + J2kStoreRgba8NativeBatch, + J2kStoreRgba16NativeBatch, + J2kStoreRgbaI16NativeBatch, J2kStoreRgb16, J2kStoreRgb16Mct, Htj2kEncodeCodeblocks, @@ -67,9 +76,18 @@ impl CudaKernelName { Self::J2kIdwtVertical97 => CudaKernel::J2kIdwtVertical97, Self::J2kInverseMct => CudaKernel::J2kInverseMct, Self::J2kStoreGray8 => CudaKernel::J2kStoreGray8, + Self::J2kStoreGray8Batch => CudaKernel::J2kStoreGray8Batch, Self::J2kStoreGray16 => CudaKernel::J2kStoreGray16, + Self::J2kStoreGray16Batch => CudaKernel::J2kStoreGray16Batch, + Self::J2kStoreGrayI16Batch => CudaKernel::J2kStoreGrayI16Batch, Self::J2kStoreRgb8 => CudaKernel::J2kStoreRgb8, Self::J2kStoreRgb8MctBatch => CudaKernel::J2kStoreRgb8MctBatch, + Self::J2kStoreRgb8NativeBatch => CudaKernel::J2kStoreRgb8NativeBatch, + Self::J2kStoreRgb16NativeBatch => CudaKernel::J2kStoreRgb16NativeBatch, + Self::J2kStoreRgbI16NativeBatch => CudaKernel::J2kStoreRgbI16NativeBatch, + Self::J2kStoreRgba8NativeBatch => CudaKernel::J2kStoreRgba8NativeBatch, + Self::J2kStoreRgba16NativeBatch => CudaKernel::J2kStoreRgba16NativeBatch, + Self::J2kStoreRgbaI16NativeBatch => CudaKernel::J2kStoreRgbaI16NativeBatch, Self::J2kStoreRgb16 => CudaKernel::J2kStoreRgb16, Self::J2kStoreRgb16Mct => CudaKernel::J2kStoreRgb16Mct, Self::Htj2kEncodeCodeblocks => CudaKernel::Htj2kEncodeCodeblocks, @@ -107,9 +125,18 @@ impl CudaKernelName { Self::J2kIdwtVertical97 => "j2k_idwt_vertical_97", Self::J2kInverseMct => "j2k_inverse_mct", Self::J2kStoreGray8 => "j2k_store_gray8", + Self::J2kStoreGray8Batch => "j2k_store_gray8_batch", Self::J2kStoreGray16 => "j2k_store_gray16", + Self::J2kStoreGray16Batch => "j2k_store_gray16_batch", + Self::J2kStoreGrayI16Batch => "j2k_store_grayi16_batch", Self::J2kStoreRgb8 => "j2k_store_rgb8", Self::J2kStoreRgb8MctBatch => "j2k_store_rgb8_mct_batch", + Self::J2kStoreRgb8NativeBatch => "j2k_store_rgb8_native_batch", + Self::J2kStoreRgb16NativeBatch => "j2k_store_rgb16_native_batch", + Self::J2kStoreRgbI16NativeBatch => "j2k_store_rgbi16_native_batch", + Self::J2kStoreRgba8NativeBatch => "j2k_store_rgba8_native_batch", + Self::J2kStoreRgba16NativeBatch => "j2k_store_rgba16_native_batch", + Self::J2kStoreRgbaI16NativeBatch => "j2k_store_rgbai16_native_batch", Self::J2kStoreRgb16 => "j2k_store_rgb16", Self::J2kStoreRgb16Mct => "j2k_store_rgb16_mct", Self::Htj2kEncodeCodeblocks => "j2k_htj2k_encode_codeblocks", diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_htj2k_decode/simt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_htj2k_decode/simt/src/main.rs index bdb936ae..47f3ce38 100644 --- a/crates/j2k-cuda-runtime/src/cuda_oxide_htj2k_decode/simt/src/main.rs +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_htj2k_decode/simt/src/main.rs @@ -362,9 +362,12 @@ fn forward_reader_fill(reader: &mut ForwardBitReader) { } else { reader.pad }; + let valid_bits = 8 - reader.unstuff as u32; + let next_unstuff = byte == 0xff; + let byte = if reader.unstuff { byte & 0x7f } else { byte }; reader.tmp |= (byte as u64) << reader.bits; - reader.bits += 8 - reader.unstuff as u32; - reader.unstuff = byte == 0xff; + reader.bits += valid_bits; + reader.unstuff = next_unstuff; } } @@ -419,10 +422,13 @@ fn reverse_reader_fill(reader: &mut ReverseBitReader) { } else { 0 }; - let d_bits = 8 - (reader.unstuff && (byte & 0x7f) == 0x7f) as u32; + let stuffed = reader.unstuff && (byte & 0x7f) == 0x7f; + let d_bits = 8 - stuffed as u32; + let next_unstuff = byte > 0x8f; + let byte = if stuffed { byte & 0x7f } else { byte }; reader.tmp |= (byte as u64) << reader.bits; reader.bits += d_bits; - reader.unstuff = byte > 0x8f; + reader.unstuff = next_unstuff; } } @@ -966,9 +972,8 @@ fn apply_significance_propagation( forward_reader_advance(&mut sigprop, cnt); } - let combined_sig = new_sig | cs; + let combined_sig = new_sig | (cs & 0xffff); prev_row_sig[idx as usize] = combined_sig as u16; - prev_row_sig[idx as usize + 1] = (combined_sig >> 16) as u16; let combined = combined_sig; let mut next_prev = combined_sig; diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/abi.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/abi.rs new file mode 100644 index 00000000..47c3a371 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/abi.rs @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreGray8Job { + pub(crate) input_width: u32, + pub(crate) source_x: u32, + pub(crate) source_y: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend: f32, + pub(crate) bit_depth: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreGray16Job { + pub(crate) input_width: u32, + pub(crate) source_x: u32, + pub(crate) source_y: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend: f32, + pub(crate) bit_depth: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreGray8BatchJob { + pub(crate) input_ptr: u64, + pub(crate) output_ptr: u64, + pub(crate) job: CudaJ2kStoreGray8Job, + pub(crate) reserved_tail: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreGray16BatchJob { + pub(crate) input_ptr: u64, + pub(crate) output_ptr: u64, + pub(crate) job: CudaJ2kStoreGray16Job, + pub(crate) reserved_tail: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreGrayI16BatchJob { + pub(crate) input_ptr: u64, + pub(crate) output_ptr: u64, + pub(crate) job: CudaJ2kStoreGray16Job, + pub(crate) reserved_tail: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kInverseMctJob { + pub(crate) len: u32, + pub(crate) irreversible97: u32, + pub(crate) addend0: f32, + pub(crate) addend1: f32, + pub(crate) addend2: f32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreRgb8Job { + pub(crate) input_width0: u32, + pub(crate) input_width1: u32, + pub(crate) input_width2: u32, + pub(crate) source_x0: u32, + pub(crate) source_y0: u32, + pub(crate) source_x1: u32, + pub(crate) source_y1: u32, + pub(crate) source_x2: u32, + pub(crate) source_y2: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend0: f32, + pub(crate) addend1: f32, + pub(crate) addend2: f32, + pub(crate) bit_depth0: u32, + pub(crate) bit_depth1: u32, + pub(crate) bit_depth2: u32, + pub(crate) rgba: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreRgb16Job { + pub(crate) input_width0: u32, + pub(crate) input_width1: u32, + pub(crate) input_width2: u32, + pub(crate) source_x0: u32, + pub(crate) source_y0: u32, + pub(crate) source_x1: u32, + pub(crate) source_y1: u32, + pub(crate) source_x2: u32, + pub(crate) source_y2: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend0: f32, + pub(crate) addend1: f32, + pub(crate) addend2: f32, + pub(crate) bit_depth0: u32, + pub(crate) bit_depth1: u32, + pub(crate) bit_depth2: u32, + pub(crate) rgba: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreRgb8MctJob { + pub(crate) store: CudaJ2kStoreRgb8Job, + pub(crate) irreversible97: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreRgb8MctBatchJob { + pub(crate) plane0_ptr: u64, + pub(crate) plane1_ptr: u64, + pub(crate) plane2_ptr: u64, + pub(crate) output_ptr: u64, + pub(crate) job: CudaJ2kStoreRgb8MctJob, + pub(crate) reserved_tail: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreRgb16MctJob { + pub(crate) store: CudaJ2kStoreRgb16Job, + pub(crate) irreversible97: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreRgbNativeJob { + pub(crate) input_width0: u32, + pub(crate) input_width1: u32, + pub(crate) input_width2: u32, + pub(crate) source_x0: u32, + pub(crate) source_y0: u32, + pub(crate) source_x1: u32, + pub(crate) source_y1: u32, + pub(crate) source_x2: u32, + pub(crate) source_y2: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend0: f32, + pub(crate) addend1: f32, + pub(crate) addend2: f32, + pub(crate) bit_depth0: u32, + pub(crate) bit_depth1: u32, + pub(crate) bit_depth2: u32, + pub(crate) layout: u32, + pub(crate) transform: u32, + pub(crate) reserved: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreRgbNativeBatchJob { + pub(crate) plane0_ptr: u64, + pub(crate) plane1_ptr: u64, + pub(crate) plane2_ptr: u64, + pub(crate) output_ptr: u64, + pub(crate) job: CudaJ2kStoreRgbNativeJob, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreRgbaNativeJob { + pub(crate) input_width0: u32, + pub(crate) input_width1: u32, + pub(crate) input_width2: u32, + pub(crate) input_width3: u32, + pub(crate) source_x0: u32, + pub(crate) source_y0: u32, + pub(crate) source_x1: u32, + pub(crate) source_y1: u32, + pub(crate) source_x2: u32, + pub(crate) source_y2: u32, + pub(crate) source_x3: u32, + pub(crate) source_y3: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend0: f32, + pub(crate) addend1: f32, + pub(crate) addend2: f32, + pub(crate) addend3: f32, + pub(crate) bit_depth0: u32, + pub(crate) bit_depth1: u32, + pub(crate) bit_depth2: u32, + pub(crate) bit_depth3: u32, + pub(crate) layout: u32, + pub(crate) transform: u32, + pub(crate) reserved: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct CudaJ2kStoreRgbaNativeBatchJob { + pub(crate) plane0_ptr: u64, + pub(crate) plane1_ptr: u64, + pub(crate) plane2_ptr: u64, + pub(crate) plane3_ptr: u64, + pub(crate) output_ptr: u64, + pub(crate) job: CudaJ2kStoreRgbaNativeJob, + pub(crate) reserved_tail: u32, +} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/color.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/color.rs new file mode 100644 index 00000000..1fc48840 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/color.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + abi::{ + CudaJ2kStoreRgb8Job, CudaJ2kStoreRgb8MctJob, CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb16MctJob, + }, + layout::{output_pixel_index, pixel_coords, source_index}, + memory::{load_f32, store_u8, store_u16}, + sample::{sample_as_u8, sample_as_u16}, + transform::inverse_mct_sample, +}; + +#[inline(always)] +pub(crate) fn store_rgb8_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u8, + job: CudaJ2kStoreRgb8Job, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let channels = if job.rgba != 0 { 4 } else { 3 }; + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; + + store_u8( + output, + dst, + sample_as_u8(load_f32(plane0, src0) + job.addend0, job.bit_depth0), + ); + store_u8( + output, + dst + 1, + sample_as_u8(load_f32(plane1, src1) + job.addend1, job.bit_depth1), + ); + store_u8( + output, + dst + 2, + sample_as_u8(load_f32(plane2, src2) + job.addend2, job.bit_depth2), + ); + if job.rgba != 0 { + store_u8(output, dst + 3, 255); + } +} + +#[inline(always)] +pub(crate) fn store_rgb16_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u16, + job: CudaJ2kStoreRgb16Job, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let channels = if job.rgba != 0 { 4 } else { 3 }; + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; + + store_u16( + output, + dst, + sample_as_u16(load_f32(plane0, src0) + job.addend0, job.bit_depth0), + ); + store_u16( + output, + dst + 1, + sample_as_u16(load_f32(plane1, src1) + job.addend1, job.bit_depth1), + ); + store_u16( + output, + dst + 2, + sample_as_u16(load_f32(plane2, src2) + job.addend2, job.bit_depth2), + ); + if job.rgba != 0 { + store_u16(output, dst + 3, 65535); + } +} + +#[inline(always)] +pub(crate) fn store_rgb8_mct_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u8, + mct_job: CudaJ2kStoreRgb8MctJob, + gid: u32, +) { + let job = mct_job.store; + let (row, col) = pixel_coords(gid, job.copy_width); + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let channels = if job.rgba != 0 { 4 } else { 3 }; + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; + let (out0, out1, out2) = inverse_mct_sample( + load_f32(plane0, src0), + load_f32(plane1, src1), + load_f32(plane2, src2), + mct_job.irreversible97, + ); + + store_u8( + output, + dst, + sample_as_u8(out0 + job.addend0, job.bit_depth0), + ); + store_u8( + output, + dst + 1, + sample_as_u8(out1 + job.addend1, job.bit_depth1), + ); + store_u8( + output, + dst + 2, + sample_as_u8(out2 + job.addend2, job.bit_depth2), + ); + if job.rgba != 0 { + store_u8(output, dst + 3, 255); + } +} + +#[inline(always)] +pub(crate) fn store_rgb16_mct_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u16, + mct_job: CudaJ2kStoreRgb16MctJob, + gid: u32, +) { + let job = mct_job.store; + let (row, col) = pixel_coords(gid, job.copy_width); + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let channels = if job.rgba != 0 { 4 } else { 3 }; + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; + let (out0, out1, out2) = inverse_mct_sample( + load_f32(plane0, src0), + load_f32(plane1, src1), + load_f32(plane2, src2), + mct_job.irreversible97, + ); + + store_u16( + output, + dst, + sample_as_u16(out0 + job.addend0, job.bit_depth0), + ); + store_u16( + output, + dst + 1, + sample_as_u16(out1 + job.addend1, job.bit_depth1), + ); + store_u16( + output, + dst + 2, + sample_as_u16(out2 + job.addend2, job.bit_depth2), + ); + if job.rgba != 0 { + store_u16(output, dst + 3, 65535); + } +} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/exports.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/exports.rs new file mode 100644 index 00000000..2d84928b --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/exports.rs @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use cuda_host::cuda_module; + +#[cuda_module] +mod kernels { + use crate::{ + abi::{ + CudaJ2kInverseMctJob, CudaJ2kStoreGray8BatchJob, CudaJ2kStoreGray8Job, + CudaJ2kStoreGray16BatchJob, CudaJ2kStoreGray16Job, CudaJ2kStoreGrayI16BatchJob, + CudaJ2kStoreRgb8Job, CudaJ2kStoreRgb8MctBatchJob, CudaJ2kStoreRgb16Job, + CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgbNativeBatchJob, CudaJ2kStoreRgbaNativeBatchJob, + }, + color::{ + store_rgb8_mct_sample, store_rgb8_sample, store_rgb16_mct_sample, store_rgb16_sample, + }, + layout::{output_pixel_index, pixel_coords, source_index}, + memory::{load_f32, load_job, store_f32, store_i16, store_u8, store_u16}, + sample::{ + sample_as_native_i16, sample_as_native_u8, sample_as_native_u16, sample_as_u8, + sample_as_u16, + }, + native_color::{ + store_rgb8_native_sample, store_rgb16_native_sample, store_rgba8_native_sample, + store_rgba16_native_sample, store_rgbai16_native_sample, store_rgbi16_native_sample, + }, + transform::inverse_mct_sample, + }; + use cuda_device::{kernel, thread}; + + #[kernel] + pub unsafe fn j2k_store_gray8( + input: *const f32, + output: *mut u8, + job_buffer: *const CudaJ2kStoreGray8Job, + ) { + let job = load_job(job_buffer); + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + let (row, col) = pixel_coords(gid, job.copy_width); + let src = source_index(job.input_width, job.source_x, job.source_y, row, col); + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + store_u8( + output, + dst, + sample_as_u8(load_f32(input, src) + job.addend, job.bit_depth), + ); + } + + #[kernel] + pub unsafe fn j2k_store_gray16( + input: *const f32, + output: *mut u16, + job_buffer: *const CudaJ2kStoreGray16Job, + ) { + let job = load_job(job_buffer); + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + let (row, col) = pixel_coords(gid, job.copy_width); + let src = source_index(job.input_width, job.source_x, job.source_y, row, col); + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + store_u16( + output, + dst, + sample_as_u16(load_f32(input, src) + job.addend, job.bit_depth), + ); + } + + #[kernel] + pub unsafe fn j2k_store_grayi16_batch(jobs: *const CudaJ2kStoreGrayI16BatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let input = item.input_ptr as usize as *const f32; + let output = item.output_ptr as usize as *mut i16; + let job = item.job; + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + + let (row, col) = pixel_coords(gid, job.copy_width); + let src = source_index(job.input_width, job.source_x, job.source_y, row, col); + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + store_i16( + output, + dst, + sample_as_native_i16(load_f32(input, src) + job.addend, job.bit_depth), + ); + } + + #[kernel] + pub unsafe fn j2k_store_gray8_batch(jobs: *const CudaJ2kStoreGray8BatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let input = item.input_ptr as usize as *const f32; + let output = item.output_ptr as usize as *mut u8; + let job = item.job; + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + + let (row, col) = pixel_coords(gid, job.copy_width); + let src = source_index(job.input_width, job.source_x, job.source_y, row, col); + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + store_u8( + output, + dst, + sample_as_native_u8(load_f32(input, src) + job.addend, job.bit_depth), + ); + } + + #[kernel] + pub unsafe fn j2k_store_gray16_batch(jobs: *const CudaJ2kStoreGray16BatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let input = item.input_ptr as usize as *const f32; + let output = item.output_ptr as usize as *mut u16; + let job = item.job; + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + + let (row, col) = pixel_coords(gid, job.copy_width); + let src = source_index(job.input_width, job.source_x, job.source_y, row, col); + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + store_u16( + output, + dst, + sample_as_native_u16(load_f32(input, src) + job.addend, job.bit_depth), + ); + } + + #[kernel] + pub unsafe fn j2k_inverse_mct( + plane0: *mut f32, + plane1: *mut f32, + plane2: *mut f32, + job_buffer: *const CudaJ2kInverseMctJob, + ) { + let job = load_job(job_buffer); + let gid = thread::index_1d().get() as u32; + if gid >= job.len { + return; + } + + let (out0, out1, out2) = inverse_mct_sample( + load_f32(plane0.cast_const(), gid), + load_f32(plane1.cast_const(), gid), + load_f32(plane2.cast_const(), gid), + job.irreversible97, + ); + store_f32(plane0, gid, out0 + job.addend0); + store_f32(plane1, gid, out1 + job.addend1); + store_f32(plane2, gid, out2 + job.addend2); + } + + #[kernel] + pub unsafe fn j2k_store_rgb8( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u8, + job_buffer: *const CudaJ2kStoreRgb8Job, + ) { + let job = load_job(job_buffer); + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + store_rgb8_sample(plane0, plane1, plane2, output, job, gid); + } + + #[kernel] + pub unsafe fn j2k_store_rgb16( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u16, + job_buffer: *const CudaJ2kStoreRgb16Job, + ) { + let job = load_job(job_buffer); + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + store_rgb16_sample(plane0, plane1, plane2, output, job, gid); + } + + #[kernel] + pub unsafe fn j2k_store_rgb8_mct_batch(jobs: *const CudaJ2kStoreRgb8MctBatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let plane0 = item.plane0_ptr as usize as *const f32; + let plane1 = item.plane1_ptr as usize as *const f32; + let plane2 = item.plane2_ptr as usize as *const f32; + let output = item.output_ptr as usize as *mut u8; + let pixels = item.job.store.copy_width * item.job.store.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + + store_rgb8_mct_sample(plane0, plane1, plane2, output, item.job, gid); + } + + #[kernel] + pub unsafe fn j2k_store_rgb16_mct( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u16, + job_buffer: *const CudaJ2kStoreRgb16MctJob, + ) { + let mct_job = load_job(job_buffer); + let pixels = mct_job.store.copy_width * mct_job.store.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + store_rgb16_mct_sample(plane0, plane1, plane2, output, mct_job, gid); + } + + #[kernel] + pub unsafe fn j2k_store_rgb8_native_batch(jobs: *const CudaJ2kStoreRgbNativeBatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let pixels = item.job.copy_width * item.job.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + store_rgb8_native_sample( + item.plane0_ptr as usize as *const f32, + item.plane1_ptr as usize as *const f32, + item.plane2_ptr as usize as *const f32, + item.output_ptr as usize as *mut u8, + item.job, + gid, + ); + } + + #[kernel] + pub unsafe fn j2k_store_rgb16_native_batch(jobs: *const CudaJ2kStoreRgbNativeBatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let pixels = item.job.copy_width * item.job.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + store_rgb16_native_sample( + item.plane0_ptr as usize as *const f32, + item.plane1_ptr as usize as *const f32, + item.plane2_ptr as usize as *const f32, + item.output_ptr as usize as *mut u16, + item.job, + gid, + ); + } + + #[kernel] + pub unsafe fn j2k_store_rgbi16_native_batch(jobs: *const CudaJ2kStoreRgbNativeBatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let pixels = item.job.copy_width * item.job.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + store_rgbi16_native_sample( + item.plane0_ptr as usize as *const f32, + item.plane1_ptr as usize as *const f32, + item.plane2_ptr as usize as *const f32, + item.output_ptr as usize as *mut i16, + item.job, + gid, + ); + } + + #[kernel] + pub unsafe fn j2k_store_rgba8_native_batch(jobs: *const CudaJ2kStoreRgbaNativeBatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let pixels = item.job.copy_width * item.job.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + store_rgba8_native_sample( + item.plane0_ptr as usize as *const f32, + item.plane1_ptr as usize as *const f32, + item.plane2_ptr as usize as *const f32, + item.plane3_ptr as usize as *const f32, + item.output_ptr as usize as *mut u8, + item.job, + gid, + ); + } + + #[kernel] + pub unsafe fn j2k_store_rgba16_native_batch(jobs: *const CudaJ2kStoreRgbaNativeBatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let pixels = item.job.copy_width * item.job.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + store_rgba16_native_sample( + item.plane0_ptr as usize as *const f32, + item.plane1_ptr as usize as *const f32, + item.plane2_ptr as usize as *const f32, + item.plane3_ptr as usize as *const f32, + item.output_ptr as usize as *mut u16, + item.job, + gid, + ); + } + + #[kernel] + pub unsafe fn j2k_store_rgbai16_native_batch(jobs: *const CudaJ2kStoreRgbaNativeBatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let pixels = item.job.copy_width * item.job.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + store_rgbai16_native_sample( + item.plane0_ptr as usize as *const f32, + item.plane1_ptr as usize as *const f32, + item.plane2_ptr as usize as *const f32, + item.plane3_ptr as usize as *const f32, + item.output_ptr as usize as *mut i16, + item.job, + gid, + ); + } +} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/layout.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/layout.rs new file mode 100644 index 00000000..48930337 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/layout.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Source and destination addressing for final-store kernels. + +#[inline(always)] +pub(crate) fn source_index( + input_width: u32, + source_x: u32, + source_y: u32, + row: u32, + col: u32, +) -> u32 { + (source_y + row) * input_width + source_x + col +} + +#[inline(always)] +pub(crate) fn output_pixel_index( + output_width: u32, + output_x: u32, + output_y: u32, + row: u32, + col: u32, +) -> u32 { + (output_y + row) * output_width + output_x + col +} + +#[inline(always)] +pub(crate) fn pixel_coords(gid: u32, copy_width: u32) -> (u32, u32) { + let row = gid / copy_width; + (row, gid - row * copy_width) +} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/main.rs index 8d96fea9..9d3aa3dc 100644 --- a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/main.rs +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/main.rs @@ -1,550 +1,14 @@ -use cuda_device::{kernel, thread}; -use cuda_host::cuda_module; +// SPDX-License-Identifier: MIT OR Apache-2.0 -include!("../../../cuda_oxide_simt_prelude.rs"); - -#[repr(C)] -#[derive(Clone, Copy)] -struct CudaJ2kStoreGray8Job { - input_width: u32, - source_x: u32, - source_y: u32, - copy_width: u32, - copy_height: u32, - output_width: u32, - output_height: u32, - output_x: u32, - output_y: u32, - addend: f32, - bit_depth: u32, -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct CudaJ2kStoreGray16Job { - input_width: u32, - source_x: u32, - source_y: u32, - copy_width: u32, - copy_height: u32, - output_width: u32, - output_height: u32, - output_x: u32, - output_y: u32, - addend: f32, - bit_depth: u32, -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct CudaJ2kInverseMctJob { - len: u32, - irreversible97: u32, - addend0: f32, - addend1: f32, - addend2: f32, -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct CudaJ2kStoreRgb8Job { - input_width0: u32, - input_width1: u32, - input_width2: u32, - source_x0: u32, - source_y0: u32, - source_x1: u32, - source_y1: u32, - source_x2: u32, - source_y2: u32, - copy_width: u32, - copy_height: u32, - output_width: u32, - output_height: u32, - output_x: u32, - output_y: u32, - addend0: f32, - addend1: f32, - addend2: f32, - bit_depth0: u32, - bit_depth1: u32, - bit_depth2: u32, - rgba: u32, -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct CudaJ2kStoreRgb16Job { - input_width0: u32, - input_width1: u32, - input_width2: u32, - source_x0: u32, - source_y0: u32, - source_x1: u32, - source_y1: u32, - source_x2: u32, - source_y2: u32, - copy_width: u32, - copy_height: u32, - output_width: u32, - output_height: u32, - output_x: u32, - output_y: u32, - addend0: f32, - addend1: f32, - addend2: f32, - bit_depth0: u32, - bit_depth1: u32, - bit_depth2: u32, - rgba: u32, -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct CudaJ2kStoreRgb8MctJob { - store: CudaJ2kStoreRgb8Job, - irreversible97: u32, -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct CudaJ2kStoreRgb8MctBatchJob { - plane0_ptr: u64, - plane1_ptr: u64, - plane2_ptr: u64, - output_ptr: u64, - job: CudaJ2kStoreRgb8MctJob, - reserved_tail: u32, -} - -#[repr(C)] -#[derive(Clone, Copy)] -struct CudaJ2kStoreRgb16MctJob { - store: CudaJ2kStoreRgb16Job, - irreversible97: u32, -} - -#[inline(always)] -fn load_f32(ptr: *const f32, index: u32) -> f32 { - simt_load(ptr, index as usize) -} - -#[inline(always)] -fn load_job(ptr: *const T) -> T { - simt_load(ptr, 0) -} - -#[inline(always)] -fn store_f32(ptr: *mut f32, index: u32, value: f32) { - simt_store(ptr, index as usize, value); -} - -#[inline(always)] -fn store_u8(ptr: *mut u8, index: u32, value: u8) { - simt_store(ptr, index as usize, value); -} - -#[inline(always)] -fn store_u16(ptr: *mut u16, index: u32, value: u16) { - simt_store(ptr, index as usize, value); -} - -#[inline(always)] -fn floor_f32(value: f32) -> f32 { - // f32::floor routes through libdevice in cuda-oxide, which emits NVVM IR - // instead of the PTX loaded by this runtime path. - let truncated = value as i32 as f32; - if truncated > value { - truncated - 1.0 - } else { - truncated - } -} - -#[inline(always)] -fn round_f32(value: f32) -> f32 { - if value >= 0.0 { - floor_f32(value + 0.5) - } else { - -floor_f32(-value + 0.5) - } -} - -#[inline(always)] -fn clamp_f32(value: f32, min: f32, max: f32) -> f32 { - if value < min { - min - } else if value > max { - max - } else { - value - } -} - -#[inline(always)] -fn max_int_for_bit_depth(bit_depth: u32) -> u32 { - if bit_depth == 0 { - 1 - } else { - (1_u32 << bit_depth) - 1 - } -} - -#[inline(always)] -fn sample_as_u8(sample: f32, bit_depth: u32) -> u8 { - let rounded = round_f32(sample); - if bit_depth >= 8 { - return clamp_f32(rounded, 0.0, 255.0) as u8; - } - - let max_int = (1_u32 << bit_depth) - 1; - let max_value = if max_int > 1 { max_int as f32 } else { 1.0 }; - round_f32((clamp_f32(rounded, 0.0, max_value) / max_value) * 255.0) as u8 -} - -#[inline(always)] -fn sample_as_u16(sample: f32, bit_depth: u32) -> u16 { - let rounded = round_f32(sample); - if bit_depth >= 16 { - return clamp_f32(rounded, 0.0, 65535.0) as u16; - } - - let max_int = max_int_for_bit_depth(bit_depth); - let max_value = if max_int > 1 { max_int as f32 } else { 1.0 }; - round_f32((clamp_f32(rounded, 0.0, max_value) / max_value) * 65535.0) as u16 -} - -#[inline(always)] -fn inverse_mct_sample(src0: f32, src1: f32, src2: f32, irreversible97: u32) -> (f32, f32, f32) { - if irreversible97 != 0 { - ( - src0 + 1.402 * src2, - src0 - 0.34413 * src1 - 0.71414 * src2, - src0 + 1.772 * src1, - ) - } else { - let green = src0 - floor_f32((src2 + src1) * 0.25); - (src2 + green, green, src1 + green) - } -} - -#[inline(always)] -fn source_index(input_width: u32, source_x: u32, source_y: u32, row: u32, col: u32) -> u32 { - (source_y + row) * input_width + source_x + col -} - -#[inline(always)] -fn output_pixel_index(output_width: u32, output_x: u32, output_y: u32, row: u32, col: u32) -> u32 { - (output_y + row) * output_width + output_x + col -} +mod abi; +mod color; +mod exports; +mod layout; +mod memory; +mod native_color; +mod sample; +mod transform; -#[inline(always)] -fn pixel_coords(gid: u32, copy_width: u32) -> (u32, u32) { - let row = gid / copy_width; - (row, gid - row * copy_width) -} - -#[inline(always)] -fn store_rgb8_sample( - plane0: *const f32, - plane1: *const f32, - plane2: *const f32, - output: *mut u8, - job: CudaJ2kStoreRgb8Job, - gid: u32, -) { - let (row, col) = pixel_coords(gid, job.copy_width); - let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); - let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); - let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); - let channels = if job.rgba != 0 { 4 } else { 3 }; - let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; - - store_u8( - output, - dst, - sample_as_u8(load_f32(plane0, src0) + job.addend0, job.bit_depth0), - ); - store_u8( - output, - dst + 1, - sample_as_u8(load_f32(plane1, src1) + job.addend1, job.bit_depth1), - ); - store_u8( - output, - dst + 2, - sample_as_u8(load_f32(plane2, src2) + job.addend2, job.bit_depth2), - ); - if job.rgba != 0 { - store_u8(output, dst + 3, 255); - } -} - -#[inline(always)] -fn store_rgb16_sample( - plane0: *const f32, - plane1: *const f32, - plane2: *const f32, - output: *mut u16, - job: CudaJ2kStoreRgb16Job, - gid: u32, -) { - let (row, col) = pixel_coords(gid, job.copy_width); - let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); - let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); - let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); - let channels = if job.rgba != 0 { 4 } else { 3 }; - let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; - - store_u16( - output, - dst, - sample_as_u16(load_f32(plane0, src0) + job.addend0, job.bit_depth0), - ); - store_u16( - output, - dst + 1, - sample_as_u16(load_f32(plane1, src1) + job.addend1, job.bit_depth1), - ); - store_u16( - output, - dst + 2, - sample_as_u16(load_f32(plane2, src2) + job.addend2, job.bit_depth2), - ); - if job.rgba != 0 { - store_u16(output, dst + 3, 65535); - } -} - -#[inline(always)] -fn store_rgb8_mct_sample( - plane0: *const f32, - plane1: *const f32, - plane2: *const f32, - output: *mut u8, - mct_job: CudaJ2kStoreRgb8MctJob, - gid: u32, -) { - let job = mct_job.store; - let (row, col) = pixel_coords(gid, job.copy_width); - let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); - let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); - let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); - let channels = if job.rgba != 0 { 4 } else { 3 }; - let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; - let (out0, out1, out2) = inverse_mct_sample( - load_f32(plane0, src0), - load_f32(plane1, src1), - load_f32(plane2, src2), - mct_job.irreversible97, - ); - - store_u8( - output, - dst, - sample_as_u8(out0 + job.addend0, job.bit_depth0), - ); - store_u8( - output, - dst + 1, - sample_as_u8(out1 + job.addend1, job.bit_depth1), - ); - store_u8( - output, - dst + 2, - sample_as_u8(out2 + job.addend2, job.bit_depth2), - ); - if job.rgba != 0 { - store_u8(output, dst + 3, 255); - } -} - -#[inline(always)] -fn store_rgb16_mct_sample( - plane0: *const f32, - plane1: *const f32, - plane2: *const f32, - output: *mut u16, - mct_job: CudaJ2kStoreRgb16MctJob, - gid: u32, -) { - let job = mct_job.store; - let (row, col) = pixel_coords(gid, job.copy_width); - let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); - let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); - let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); - let channels = if job.rgba != 0 { 4 } else { 3 }; - let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; - let (out0, out1, out2) = inverse_mct_sample( - load_f32(plane0, src0), - load_f32(plane1, src1), - load_f32(plane2, src2), - mct_job.irreversible97, - ); - - store_u16( - output, - dst, - sample_as_u16(out0 + job.addend0, job.bit_depth0), - ); - store_u16( - output, - dst + 1, - sample_as_u16(out1 + job.addend1, job.bit_depth1), - ); - store_u16( - output, - dst + 2, - sample_as_u16(out2 + job.addend2, job.bit_depth2), - ); - if job.rgba != 0 { - store_u16(output, dst + 3, 65535); - } -} - -#[cuda_module] -mod kernels { - use super::*; - - #[kernel] - pub unsafe fn j2k_store_gray8( - input: *const f32, - output: *mut u8, - job_buffer: *const CudaJ2kStoreGray8Job, - ) { - let job = load_job(job_buffer); - let pixels = job.copy_width * job.copy_height; - let gid = thread::index_1d().get() as u32; - if gid >= pixels { - return; - } - - let (row, col) = pixel_coords(gid, job.copy_width); - let src = source_index(job.input_width, job.source_x, job.source_y, row, col); - let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); - store_u8( - output, - dst, - sample_as_u8(load_f32(input, src) + job.addend, job.bit_depth), - ); - } - - #[kernel] - pub unsafe fn j2k_store_gray16( - input: *const f32, - output: *mut u16, - job_buffer: *const CudaJ2kStoreGray16Job, - ) { - let job = load_job(job_buffer); - let pixels = job.copy_width * job.copy_height; - let gid = thread::index_1d().get() as u32; - if gid >= pixels { - return; - } - - let (row, col) = pixel_coords(gid, job.copy_width); - let src = source_index(job.input_width, job.source_x, job.source_y, row, col); - let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); - store_u16( - output, - dst, - sample_as_u16(load_f32(input, src) + job.addend, job.bit_depth), - ); - } - - #[kernel] - pub unsafe fn j2k_inverse_mct( - plane0: *mut f32, - plane1: *mut f32, - plane2: *mut f32, - job_buffer: *const CudaJ2kInverseMctJob, - ) { - let job = load_job(job_buffer); - let gid = thread::index_1d().get() as u32; - if gid >= job.len { - return; - } - - let (out0, out1, out2) = inverse_mct_sample( - load_f32(plane0.cast_const(), gid), - load_f32(plane1.cast_const(), gid), - load_f32(plane2.cast_const(), gid), - job.irreversible97, - ); - store_f32(plane0, gid, out0 + job.addend0); - store_f32(plane1, gid, out1 + job.addend1); - store_f32(plane2, gid, out2 + job.addend2); - } - - #[kernel] - pub unsafe fn j2k_store_rgb8( - plane0: *const f32, - plane1: *const f32, - plane2: *const f32, - output: *mut u8, - job_buffer: *const CudaJ2kStoreRgb8Job, - ) { - let job = load_job(job_buffer); - let pixels = job.copy_width * job.copy_height; - let gid = thread::index_1d().get() as u32; - if gid >= pixels { - return; - } - - store_rgb8_sample(plane0, plane1, plane2, output, job, gid); - } - - #[kernel] - pub unsafe fn j2k_store_rgb16( - plane0: *const f32, - plane1: *const f32, - plane2: *const f32, - output: *mut u16, - job_buffer: *const CudaJ2kStoreRgb16Job, - ) { - let job = load_job(job_buffer); - let pixels = job.copy_width * job.copy_height; - let gid = thread::index_1d().get() as u32; - if gid >= pixels { - return; - } - - store_rgb16_sample(plane0, plane1, plane2, output, job, gid); - } - - #[kernel] - pub unsafe fn j2k_store_rgb8_mct_batch(jobs: *const CudaJ2kStoreRgb8MctBatchJob) { - let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); - let plane0 = item.plane0_ptr as usize as *const f32; - let plane1 = item.plane1_ptr as usize as *const f32; - let plane2 = item.plane2_ptr as usize as *const f32; - let output = item.output_ptr as usize as *mut u8; - let pixels = item.job.store.copy_width * item.job.store.copy_height; - let gid = thread::index_2d_col() as u32; - if gid >= pixels { - return; - } - - store_rgb8_mct_sample(plane0, plane1, plane2, output, item.job, gid); - } - - #[kernel] - pub unsafe fn j2k_store_rgb16_mct( - plane0: *const f32, - plane1: *const f32, - plane2: *const f32, - output: *mut u16, - job_buffer: *const CudaJ2kStoreRgb16MctJob, - ) { - let mct_job = load_job(job_buffer); - let pixels = mct_job.store.copy_width * mct_job.store.copy_height; - let gid = thread::index_1d().get() as u32; - if gid >= pixels { - return; - } - - store_rgb16_mct_sample(plane0, plane1, plane2, output, mct_job, gid); - } -} +include!("../../../cuda_oxide_simt_prelude.rs"); fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/memory.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/memory.rs new file mode 100644 index 00000000..66eb0461 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/memory.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Typed SIMT memory access for final-store kernels. + +use crate::{simt_load, simt_store}; + +#[inline(always)] +pub(crate) fn load_f32(ptr: *const f32, index: u32) -> f32 { + simt_load(ptr, index as usize) +} + +#[inline(always)] +pub(crate) fn load_job(ptr: *const T) -> T { + simt_load(ptr, 0) +} + +#[inline(always)] +pub(crate) fn store_f32(ptr: *mut f32, index: u32, value: f32) { + simt_store(ptr, index as usize, value); +} + +#[inline(always)] +pub(crate) fn store_u8(ptr: *mut u8, index: u32, value: u8) { + simt_store(ptr, index as usize, value); +} + +#[inline(always)] +pub(crate) fn store_u16(ptr: *mut u16, index: u32, value: u16) { + simt_store(ptr, index as usize, value); +} + +#[inline(always)] +pub(crate) fn store_i16(ptr: *mut i16, index: u32, value: i16) { + simt_store(ptr, index as usize, value); +} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/native_color.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/native_color.rs new file mode 100644 index 00000000..4bfe5073 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/native_color.rs @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + abi::{CudaJ2kStoreRgbNativeJob, CudaJ2kStoreRgbaNativeJob}, + layout::{output_pixel_index, pixel_coords, source_index}, + memory::{load_f32, store_i16, store_u8, store_u16}, + sample::{sample_as_native_i16, sample_as_native_u8, sample_as_native_u16}, + transform::inverse_mct_sample, +}; + +#[inline(always)] +pub(crate) fn exact_rgb_samples( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + job: CudaJ2kStoreRgbNativeJob, + row: u32, + col: u32, +) -> (f32, f32, f32) { + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let samples = ( + load_f32(plane0, src0), + load_f32(plane1, src1), + load_f32(plane2, src2), + ); + let (out0, out1, out2) = if job.transform == 0 { + samples + } else { + inverse_mct_sample( + samples.0, + samples.1, + samples.2, + u32::from(job.transform == 2), + ) + }; + (out0 + job.addend0, out1 + job.addend1, out2 + job.addend2) +} + +#[inline(always)] +pub(crate) fn exact_rgb_destination( + job: CudaJ2kStoreRgbNativeJob, + pixel: u32, + channel: u32, +) -> u32 { + if job.layout == 0 { + pixel * 3 + channel + } else { + channel * job.output_width * job.output_height + pixel + } +} + +#[inline(always)] +pub(crate) fn store_rgb8_native_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u8, + job: CudaJ2kStoreRgbNativeJob, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let pixel = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + let samples = exact_rgb_samples(plane0, plane1, plane2, job, row, col); + store_u8( + output, + exact_rgb_destination(job, pixel, 0), + sample_as_native_u8(samples.0, job.bit_depth0), + ); + store_u8( + output, + exact_rgb_destination(job, pixel, 1), + sample_as_native_u8(samples.1, job.bit_depth1), + ); + store_u8( + output, + exact_rgb_destination(job, pixel, 2), + sample_as_native_u8(samples.2, job.bit_depth2), + ); +} + +#[inline(always)] +pub(crate) fn store_rgb16_native_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u16, + job: CudaJ2kStoreRgbNativeJob, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let pixel = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + let samples = exact_rgb_samples(plane0, plane1, plane2, job, row, col); + store_u16( + output, + exact_rgb_destination(job, pixel, 0), + sample_as_native_u16(samples.0, job.bit_depth0), + ); + store_u16( + output, + exact_rgb_destination(job, pixel, 1), + sample_as_native_u16(samples.1, job.bit_depth1), + ); + store_u16( + output, + exact_rgb_destination(job, pixel, 2), + sample_as_native_u16(samples.2, job.bit_depth2), + ); +} + +#[inline(always)] +pub(crate) fn store_rgbi16_native_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut i16, + job: CudaJ2kStoreRgbNativeJob, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let pixel = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + let samples = exact_rgb_samples(plane0, plane1, plane2, job, row, col); + store_i16( + output, + exact_rgb_destination(job, pixel, 0), + sample_as_native_i16(samples.0, job.bit_depth0), + ); + store_i16( + output, + exact_rgb_destination(job, pixel, 1), + sample_as_native_i16(samples.1, job.bit_depth1), + ); + store_i16( + output, + exact_rgb_destination(job, pixel, 2), + sample_as_native_i16(samples.2, job.bit_depth2), + ); +} + +#[inline(always)] +pub(crate) fn exact_rgba_samples( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + plane3: *const f32, + job: CudaJ2kStoreRgbaNativeJob, + row: u32, + col: u32, +) -> (f32, f32, f32, f32) { + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let src3 = source_index(job.input_width3, job.source_x3, job.source_y3, row, col); + let rgb = ( + load_f32(plane0, src0), + load_f32(plane1, src1), + load_f32(plane2, src2), + ); + let (out0, out1, out2) = if job.transform == 0 { + rgb + } else { + inverse_mct_sample(rgb.0, rgb.1, rgb.2, u32::from(job.transform == 2)) + }; + ( + out0 + job.addend0, + out1 + job.addend1, + out2 + job.addend2, + load_f32(plane3, src3) + job.addend3, + ) +} + +#[inline(always)] +pub(crate) fn exact_rgba_destination( + job: CudaJ2kStoreRgbaNativeJob, + pixel: u32, + channel: u32, +) -> u32 { + if job.layout == 0 { + pixel * 4 + channel + } else { + channel * job.output_width * job.output_height + pixel + } +} + +#[inline(always)] +pub(crate) fn store_rgba8_native_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + plane3: *const f32, + output: *mut u8, + job: CudaJ2kStoreRgbaNativeJob, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let pixel = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + let samples = exact_rgba_samples(plane0, plane1, plane2, plane3, job, row, col); + store_u8( + output, + exact_rgba_destination(job, pixel, 0), + sample_as_native_u8(samples.0, job.bit_depth0), + ); + store_u8( + output, + exact_rgba_destination(job, pixel, 1), + sample_as_native_u8(samples.1, job.bit_depth1), + ); + store_u8( + output, + exact_rgba_destination(job, pixel, 2), + sample_as_native_u8(samples.2, job.bit_depth2), + ); + store_u8( + output, + exact_rgba_destination(job, pixel, 3), + sample_as_native_u8(samples.3, job.bit_depth3), + ); +} + +#[inline(always)] +pub(crate) fn store_rgba16_native_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + plane3: *const f32, + output: *mut u16, + job: CudaJ2kStoreRgbaNativeJob, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let pixel = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + let samples = exact_rgba_samples(plane0, plane1, plane2, plane3, job, row, col); + store_u16( + output, + exact_rgba_destination(job, pixel, 0), + sample_as_native_u16(samples.0, job.bit_depth0), + ); + store_u16( + output, + exact_rgba_destination(job, pixel, 1), + sample_as_native_u16(samples.1, job.bit_depth1), + ); + store_u16( + output, + exact_rgba_destination(job, pixel, 2), + sample_as_native_u16(samples.2, job.bit_depth2), + ); + store_u16( + output, + exact_rgba_destination(job, pixel, 3), + sample_as_native_u16(samples.3, job.bit_depth3), + ); +} + +#[inline(always)] +pub(crate) fn store_rgbai16_native_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + plane3: *const f32, + output: *mut i16, + job: CudaJ2kStoreRgbaNativeJob, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let pixel = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + let samples = exact_rgba_samples(plane0, plane1, plane2, plane3, job, row, col); + store_i16( + output, + exact_rgba_destination(job, pixel, 0), + sample_as_native_i16(samples.0, job.bit_depth0), + ); + store_i16( + output, + exact_rgba_destination(job, pixel, 1), + sample_as_native_i16(samples.1, job.bit_depth1), + ); + store_i16( + output, + exact_rgba_destination(job, pixel, 2), + sample_as_native_i16(samples.2, job.bit_depth2), + ); + store_i16( + output, + exact_rgba_destination(job, pixel, 3), + sample_as_native_i16(samples.3, job.bit_depth3), + ); +} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/sample.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/sample.rs new file mode 100644 index 00000000..e5223e39 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/sample.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Native and display-width sample conversion for final-store kernels. + +#[inline(always)] +pub(crate) fn floor_f32(value: f32) -> f32 { + // f32::floor routes through libdevice in cuda-oxide, which emits NVVM IR + // instead of the PTX loaded by this runtime path. + let truncated = value as i32 as f32; + if truncated > value { + truncated - 1.0 + } else { + truncated + } +} + +#[inline(always)] +fn round_f32(value: f32) -> f32 { + if value >= 0.0 { + floor_f32(value + 0.5) + } else { + -floor_f32(-value + 0.5) + } +} + +#[inline(always)] +fn clamp_f32(value: f32, min: f32, max: f32) -> f32 { + if value < min { + min + } else if value > max { + max + } else { + value + } +} + +#[inline(always)] +fn max_int_for_bit_depth(bit_depth: u32) -> u32 { + if bit_depth == 0 { + 1 + } else { + (1_u32 << bit_depth) - 1 + } +} + +#[inline(always)] +pub(crate) fn sample_as_u8(sample: f32, bit_depth: u32) -> u8 { + let rounded = round_f32(sample); + if bit_depth >= 8 { + return clamp_f32(rounded, 0.0, 255.0) as u8; + } + + let max_int = (1_u32 << bit_depth) - 1; + let max_value = if max_int > 1 { max_int as f32 } else { 1.0 }; + round_f32((clamp_f32(rounded, 0.0, max_value) / max_value) * 255.0) as u8 +} + +#[inline(always)] +pub(crate) fn sample_as_u16(sample: f32, bit_depth: u32) -> u16 { + let rounded = round_f32(sample); + if bit_depth >= 16 { + return clamp_f32(rounded, 0.0, 65535.0) as u16; + } + + let max_int = max_int_for_bit_depth(bit_depth); + let max_value = if max_int > 1 { max_int as f32 } else { 1.0 }; + round_f32((clamp_f32(rounded, 0.0, max_value) / max_value) * 65535.0) as u16 +} + +#[inline(always)] +pub(crate) fn sample_as_native_u8(sample: f32, bit_depth: u32) -> u8 { + let max_value = max_int_for_bit_depth(bit_depth.min(8)) as f32; + clamp_f32(round_f32(sample), 0.0, max_value) as u8 +} + +#[inline(always)] +pub(crate) fn sample_as_native_u16(sample: f32, bit_depth: u32) -> u16 { + let max_value = max_int_for_bit_depth(bit_depth.min(16)) as f32; + clamp_f32(round_f32(sample), 0.0, max_value) as u16 +} + +#[inline(always)] +pub(crate) fn sample_as_native_i16(sample: f32, bit_depth: u32) -> i16 { + let precision = bit_depth.clamp(1, 16); + let magnitude = 1_i32 << (precision - 1); + clamp_f32( + round_f32(sample), + (-magnitude) as f32, + (magnitude - 1) as f32, + ) as i16 +} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/transform.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/transform.rs new file mode 100644 index 00000000..7a6c9bbf --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/transform.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Inverse JPEG 2000 color transform for final-store kernels. + +use crate::sample::floor_f32; + +#[inline(always)] +pub(crate) fn inverse_mct_sample( + src0: f32, + src1: f32, + src2: f32, + irreversible97: u32, +) -> (f32, f32, f32) { + if irreversible97 != 0 { + ( + src0 + 1.402 * src2, + src0 - 0.34413 * src1 - 0.71414 * src2, + src0 + 1.772 * src1, + ) + } else { + let green = src0 - floor_f32((src2 + src1) * 0.25); + (src2 + green, green, src1 + green) + } +} diff --git a/crates/j2k-cuda-runtime/src/driver.rs b/crates/j2k-cuda-runtime/src/driver.rs index b8eb0957..467263ab 100644 --- a/crates/j2k-cuda-runtime/src/driver.rs +++ b/crates/j2k-cuda-runtime/src/driver.rs @@ -59,6 +59,9 @@ pub(crate) type CuMemsetD8 = unsafe extern "C" fn(CuDevicePtr, u8, usize) -> CuR pub(crate) type CuMemsetD32 = unsafe extern "C" fn(CuDevicePtr, c_uint, usize) -> CuResult; +pub(crate) type CuMemsetD32Async = + unsafe extern "C" fn(CuDevicePtr, c_uint, usize, CuStream) -> CuResult; + pub(crate) type CuGetErrorName = unsafe extern "C" fn(CuResult, *mut *const c_char) -> CuResult; pub(crate) type CuModuleLoadData = unsafe extern "C" fn(*mut CuModule, *const c_void) -> CuResult; @@ -96,8 +99,12 @@ pub(crate) type CuEventDestroy = unsafe extern "C" fn(CuEvent) -> CuResult; pub(crate) type CuEventRecord = unsafe extern "C" fn(CuEvent, CuStream) -> CuResult; +pub(crate) type CuStreamWaitEvent = unsafe extern "C" fn(CuStream, CuEvent, c_uint) -> CuResult; + pub(crate) type CuEventSynchronize = unsafe extern "C" fn(CuEvent) -> CuResult; +pub(crate) type CuEventQuery = unsafe extern "C" fn(CuEvent) -> CuResult; + pub(crate) type CuEventElapsedTime = unsafe extern "C" fn(*mut f32, CuEvent, CuEvent) -> CuResult; #[cfg(feature = "cuda-profiling")] @@ -125,6 +132,7 @@ pub(crate) struct Driver { pub(crate) cu_memcpy_dtoh: CuMemcpyDtoH, pub(crate) cu_memset_d8: CuMemsetD8, pub(crate) cu_memset_d32: CuMemsetD32, + pub(crate) cu_memset_d32_async: CuMemsetD32Async, pub(crate) cu_get_error_name: CuGetErrorName, #[cfg_attr( not(j2k_cuda_oxide_enabled), @@ -149,7 +157,9 @@ pub(crate) struct Driver { pub(crate) cu_event_create: CuEventCreate, pub(crate) cu_event_destroy: CuEventDestroy, pub(crate) cu_event_record: CuEventRecord, + pub(crate) cu_stream_wait_event: CuStreamWaitEvent, pub(crate) cu_event_synchronize: CuEventSynchronize, + pub(crate) cu_event_query: CuEventQuery, pub(crate) cu_event_elapsed_time: CuEventElapsedTime, } @@ -200,6 +210,7 @@ impl Driver { cu_memcpy_dtoh: load_symbol(&library, b"cuMemcpyDtoH_v2\0")?, cu_memset_d8: load_symbol(&library, b"cuMemsetD8_v2\0")?, cu_memset_d32: load_symbol(&library, b"cuMemsetD32_v2\0")?, + cu_memset_d32_async: load_symbol(&library, b"cuMemsetD32Async\0")?, cu_get_error_name: load_symbol(&library, b"cuGetErrorName\0")?, cu_module_load_data: load_symbol(&library, b"cuModuleLoadData\0")?, cu_module_unload: load_symbol(&library, b"cuModuleUnload\0")?, @@ -213,7 +224,9 @@ impl Driver { cu_event_create: load_symbol(&library, b"cuEventCreate\0")?, cu_event_destroy: load_symbol(&library, b"cuEventDestroy_v2\0")?, cu_event_record: load_symbol(&library, b"cuEventRecord\0")?, + cu_stream_wait_event: load_symbol(&library, b"cuStreamWaitEvent\0")?, cu_event_synchronize: load_symbol(&library, b"cuEventSynchronize\0")?, + cu_event_query: load_symbol(&library, b"cuEventQuery\0")?, cu_event_elapsed_time: load_symbol(&library, b"cuEventElapsedTime\0")?, _library: library, }) diff --git a/crates/j2k-cuda-runtime/src/error.rs b/crates/j2k-cuda-runtime/src/error.rs index 78a8c9c5..cdc4d241 100644 --- a/crates/j2k-cuda-runtime/src/error.rs +++ b/crates/j2k-cuda-runtime/src/error.rs @@ -106,6 +106,18 @@ pub enum CudaError { /// Kernel-defined detail code. detail: u32, }, + /// A batched J2K CUDA kernel reported a failure for one descriptor. + #[error("CUDA kernel {kernel} reported status {code} detail {detail} for job {job_index}")] + KernelJobStatus { + /// Kernel entry point or logical stage name. + kernel: &'static str, + /// Zero-based descriptor index within this kernel launch. + job_index: usize, + /// Kernel-defined status code. + code: u32, + /// Kernel-defined detail code. + detail: u32, + }, /// Caller supplied arguments that cannot be represented by this runtime API. #[error("CUDA invalid argument: {message}")] InvalidArgument { @@ -132,6 +144,56 @@ impl CudaError { _ => false, } } + + /// Whether an error can mean that previously submitted CUDA work still + /// references caller-owned allocations. + /// + /// External-runtime adapters use this conservative classification to + /// quarantine allocations instead of freeing storage after an uncertain + /// completion boundary. Pure validation, capacity, and kernel-status + /// errors return `false`. + #[doc(hidden)] + pub fn completion_is_uncertain(&self) -> bool { + matches!( + self, + Self::Driver { .. } + | Self::StatePoisoned { .. } + | Self::CompletionFailed { .. } + | Self::ResourceReleaseFailed { .. } + ) + } + + /// Descriptor index reported by a batched kernel status, when available. + #[doc(hidden)] + pub fn kernel_job_index(&self) -> Option { + match self { + Self::KernelJobStatus { job_index, .. } => Some(*job_index), + Self::CompletionFailed { primary, .. } + | Self::ResourceReleaseFailed { + primary, + release: _, + } => primary.kernel_job_index(), + _ => None, + } + } + + /// Whether the retained runtime session must not execute later groups. + /// + /// Validated argument, capacity, and kernel-status errors are scoped to + /// the affected submission. Driver, poisoned-state, and uncertain cleanup + /// errors can invalidate ordering or retained resources and therefore + /// terminate a persistent batch call. + #[doc(hidden)] + pub fn session_is_unusable(&self) -> bool { + matches!( + self, + Self::Unavailable { .. } + | Self::Driver { .. } + | Self::StatePoisoned { .. } + | Self::CompletionFailed { .. } + | Self::ResourceReleaseFailed { .. } + ) + } } pub(crate) fn select_uncertain_completion_error( @@ -215,4 +277,19 @@ mod tests { assert!(rendered.contains("primary unavailable")); assert!(rendered.contains("release pool hold")); } + + #[test] + fn completion_uncertainty_excludes_preflight_and_validated_kernel_status() { + assert!(!CudaError::InvalidArgument { + message: "preflight".to_string(), + } + .completion_is_uncertain()); + assert!(!CudaError::KernelStatus { + kernel: "test", + code: 1, + detail: 2, + } + .completion_is_uncertain()); + assert!(driver("cuEventSynchronize").completion_is_uncertain()); + } } diff --git a/crates/j2k-cuda-runtime/src/execution.rs b/crates/j2k-cuda-runtime/src/execution.rs index 8e9f485d..22ae5efb 100644 --- a/crates/j2k-cuda-runtime/src/execution.rs +++ b/crates/j2k-cuda-runtime/src/execution.rs @@ -7,7 +7,6 @@ mod queued; pub(crate) use completion::{select_uncertain_completion_error, CudaSynchronizationOutcome}; pub(crate) use events::elapsed_event_us_ceil; -#[cfg(test)] pub(crate) use events::CudaEvent; pub use queued::{ CudaExecutionStats, CudaKernelBatchOutput, CudaKernelContiguousBatchOutput, CudaKernelOutput, @@ -31,16 +30,6 @@ pub(crate) enum CudaLaunchMode { Async, } -impl CudaLaunchMode { - pub(crate) fn from_synchronize(synchronize: bool) -> Self { - if synchronize { - Self::Sync - } else { - Self::Async - } - } -} - /// Marker for values that can be passed by address to a CUDA kernel launch. /// /// # Safety @@ -153,7 +142,9 @@ impl CudaContext { ) }; self.inner.driver.check("cuLaunchKernel", launch_status) - }) + })?; + self.record_kernel_launch(); + Ok(()) } pub(crate) fn copy_device_to_device_with_kernel( @@ -246,7 +237,10 @@ impl CudaContext { self.inner.driver.check("cuCtxSynchronize", status) }); match result { - Ok(()) => CudaSynchronizationOutcome::Completed, + Ok(()) => { + self.record_context_host_synchronization(); + CudaSynchronizationOutcome::Completed + } Err(error) => { // The CUDA API may return both precondition failures and fatal // asynchronous errors here. Neither is sufficient evidence diff --git a/crates/j2k-cuda-runtime/src/execution/events.rs b/crates/j2k-cuda-runtime/src/execution/events.rs index 6c9d5615..1f423002 100644 --- a/crates/j2k-cuda-runtime/src/execution/events.rs +++ b/crates/j2k-cuda-runtime/src/execution/events.rs @@ -1,11 +1,14 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use crate::{ - build_flags::cuda_stage_timings_disabled, context::CudaContext, driver::CudaNvtxRange, - error::CudaError, + build_flags::cuda_stage_timings_disabled, + context::CudaContext, + driver::CudaNvtxRange, + error::{select_resource_release_error, CudaError}, }; mod handles; +mod interop; pub(crate) use handles::CudaEvent; #[cfg(test)] @@ -48,6 +51,20 @@ impl CudaContext { /// Create a CUDA timing event owned by this context. pub(crate) fn create_event(&self) -> Result { + if let Some(event) = self + .inner + .event_pool + .lock() + .map_err(|error| CudaError::StatePoisoned { + message: error.to_string(), + })? + .take() + { + return Ok(CudaEvent { + context: self.clone(), + event, + }); + } let mut event = std::ptr::null_mut(); self.inner.with_current_stateful_operation(|| { // SAFETY: CUDA writes a new event handle while the context @@ -60,6 +77,26 @@ impl CudaContext { "CUDA returned a null event after successful creation", ) })?; + match self.inner.event_pool.lock() { + Ok(mut events) => events.record_driver_allocation(), + Err(error) => { + let primary = CudaError::StatePoisoned { + message: error.to_string(), + }; + let release = self.inner.with_current_stateful_operation(|| { + // SAFETY: event was just created by this context and has + // not escaped. A poisoned diagnostics/cache lock must not + // leak the raw driver allocation. + self.inner.driver.check("cuEventDestroy_v2", unsafe { + (self.inner.driver.cu_event_destroy)(event) + }) + }); + return match release { + Ok(()) => Err(primary), + Err(release) => Err(select_resource_release_error(primary, release)), + }; + } + } Ok(CudaEvent { context: self.clone(), event, diff --git a/crates/j2k-cuda-runtime/src/execution/events/handles.rs b/crates/j2k-cuda-runtime/src/execution/events/handles.rs index 46bd83b5..2d95509e 100644 --- a/crates/j2k-cuda-runtime/src/execution/events/handles.rs +++ b/crates/j2k-cuda-runtime/src/execution/events/handles.rs @@ -1,42 +1,12 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -#[cfg(test)] -use crate::driver::CuStream; +use crate::{build_flags::CUDA_ERROR_NOT_READY, driver::CuStream}; use crate::{context::CudaContext, driver::CuEvent, error::CudaError}; -/// CUDA stream RAII handle. -#[cfg(test)] -#[derive(Debug)] -pub(crate) struct CudaStream { - pub(crate) context: CudaContext, - pub(crate) stream: CuStream, -} - #[cfg(test)] -impl Drop for CudaStream { - fn drop(&mut self) { - if !self.stream.is_null() { - let destroy_result = self.context.inner.with_current_stateful_operation(|| { - // SAFETY: stream was created by this context and the context - // lifecycle gate is held during destruction. - self.context - .inner - .driver - .check("cuStreamDestroy_v2", unsafe { - (self.context.inner.driver.cu_stream_destroy)(self.stream) - }) - }); - if destroy_result.is_err() { - std::mem::forget(self.context.clone()); - } - } - } -} - -// SAFETY: CUDA stream handles are driver-owned resources. The Rust handle owns -// destruction and does not expose mutable aliasing of Rust memory. +mod stream; #[cfg(test)] -unsafe impl Send for CudaStream {} +pub(crate) use stream::CudaStream; /// CUDA event RAII handle for timing and synchronization. #[derive(Debug)] @@ -73,6 +43,33 @@ impl CudaEvent { }) } + pub(crate) fn record_raw_stream(&self, stream: CuStream) -> Result<(), CudaError> { + self.context.inner.with_current_resource_operation(|| { + // SAFETY: the caller's guarded stream handle has been bound to + // this retained primary context for the duration of interop. + self.context.inner.driver.check("cuEventRecord", unsafe { + (self.context.inner.driver.cu_event_record)(self.event, stream) + }) + }) + } + + pub(crate) fn wait_on_default_stream(&self) -> Result<(), CudaError> { + self.wait_on_raw_stream(std::ptr::null_mut()) + } + + pub(crate) fn wait_on_raw_stream(&self, stream: CuStream) -> Result<(), CudaError> { + self.context.inner.with_current_resource_operation(|| { + // SAFETY: the event belongs to the current retained primary + // context; CUDA validates the guarded stream handle. + self.context + .inner + .driver + .check("cuStreamWaitEvent", unsafe { + (self.context.inner.driver.cu_stream_wait_event)(stream, self.event, 0) + }) + }) + } + /// Wait for this event to complete. pub(crate) fn synchronize(&self) -> Result<(), CudaError> { self.context.inner.with_current_resource_operation(|| { @@ -84,6 +81,25 @@ impl CudaEvent { .check("cuEventSynchronize", unsafe { (self.context.inner.driver.cu_event_synchronize)(self.event) }) + })?; + self.context.record_event_host_synchronization(); + Ok(()) + } + + /// Query whether this event has completed without waiting on the host. + pub(crate) fn is_complete(&self) -> Result { + self.context.inner.with_current_resource_operation(|| { + // SAFETY: event is a live CUDA event owned by this context, and + // the context lifecycle gate is held for the query. + let status = unsafe { (self.context.inner.driver.cu_event_query)(self.event) }; + if status == CUDA_ERROR_NOT_READY { + return Ok(false); + } + self.context + .inner + .driver + .check("cuEventQuery", status) + .map(|()| true) }) } @@ -114,6 +130,21 @@ impl CudaEvent { impl Drop for CudaEvent { fn drop(&mut self) { if !self.event.is_null() { + // CUDA stream waits capture the event's most recently recorded + // generation when cuStreamWaitEvent is called. Re-recording this + // handle later does not alter an already-enqueued wait, so the + // handle can return to this context's cache without a host wait. + let recycle_result = self + .context + .inner + .event_pool + .lock() + .map_err(|_| ()) + .and_then(|mut events| events.recycle(self.event)); + if recycle_result.is_ok() { + self.event = std::ptr::null_mut(); + return; + } let destroy_result = self.context.inner.with_current_stateful_operation(|| { // SAFETY: event was created by this context and the context // lifecycle gate is held during destruction. diff --git a/crates/j2k-cuda-runtime/src/execution/events/handles/stream.rs b/crates/j2k-cuda-runtime/src/execution/events/handles/stream.rs new file mode 100644 index 00000000..1dd7ab07 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/execution/events/handles/stream.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{context::CudaContext, driver::CuStream}; + +/// CUDA stream RAII handle. +#[derive(Debug)] +pub(crate) struct CudaStream { + pub(crate) context: CudaContext, + pub(crate) stream: CuStream, +} + +impl Drop for CudaStream { + fn drop(&mut self) { + if !self.stream.is_null() { + let destroy_result = self.context.inner.with_current_stateful_operation(|| { + // SAFETY: stream was created by this context and the context + // lifecycle gate is held during destruction. + self.context + .inner + .driver + .check("cuStreamDestroy_v2", unsafe { + (self.context.inner.driver.cu_stream_destroy)(self.stream) + }) + }); + if destroy_result.is_err() { + std::mem::forget(self.context.clone()); + } + } + } +} + +// SAFETY: CUDA stream handles are driver-owned resources. The Rust handle owns +// destruction and does not expose mutable aliasing of Rust memory. +unsafe impl Send for CudaStream {} diff --git a/crates/j2k-cuda-runtime/src/execution/events/interop.rs b/crates/j2k-cuda-runtime/src/execution/events/interop.rs new file mode 100644 index 00000000..b386012d --- /dev/null +++ b/crates/j2k-cuda-runtime/src/execution/events/interop.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + context::{ContextOwnership, CudaContext}, + driver::CuStream, + error::CudaError, +}; + +impl CudaContext { + /// Order default-stream codec work between operations on a caller-owned + /// CUDA stream without a host or context-wide synchronization. + /// + /// An event recorded on `raw_stream` is made a dependency of the codec's + /// default stream before `work`. After `work` submits its codec graph, a + /// second event makes the caller stream wait for the codec final store. + /// The raw stream is confined to this guarded closure and is never stored + /// or returned. + /// + /// # Safety + /// + /// `raw_stream` must be a live CUDA stream from this context's retained + /// primary context and remain live for the complete call. `_managed_owner` + /// must exclusively guard the external runtime resource that owns the + /// stream. `work` must retain every resource reachable by asynchronously + /// submitted codec work in its returned value or another typed guard. + #[doc(hidden)] + pub unsafe fn with_primary_stream_ordering( + &self, + raw_stream: u64, + _managed_owner: &mut Owner, + work: impl FnOnce() -> Result, + ) -> Result, CudaError> { + if !matches!( + self.inner.ownership, + ContextOwnership::RetainedPrimary { .. } + ) { + return Err(CudaError::InvalidArgument { + message: "external CUDA stream interop requires a retained primary context" + .to_string(), + }); + } + if raw_stream == 0 { + return Err(CudaError::InvalidArgument { + message: "external CUDA stream handle must not be null".to_string(), + }); + } + let stream_address = usize::try_from(raw_stream) + .map_err(|_| CudaError::LengthTooLarge { len: usize::MAX })?; + let stream = stream_address as CuStream; + let producer_ready = self.create_event()?; + producer_ready.record_raw_stream(stream)?; + producer_ready.wait_on_default_stream()?; + + let output = work(); + let codec_ready = match self.create_event() { + Ok(event) => event, + Err(error) => { + // `work` may already have enqueued kernels that reference an + // external allocation. Keep `output` live while establishing + // a context completion boundary before propagating failure. + return self.synchronize_then_error(error); + } + }; + if let Err(error) = codec_ready + .record_default_stream() + .and_then(|()| codec_ready.wait_on_raw_stream(stream)) + { + return self.synchronize_then_error(error); + } + Ok(output) + } +} + +#[cfg(test)] +mod tests { + use crate::{CudaContext, CudaError}; + + #[test] + fn primary_stream_bridge_orders_both_streams_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let context = CudaContext::retain_primary(0).expect("retained primary context"); + let mut stream = context.create_stream().expect("caller stream"); + let raw_stream = stream.stream as usize as u64; + + // SAFETY: the test stream was created by this retained primary context + // and `stream` remains exclusively borrowed for the complete bridge. + let output = unsafe { + context.with_primary_stream_ordering(raw_stream, &mut stream, || { + Ok::<_, CudaError>(17_u32) + }) + } + .expect("event ordering") + .expect("guarded work"); + + assert_eq!(output, 17); + } + + #[test] + fn primary_stream_bridge_reuses_captured_event_generations_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let context = CudaContext::retain_primary(0).expect("retained primary context"); + let mut stream = context.create_stream().expect("caller stream"); + let raw_stream = stream.stream as usize as u64; + + // Warm the two-handle bridge cache, then establish a clean diagnostic + // baseline. Each later cuStreamWaitEvent captures the current record, + // so re-recording a recycled handle cannot retarget an earlier wait. + // SAFETY: the test stream belongs to this retained primary context and + // remains exclusively borrowed throughout each bridge call. + unsafe { + context.with_primary_stream_ordering(raw_stream, &mut stream, || Ok::<_, CudaError>(())) + } + .expect("warm bridge ordering") + .expect("warm guarded work"); + context.synchronize().expect("warm bridge completion"); + let before = context.diagnostics().expect("diagnostics before reuse"); + + for value in 0_u32..32 { + // SAFETY: identical ownership and lifetime proof to the warm call. + let output = unsafe { + context.with_primary_stream_ordering(raw_stream, &mut stream, || { + Ok::<_, CudaError>(value) + }) + } + .expect("reused bridge ordering") + .expect("reused guarded work"); + assert_eq!(output, value); + } + context.synchronize().expect("reused bridge completion"); + let after = context.diagnostics().expect("diagnostics after reuse"); + + assert_eq!( + after.event_driver_allocations, + before.event_driver_allocations + ); + assert_eq!(after.event_reuses - before.event_reuses, 64); + } + + #[test] + fn primary_stream_bridge_has_no_context_wide_sync_on_success_path() { + let bridge = production_bridge_source(); + assert!(bridge.contains("wait_on_default_stream")); + assert!(bridge.contains("wait_on_raw_stream")); + assert!(!bridge.contains("cuCtxSynchronize")); + } + + #[test] + fn primary_stream_bridge_recovers_post_submit_event_creation_failure() { + let bridge = production_bridge_source(); + let work = bridge.find("let output = work();").expect("submitted work"); + let recovery = bridge + .find("return self.synchronize_then_error(error);") + .expect("post-submit recovery"); + assert!(recovery > work); + } + + #[test] + fn completion_event_can_be_polled_without_a_host_wait_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let context = CudaContext::system_default().expect("CUDA context"); + let completion = context.create_event().expect("completion event"); + completion + .record_default_stream() + .expect("record completion event"); + + let _may_already_be_complete = completion.is_complete().expect("query completion event"); + completion.synchronize().expect("wait completion event"); + assert!(completion.is_complete().expect("query completed event")); + } + + fn production_bridge_source() -> &'static str { + include_str!("interop.rs") + .split("#[cfg(test)]") + .next() + .expect("production primary stream bridge") + } +} diff --git a/crates/j2k-cuda-runtime/src/execution/memory_ops.rs b/crates/j2k-cuda-runtime/src/execution/memory_ops.rs index 6ab53a0b..afdbe75e 100644 --- a/crates/j2k-cuda-runtime/src/execution/memory_ops.rs +++ b/crates/j2k-cuda-runtime/src/execution/memory_ops.rs @@ -62,4 +62,30 @@ impl CudaContext { }) }) } + + pub(crate) fn memset_d32_async( + &self, + dst: &CudaDeviceBuffer, + value: c_uint, + words: usize, + ) -> Result<(), CudaError> { + let required = words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: words })?; + if !self.validate_memset_target(dst, required)? { + return Ok(()); + } + self.inner.with_current_resource_operation(|| { + // SAFETY: the destination is live and bounds-checked. A null + // stream orders the memset with the codec's default-stream graph. + self.inner.driver.check("cuMemsetD32Async", unsafe { + (self.inner.driver.cu_memset_d32_async)( + dst.device_ptr(), + value, + words, + std::ptr::null_mut(), + ) + }) + }) + } } diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode.rs b/crates/j2k-cuda-runtime/src/htj2k_decode.rs index a277c240..b44ac514 100644 --- a/crates/j2k-cuda-runtime/src/htj2k_decode.rs +++ b/crates/j2k-cuda-runtime/src/htj2k_decode.rs @@ -8,6 +8,7 @@ pub(crate) mod output_regions; mod planning; mod queued; mod status; +mod status_group; mod types; pub(crate) use self::planning::htj2k_decode_needs_zero_fill; @@ -16,12 +17,14 @@ pub(crate) use self::planning::{ htj2k_decode_multi_cleanup_dequant_kernel_for_jobs, htj2k_decode_multi_kernel_for_jobs, }; pub use self::queued::CudaQueuedHtj2kCleanup; +pub use self::status_group::CudaQueuedHtj2kCleanupGroup; +pub use self::types::{ + htj2k_cleanup_multi_descriptor_bytes, CudaHtj2kCleanupTarget, CudaHtj2kCodeBlockJob, + CudaHtj2kDecodeOutput, CudaHtj2kDecodeResources, CudaHtj2kDecodeStageTimings, + CudaHtj2kDecodeTableResources, CudaHtj2kDecodeTables, CudaHtj2kDequantizeTarget, + CudaHtj2kStatus, CudaPooledHtj2kDecodeOutput, +}; pub(crate) use self::types::{ CudaHtj2kCleanupMultiKernelJob, CudaHtj2kCodeBlockKernelJob, CudaHtj2kDecodePayload, CudaHtj2kDequantizeKernelJob, HTJ2K_STATUS_OK, HTJ2K_STATUS_UNSUPPORTED, }; -pub use self::types::{ - CudaHtj2kCleanupTarget, CudaHtj2kCodeBlockJob, CudaHtj2kDecodeOutput, CudaHtj2kDecodeResources, - CudaHtj2kDecodeStageTimings, CudaHtj2kDecodeTableResources, CudaHtj2kDecodeTables, - CudaHtj2kDequantizeTarget, CudaHtj2kStatus, CudaPooledHtj2kDecodeOutput, -}; diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/api.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/api.rs index 405018f1..56766dc9 100644 --- a/crates/j2k-cuda-runtime/src/htj2k_decode/api.rs +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/api.rs @@ -169,8 +169,7 @@ impl CudaContext { let coefficients = pool.take(output_layout.output_bytes)?; let coefficient_buffer = pooled_device_buffer(&coefficients)?; if output_layout.needs_zero_fill { - self.memset_d32(coefficient_buffer, 0, output_words)?; - self.synchronize()?; + self.memset_d32_async(coefficient_buffer, 0, output_words)?; } Ok(CudaPooledHtj2kDecodeOutput { coefficients, diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/completion.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/completion.rs index 32e01473..d5f855ed 100644 --- a/crates/j2k-cuda-runtime/src/htj2k_decode/completion.rs +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/completion.rs @@ -2,7 +2,11 @@ use std::time::Instant; +mod cleanup_dequant_enqueue; +mod cleanup_enqueue; mod dequant; +#[cfg(test)] +mod tests; use crate::{ allocation::HostPhaseBudget, @@ -24,7 +28,6 @@ use super::{ htj2k_decode_multi_cleanup_dequant_kernel_for_jobs, htj2k_decode_multi_kernel_for_jobs, htj2k_kernel_jobs, }, - queued::CudaQueuedHtj2kCleanup, status::{first_status_error, select_status_release_result}, types::{ htj2k_decode_kernel_tables, CudaHtj2kCleanupMultiKernelJob, CudaHtj2kCleanupTarget, @@ -35,125 +38,6 @@ use super::{ }; impl CudaContext { - /// Enqueue HTJ2K cleanup passes for multiple output buffers with one CUDA - /// dispatch. The returned value must be kept live until `finish` validates - /// the kernel statuses after the default stream has completed. - /// - /// # Safety - /// - /// Every target coefficient buffer must remain allocated and must not be - /// mutated or reused until the returned cleanup is finished or dropped. - /// Target allocations and each target's job write regions must be - /// pairwise disjoint; both conditions are validated before launch. - /// The decode payload and table resources must remain live for the same - /// duration. The resources, targets, and pool must belong to this context - /// (validated at runtime), and all pool clones must remain confined to this - /// context's default stream until that completion point. - #[doc(hidden)] - pub unsafe fn decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool( - &self, - resources: &CudaHtj2kDecodeResources, - targets: &[CudaHtj2kCleanupTarget<'_>], - pool: &CudaBufferPool, - ) -> Result { - // SAFETY: this wrapper preserves the caller's target and pool lifetime - // requirements and contributes no additional caller-live host owners. - unsafe { - self.decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool_and_live_host_bytes( - resources, - targets, - pool, - 0, - ) - } - } - - /// Enqueue HTJ2K cleanup while accounting caller-live host metadata. - /// - /// # Safety - /// - /// The target buffers, resources, and pool must satisfy the same lifetime, - /// aliasing, context, and stream-confinement requirements as - /// [`Self::decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool`]. - #[doc(hidden)] - pub unsafe fn decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool_and_live_host_bytes( - &self, - resources: &CudaHtj2kDecodeResources, - targets: &[CudaHtj2kCleanupTarget<'_>], - pool: &CudaBufferPool, - live_host_bytes: usize, - ) -> Result { - validate_cleanup_context(self, resources, targets, pool)?; - let kernel_jobs = htj2k_cleanup_multi_kernel_jobs_with_live_host_bytes( - targets, - resources.payload_len, - live_host_bytes, - )?; - if kernel_jobs.is_empty() { - return Ok(CudaQueuedHtj2kCleanup { - context: self.clone(), - resources: Vec::new(), - status_buffer: None, - status_count: 0, - kernel_name: "j2k_htj2k_decode_codeblocks_multi", - execution: CudaExecutionStats::default(), - pool_reuse_guard: None, - finish_host_live_bytes: 0, - }); - } - self.inner.set_current()?; - let (decode_kernel, decode_kernel_name) = htj2k_decode_multi_kernel_for_jobs(&kernel_jobs); - let tables = htj2k_decode_kernel_tables(resources)?; - - let mut host_budget = HostPhaseBudget::with_live_bytes( - "CUDA queued HTJ2K cleanup metadata", - live_host_bytes, - )?; - host_budget.account_vec(&kernel_jobs)?; - let mut queued_resources = host_budget.try_vec_with_capacity(1)?; - let jobs_buffer = pool.upload(htj2k_cleanup_multi_jobs_as_bytes(&kernel_jobs))?; - queued_resources.push(jobs_buffer); - let mut finish_budget = HostPhaseBudget::with_live_bytes( - "CUDA queued HTJ2K cleanup retained metadata", - live_host_bytes, - )?; - finish_budget.account_vec(&queued_resources)?; - let status_buffer = pool.take(htj2k_statuses_byte_len(kernel_jobs.len())?)?; - let payload_buffer = resources.payload.buffer()?; - let jobs_device_buffer = pooled_device_buffer(&queued_resources[0])?; - let status_device_buffer = pooled_device_buffer(&status_buffer)?; - let pool_reuse_guard = pool.defer_reuse()?; - let launch_result = - self.launch_htj2k_decode_codeblocks_multi(Htj2kDecodeCodeblocksMultiLaunch { - kernel: decode_kernel, - payload: payload_buffer, - jobs: jobs_device_buffer, - tables, - statuses: status_device_buffer, - job_count: kernel_jobs.len(), - mode: CudaLaunchMode::Async, - }); - if let Err(error) = launch_result { - return pool_reuse_guard.synchronize_then_error(error); - } - - Ok(CudaQueuedHtj2kCleanup { - context: self.clone(), - resources: queued_resources, - status_buffer: Some(status_buffer), - status_count: kernel_jobs.len(), - kernel_name: decode_kernel_name, - execution: CudaExecutionStats { - kernel_dispatches: 1, - copy_kernel_dispatches: 0, - decode_kernel_dispatches: 1, - hardware_decode: false, - }, - pool_reuse_guard: Some(pool_reuse_guard), - finish_host_live_bytes: finish_budget.live_bytes(), - }) - } - fn run_htj2k_cleanup_multi_kernel( &self, resources: &CudaHtj2kDecodeResources, @@ -185,6 +69,7 @@ impl CudaContext { jobs: jobs_device_buffer, tables, statuses: status_device_buffer, + status_byte_offset: 0, job_count: kernel_jobs.len(), mode, }); @@ -504,21 +389,3 @@ impl CudaContext { Ok(0) } } - -#[cfg(test)] -mod tests { - #[test] - fn ht_status_timing_excludes_pool_release() { - let source = include_str!("completion.rs"); - let status_copy = source - .find("status_buffer.copy_to_host") - .expect("HT status copy"); - let status_timing = source - .find("let status_d2h_us") - .expect("HT status timing result"); - let pool_release = source - .find("let release_result = pending_pool_reuse") - .expect("HT pool release"); - assert!(status_copy < status_timing && status_timing < pool_release); - } -} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_dequant_enqueue.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_dequant_enqueue.rs new file mode 100644 index 00000000..ee1bc4db --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_dequant_enqueue.rs @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + allocation::HostPhaseBudget, + bytes::{htj2k_cleanup_multi_jobs_as_bytes, htj2k_statuses_byte_len}, + context::CudaContext, + error::CudaError, + execution::{CudaExecutionStats, CudaLaunchMode}, + memory::{pooled_device_buffer, CudaBufferPool}, +}; + +use super::super::{ + context_validation::validate_cleanup_context, + planning::{ + htj2k_cleanup_multi_kernel_jobs_with_live_host_bytes, + htj2k_decode_multi_cleanup_dequant_kernel_for_jobs, + }, + queued::CudaQueuedHtj2kCleanup, + status_group::CudaQueuedHtj2kCleanupGroup, + types::{ + htj2k_decode_kernel_tables, CudaHtj2kCleanupTarget, CudaHtj2kDecodeResources, + Htj2kDecodeCodeblocksMultiLaunch, + }, +}; + +impl CudaContext { + /// Enqueue cleanup-only HTJ2K decode and dequantization in one dispatch. + /// + /// The returned guard retains status and metadata resources until + /// [`CudaQueuedHtj2kCleanup::finish`] validates the completed launch. + /// + /// # Safety + /// + /// Every target, decode resource, and pool allocation must remain live and + /// unmodified until the returned guard is finished or dropped. Targets + /// must be pairwise disjoint and belong to this context. + #[doc(hidden)] + pub unsafe fn decode_htj2k_codeblocks_cleanup_dequantize_multi_enqueue_with_resources_and_pool( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + live_host_bytes: usize, + ) -> Result { + // SAFETY: this forwards the caller's target/resource lifetime contract + // and requests an owned per-launch status allocation. + unsafe { + self.enqueue_htj2k_cleanup_dequantize_multi_impl( + resources, + targets, + pool, + live_host_bytes, + None, + ) + } + } + + /// Enqueue fused cleanup/dequantization into a group-owned status arena. + /// + /// # Safety + /// + /// In addition to the normal queued-cleanup requirements, `status_group` + /// must outlive the returned cleanup and finish it before exposing output. + #[doc(hidden)] + pub unsafe fn decode_htj2k_codeblocks_cleanup_dequantize_multi_enqueue_into_status_group( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + live_host_bytes: usize, + status_group: &CudaQueuedHtj2kCleanupGroup, + status_offset: usize, + ) -> Result { + // SAFETY: this forwards the caller's target/resource/status-group + // lifetime and aliasing requirements unchanged. + unsafe { + self.enqueue_htj2k_cleanup_dequantize_multi_impl( + resources, + targets, + pool, + live_host_bytes, + Some((status_group, status_offset)), + ) + } + } + + unsafe fn enqueue_htj2k_cleanup_dequantize_multi_impl( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + live_host_bytes: usize, + status_group: Option<(&CudaQueuedHtj2kCleanupGroup, usize)>, + ) -> Result { + validate_cleanup_context(self, resources, targets, pool)?; + let kernel_jobs = htj2k_cleanup_multi_kernel_jobs_with_live_host_bytes( + targets, + resources.payload_len, + live_host_bytes, + )?; + if kernel_jobs.is_empty() { + return Ok(CudaQueuedHtj2kCleanup { + context: self.clone(), + resources: Vec::new(), + status_buffer: None, + status_count: 0, + status_offset: status_group.map_or(0, |(_, offset)| offset), + uses_external_status_group: status_group.is_some(), + kernel_name: "j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize", + execution: CudaExecutionStats::default(), + pool_reuse_guard: None, + finish_host_live_bytes: 0, + }); + } + let Some((decode_kernel, decode_kernel_name)) = + htj2k_decode_multi_cleanup_dequant_kernel_for_jobs(&kernel_jobs) + else { + return Err(CudaError::InvalidArgument { + message: "queued fused HTJ2K cleanup/dequantize requires cleanup-only jobs" + .to_string(), + }); + }; + self.inner.set_current()?; + let tables = htj2k_decode_kernel_tables(resources)?; + let mut host_budget = HostPhaseBudget::with_live_bytes( + "CUDA queued fused HTJ2K cleanup/dequantize metadata", + live_host_bytes, + )?; + host_budget.account_vec(&kernel_jobs)?; + let mut queued_resources = host_budget.try_vec_with_capacity(1)?; + let jobs_buffer = pool.upload(htj2k_cleanup_multi_jobs_as_bytes(&kernel_jobs))?; + queued_resources.push(jobs_buffer); + let mut finish_budget = HostPhaseBudget::with_live_bytes( + "CUDA queued fused HTJ2K cleanup/dequantize retained metadata", + live_host_bytes, + )?; + finish_budget.account_vec(&queued_resources)?; + let status_buffer = status_group + .is_none() + .then(|| pool.take(htj2k_statuses_byte_len(kernel_jobs.len())?)) + .transpose()?; + let payload_buffer = resources.payload.buffer()?; + let jobs_device_buffer = pooled_device_buffer(&queued_resources[0])?; + let (status_device_buffer, status_byte_offset, status_offset) = match status_group { + Some((group, offset)) => { + let (buffer, byte_offset) = group.status_destination(offset, kernel_jobs.len())?; + (buffer, byte_offset, offset) + } + None => ( + pooled_device_buffer(status_buffer.as_ref().ok_or( + CudaError::InternalInvariant { + what: "owned fused HTJ2K status allocation disappeared before launch", + }, + )?)?, + 0, + 0, + ), + }; + let pool_reuse_guard = pool.defer_reuse()?; + let launch_result = + self.launch_htj2k_decode_codeblocks_multi(Htj2kDecodeCodeblocksMultiLaunch { + kernel: decode_kernel, + payload: payload_buffer, + jobs: jobs_device_buffer, + tables, + statuses: status_device_buffer, + status_byte_offset, + job_count: kernel_jobs.len(), + mode: CudaLaunchMode::Async, + }); + if let Err(error) = launch_result { + return pool_reuse_guard.synchronize_then_error(error); + } + + Ok(CudaQueuedHtj2kCleanup { + context: self.clone(), + resources: queued_resources, + status_buffer, + status_count: kernel_jobs.len(), + status_offset, + uses_external_status_group: status_group.is_some(), + kernel_name: decode_kernel_name, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + pool_reuse_guard: Some(pool_reuse_guard), + finish_host_live_bytes: finish_budget.live_bytes(), + }) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn queued_cleanup_dequantize_launches_asynchronously() { + let source = include_str!("cleanup_dequant_enqueue.rs") + .split("#[cfg(test)]") + .next() + .expect("production completion source"); + let function = source + .split("unsafe fn enqueue_htj2k_cleanup_dequantize_multi_impl") + .nth(1) + .expect("queued fused cleanup/dequantize implementation"); + let async_mode = function + .find("mode: CudaLaunchMode::Async") + .expect("queued fused cleanup/dequantize async launch"); + let function_end = function + .find("\n }") + .expect("queued fused cleanup/dequantize function end"); + assert!(async_mode < function_end); + } +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_enqueue.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_enqueue.rs new file mode 100644 index 00000000..c820bb2b --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_enqueue.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + allocation::HostPhaseBudget, + bytes::{htj2k_cleanup_multi_jobs_as_bytes, htj2k_statuses_byte_len}, + context::CudaContext, + error::CudaError, + execution::{CudaExecutionStats, CudaLaunchMode}, + memory::{pooled_device_buffer, CudaBufferPool}, +}; + +use super::super::{ + context_validation::validate_cleanup_context, + planning::{ + htj2k_cleanup_multi_kernel_jobs_with_live_host_bytes, htj2k_decode_multi_kernel_for_jobs, + }, + queued::CudaQueuedHtj2kCleanup, + status_group::CudaQueuedHtj2kCleanupGroup, + types::{ + htj2k_decode_kernel_tables, CudaHtj2kCleanupTarget, CudaHtj2kDecodeResources, + Htj2kDecodeCodeblocksMultiLaunch, + }, +}; + +impl CudaContext { + /// Enqueue HTJ2K cleanup passes for multiple output buffers with one CUDA + /// dispatch. The returned value must be kept live until `finish` validates + /// the kernel statuses after the default stream has completed. + /// + /// # Safety + /// + /// Every target coefficient buffer must remain allocated and must not be + /// mutated or reused until the returned cleanup is finished or dropped. + /// Target allocations and each target's job write regions must be + /// pairwise disjoint; both conditions are validated before launch. + /// The decode payload and table resources must remain live for the same + /// duration. The resources, targets, and pool must belong to this context + /// (validated at runtime), and all pool clones must remain confined to this + /// context's default stream until that completion point. + #[doc(hidden)] + pub unsafe fn decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + ) -> Result { + // SAFETY: this wrapper preserves the caller's target and pool lifetime + // requirements and contributes no additional caller-live host owners. + unsafe { + self.decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool_and_live_host_bytes( + resources, + targets, + pool, + 0, + ) + } + } + + /// Enqueue HTJ2K cleanup while accounting caller-live host metadata. + /// + /// # Safety + /// + /// The target buffers, resources, and pool must satisfy the same lifetime, + /// aliasing, context, and stream-confinement requirements as + /// [`Self::decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool`]. + #[doc(hidden)] + pub unsafe fn decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool_and_live_host_bytes( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + live_host_bytes: usize, + ) -> Result { + // SAFETY: this forwards the caller's complete target/resource lifetime + // contract and requests an owned per-launch status allocation. + unsafe { + self.enqueue_htj2k_cleanup_multi_impl(resources, targets, pool, live_host_bytes, None) + } + } + + /// Enqueue one HTJ2K cleanup launch into a group-owned status arena. + /// + /// # Safety + /// + /// In addition to the normal queued-cleanup requirements, `status_group` + /// must outlive the returned cleanup and finish it before exposing output. + #[doc(hidden)] + pub unsafe fn decode_htj2k_codeblocks_cleanup_multi_enqueue_into_status_group( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + live_host_bytes: usize, + status_group: &CudaQueuedHtj2kCleanupGroup, + status_offset: usize, + ) -> Result { + // SAFETY: this forwards the caller's target/resource/status-group + // lifetime and aliasing requirements unchanged. + unsafe { + self.enqueue_htj2k_cleanup_multi_impl( + resources, + targets, + pool, + live_host_bytes, + Some((status_group, status_offset)), + ) + } + } + + unsafe fn enqueue_htj2k_cleanup_multi_impl( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + live_host_bytes: usize, + status_group: Option<(&CudaQueuedHtj2kCleanupGroup, usize)>, + ) -> Result { + validate_cleanup_context(self, resources, targets, pool)?; + let kernel_jobs = htj2k_cleanup_multi_kernel_jobs_with_live_host_bytes( + targets, + resources.payload_len, + live_host_bytes, + )?; + if kernel_jobs.is_empty() { + return Ok(CudaQueuedHtj2kCleanup { + context: self.clone(), + resources: Vec::new(), + status_buffer: None, + status_count: 0, + status_offset: status_group.map_or(0, |(_, offset)| offset), + uses_external_status_group: status_group.is_some(), + kernel_name: "j2k_htj2k_decode_codeblocks_multi", + execution: CudaExecutionStats::default(), + pool_reuse_guard: None, + finish_host_live_bytes: 0, + }); + } + self.inner.set_current()?; + let (decode_kernel, decode_kernel_name) = htj2k_decode_multi_kernel_for_jobs(&kernel_jobs); + let tables = htj2k_decode_kernel_tables(resources)?; + + let mut host_budget = HostPhaseBudget::with_live_bytes( + "CUDA queued HTJ2K cleanup metadata", + live_host_bytes, + )?; + host_budget.account_vec(&kernel_jobs)?; + let mut queued_resources = host_budget.try_vec_with_capacity(1)?; + let jobs_buffer = pool.upload(htj2k_cleanup_multi_jobs_as_bytes(&kernel_jobs))?; + queued_resources.push(jobs_buffer); + let mut finish_budget = HostPhaseBudget::with_live_bytes( + "CUDA queued HTJ2K cleanup retained metadata", + live_host_bytes, + )?; + finish_budget.account_vec(&queued_resources)?; + let status_buffer = status_group + .is_none() + .then(|| pool.take(htj2k_statuses_byte_len(kernel_jobs.len())?)) + .transpose()?; + let payload_buffer = resources.payload.buffer()?; + let jobs_device_buffer = pooled_device_buffer(&queued_resources[0])?; + let (status_device_buffer, status_byte_offset, status_offset) = match status_group { + Some((group, offset)) => { + let (buffer, byte_offset) = group.status_destination(offset, kernel_jobs.len())?; + (buffer, byte_offset, offset) + } + None => ( + pooled_device_buffer(status_buffer.as_ref().ok_or( + CudaError::InternalInvariant { + what: "owned HTJ2K status allocation disappeared before launch", + }, + )?)?, + 0, + 0, + ), + }; + let pool_reuse_guard = pool.defer_reuse()?; + let launch_result = + self.launch_htj2k_decode_codeblocks_multi(Htj2kDecodeCodeblocksMultiLaunch { + kernel: decode_kernel, + payload: payload_buffer, + jobs: jobs_device_buffer, + tables, + statuses: status_device_buffer, + status_byte_offset, + job_count: kernel_jobs.len(), + mode: CudaLaunchMode::Async, + }); + if let Err(error) = launch_result { + return pool_reuse_guard.synchronize_then_error(error); + } + + Ok(CudaQueuedHtj2kCleanup { + context: self.clone(), + resources: queued_resources, + status_buffer, + status_count: kernel_jobs.len(), + status_offset, + uses_external_status_group: status_group.is_some(), + kernel_name: decode_kernel_name, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + pool_reuse_guard: Some(pool_reuse_guard), + finish_host_live_bytes: finish_budget.live_bytes(), + }) + } +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant.rs index 9ddc00dd..a2e501c1 100644 --- a/crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant.rs +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant.rs @@ -13,6 +13,8 @@ use super::super::{ planning::htj2k_dequantize_kernel_jobs_with_live_host_bytes, types::CudaHtj2kDequantizeTarget, }; +mod queued; + impl CudaContext { pub(super) fn submit_htj2k_dequantize_htj2k_codeblocks( &self, @@ -49,7 +51,12 @@ impl CudaContext { } /// Dequantize HTJ2K code-block outputs that live in multiple device buffers - /// with one CUDA dispatch, reusing caller-owned transient storage. + /// with one completed CUDA dispatch, reusing caller-owned transient storage. + /// + /// The completion boundary keeps the locally uploaded job descriptor alive. + /// Asynchronous decode pipelines must use + /// [`Self::j2k_dequantize_queued_htj2k_cleanup_enqueue`], whose typed cleanup + /// guard retains the descriptor through the later group completion. #[doc(hidden)] pub fn j2k_dequantize_htj2k_codeblocks_multi_device_with_pool( &self, @@ -72,7 +79,6 @@ impl CudaContext { self.j2k_dequantize_htj2k_codeblocks_multi_device_with_pool_impl( targets, pool, - true, live_host_bytes, ) } @@ -81,7 +87,6 @@ impl CudaContext { &self, targets: &[CudaHtj2kDequantizeTarget<'_>], pool: &CudaBufferPool, - synchronize_each_launch: bool, live_host_bytes: usize, ) -> Result { validate_dequantize_context(self, targets, pool)?; @@ -95,7 +100,7 @@ impl CudaContext { self.launch_j2k_dequantize_htj2k_codeblocks_multi( pooled_device_buffer(&jobs_buffer)?, kernel_jobs.len(), - CudaLaunchMode::from_synchronize(synchronize_each_launch), + CudaLaunchMode::Sync, )?; Ok(CudaExecutionStats { kernel_dispatches: 1, diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant/queued.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant/queued.rs new file mode 100644 index 00000000..855c2a78 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant/queued.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + context::CudaContext, + error::CudaError, + execution::{CudaExecutionStats, CudaLaunchMode}, + memory::pooled_device_buffer, +}; + +use super::super::super::queued::CudaQueuedHtj2kCleanup; + +impl CudaContext { + /// Dequantize HTJ2K cleanup outputs using the metadata buffer already held + /// live by a queued cleanup launch. + #[doc(hidden)] + pub fn j2k_dequantize_queued_htj2k_cleanup_with_pool( + &self, + cleanup: &CudaQueuedHtj2kCleanup, + ) -> Result { + self.j2k_dequantize_queued_htj2k_cleanup(cleanup, CudaLaunchMode::Sync) + } + + /// Enqueue dequantization using metadata retained by queued HTJ2K cleanup. + /// + /// # Safety + /// + /// `cleanup`, all coefficient targets referenced by its metadata, and the + /// owning pool must remain live and unavailable for reuse until a later + /// same-context completion point. + #[doc(hidden)] + pub unsafe fn j2k_dequantize_queued_htj2k_cleanup_enqueue( + &self, + cleanup: &CudaQueuedHtj2kCleanup, + ) -> Result { + self.j2k_dequantize_queued_htj2k_cleanup(cleanup, CudaLaunchMode::Async) + } + + fn j2k_dequantize_queued_htj2k_cleanup( + &self, + cleanup: &CudaQueuedHtj2kCleanup, + mode: CudaLaunchMode, + ) -> Result { + if !self.is_same_context(&cleanup.context) { + return Err(CudaError::InvalidArgument { + message: "queued HTJ2K cleanup belongs to a different CUDA context".to_string(), + }); + } + self.inner.set_current()?; + if cleanup.status_count == 0 { + return Ok(CudaExecutionStats::default()); + } + let Some(jobs_buffer) = cleanup.resources.first() else { + return Err(CudaError::InvalidArgument { + message: "queued HTJ2K cleanup has no metadata buffer".to_string(), + }); + }; + self.launch_j2k_dequantize_htj2k_cleanup_jobs_multi( + pooled_device_buffer(jobs_buffer)?, + cleanup.status_count, + mode, + )?; + Ok(CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }) + } +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/completion/tests.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/tests.rs new file mode 100644 index 00000000..dead77fa --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/completion/tests.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[test] +fn ht_status_timing_excludes_pool_release() { + let source = include_str!("../completion.rs"); + let status_copy = source + .find("status_buffer.copy_to_host") + .expect("HT status copy"); + let status_timing = source + .find("let status_d2h_us") + .expect("HT status timing result"); + let pool_release = source + .find("let release_result = pending_pool_reuse") + .expect("HT pool release"); + assert!(status_copy < status_timing && status_timing < pool_release); +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/launch.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/launch.rs index caf35aa9..e06e0561 100644 --- a/crates/j2k-cuda-runtime/src/htj2k_decode/launch.rs +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/launch.rs @@ -82,6 +82,7 @@ impl CudaContext { jobs, tables, statuses, + status_byte_offset, job_count, mode, } = launch; @@ -91,7 +92,16 @@ impl CudaContext { let mut vlc_table1_ptr = tables.vlc_table1.device_ptr(); let mut uvlc_table0_ptr = tables.uvlc_table0.device_ptr(); let mut uvlc_table1_ptr = tables.uvlc_table1.device_ptr(); - let mut statuses_ptr = statuses.device_ptr(); + let mut statuses_ptr = statuses + .device_ptr() + .checked_add(u64::try_from(status_byte_offset).map_err(|_| { + CudaError::LengthTooLarge { + len: status_byte_offset, + } + })?) + .ok_or(CudaError::LengthTooLarge { + len: status_byte_offset, + })?; let mut job_count = c_uint::try_from(job_count) .map_err(|_| CudaError::LengthTooLarge { len: job_count })?; let mut params = cuda_kernel_params!( diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/queued.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/queued.rs index 92a81141..6effd24f 100644 --- a/crates/j2k-cuda-runtime/src/htj2k_decode/queued.rs +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/queued.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 mod drop_guard; +mod lifecycle; use crate::{ allocation::HostPhaseBudget, bytes::htj2k_statuses_as_bytes_mut, context::CudaContext, - error::{select_resource_release_error, select_uncertain_completion_error, CudaError}, + error::CudaError, execution::CudaExecutionStats, memory::{CudaBufferPoolReuseGuard, CudaPooledDeviceBuffer}, }; @@ -26,6 +27,8 @@ pub struct CudaQueuedHtj2kCleanup { pub(crate) resources: Vec, pub(crate) status_buffer: Option, pub(crate) status_count: usize, + pub(crate) status_offset: usize, + pub(crate) uses_external_status_group: bool, pub(crate) kernel_name: &'static str, pub(crate) execution: CudaExecutionStats, pub(crate) pool_reuse_guard: Option, @@ -33,56 +36,10 @@ pub struct CudaQueuedHtj2kCleanup { } impl CudaQueuedHtj2kCleanup { - fn release_after_stream_completion(&mut self) -> Result<(), CudaError> { - self.status_buffer.take(); - self.resources.clear(); - if let Some(guard) = self.pool_reuse_guard.take() { - guard.release()?; - } - Ok(()) - } - - fn synchronize_and_release(&mut self) -> Result<(), CudaError> { - if self.pool_reuse_guard.is_none() { - return self.release_after_stream_completion(); - } - let outcome = self.context.synchronize_for_resource_release(); - if !outcome.completion_established() { - return outcome.into_result(); - } - - self.release_after_stream_completion() - } - - fn abandon_resources(&mut self) { - self.status_buffer.take(); - self.resources.clear(); - if let Some(guard) = self.pool_reuse_guard.take() { - guard.abandon(); - } - } - - fn release_after_recoverable_operation_error(&mut self, primary_error: CudaError) -> CudaError { - if self.context.inner.resource_lifetimes_poisoned() { - self.abandon_resources(); - return primary_error; - } - match self.release_after_stream_completion() { - Ok(()) => primary_error, - Err(release_error) => select_resource_release_error(primary_error, release_error), - } - } - - fn synchronize_release_after_error(&mut self, primary_error: CudaError) -> CudaError { - let outcome = self.context.synchronize_for_resource_release(); - if let Err(completion_error) = outcome.into_result() { - self.abandon_resources(); - return select_uncertain_completion_error(primary_error, Some(completion_error)); - } - match self.release_after_stream_completion() { - Ok(()) => primary_error, - Err(release_error) => select_resource_release_error(primary_error, release_error), - } + /// Number of statuses downloaded when this guard is finished directly. + #[doc(hidden)] + pub const fn status_count(&self) -> usize { + self.status_count } /// CUDA execution counters for the enqueued cleanup work. @@ -97,6 +54,13 @@ impl CudaQueuedHtj2kCleanup { /// Synchronize through status download and validate kernel statuses. pub fn finish(mut self) -> Result { + if self.uses_external_status_group && self.status_count != 0 { + let error = CudaError::InvalidArgument { + message: "group-status HTJ2K cleanup must be finished by its status group" + .to_string(), + }; + return Err(self.synchronize_release_after_error(error)); + } if self.status_buffer.is_none() { if self.pool_reuse_guard.is_some() { self.synchronize_and_release()?; @@ -130,6 +94,10 @@ impl CudaQueuedHtj2kCleanup { if let Err(error) = copy_result { return Err(self.release_after_recoverable_operation_error(error)); } + self.context.record_status_device_to_host_copy( + self.status_count + .saturating_mul(core::mem::size_of::()), + ); let status_error = first_status_error(&statuses, self.kernel_name); let release_result = self.release_after_stream_completion(); select_status_release_result(self.execution, status_error, release_result) diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/queued/lifecycle.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/queued/lifecycle.rs new file mode 100644 index 00000000..8fd5fefb --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/queued/lifecycle.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::error::{select_resource_release_error, select_uncertain_completion_error, CudaError}; + +use super::CudaQueuedHtj2kCleanup; + +impl CudaQueuedHtj2kCleanup { + pub(in crate::htj2k_decode) fn release_after_stream_completion( + &mut self, + ) -> Result<(), CudaError> { + self.status_buffer.take(); + self.resources.clear(); + if let Some(guard) = self.pool_reuse_guard.take() { + guard.release()?; + } + Ok(()) + } + + pub(in crate::htj2k_decode) fn synchronize_and_release(&mut self) -> Result<(), CudaError> { + if self.pool_reuse_guard.is_none() { + return self.release_after_stream_completion(); + } + let outcome = self.context.synchronize_for_resource_release(); + if !outcome.completion_established() { + return outcome.into_result(); + } + + self.release_after_stream_completion() + } + + pub(super) fn abandon_resources(&mut self) { + self.status_buffer.take(); + self.resources.clear(); + if let Some(guard) = self.pool_reuse_guard.take() { + guard.abandon(); + } + } + + pub(super) fn release_after_recoverable_operation_error( + &mut self, + primary_error: CudaError, + ) -> CudaError { + if self.context.inner.resource_lifetimes_poisoned() { + self.abandon_resources(); + return primary_error; + } + match self.release_after_stream_completion() { + Ok(()) => primary_error, + Err(release_error) => select_resource_release_error(primary_error, release_error), + } + } + + pub(super) fn synchronize_release_after_error( + &mut self, + primary_error: CudaError, + ) -> CudaError { + let outcome = self.context.synchronize_for_resource_release(); + if let Err(completion_error) = outcome.into_result() { + self.abandon_resources(); + return select_uncertain_completion_error(primary_error, Some(completion_error)); + } + match self.release_after_stream_completion() { + Ok(()) => primary_error, + Err(release_error) => select_resource_release_error(primary_error, release_error), + } + } +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/status.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/status.rs index b6f3a9c3..9637b6a5 100644 --- a/crates/j2k-cuda-runtime/src/htj2k_decode/status.rs +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/status.rs @@ -7,6 +7,13 @@ use crate::{ use super::CudaHtj2kStatus; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct CudaHtj2kStatusSpan { + pub(super) start: usize, + pub(super) count: usize, + pub(super) kernel: &'static str, +} + pub(super) fn first_status_error( statuses: &[CudaHtj2kStatus], kernel: &'static str, @@ -14,14 +21,40 @@ pub(super) fn first_status_error( statuses .iter() .copied() - .find(|status| !status.is_ok()) - .map(|status| CudaError::KernelStatus { + .enumerate() + .find(|(_, status)| !status.is_ok()) + .map(|(job_index, status)| CudaError::KernelJobStatus { kernel, + job_index, code: status.code, detail: status.detail, }) } +pub(super) fn first_group_status_error( + statuses: &[CudaHtj2kStatus], + spans: &[CudaHtj2kStatusSpan], +) -> Option { + for span in spans { + let end = span.start.checked_add(span.count)?; + let span_statuses = statuses.get(span.start..end)?; + if let Some((local_index, status)) = span_statuses + .iter() + .copied() + .enumerate() + .find(|(_, status)| !status.is_ok()) + { + return Some(CudaError::KernelJobStatus { + kernel: span.kernel, + job_index: span.start.saturating_add(local_index), + code: status.code, + detail: status.detail, + }); + } + } + None +} + pub(super) fn select_status_release_result( execution: CudaExecutionStats, status_error: Option, diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/status/tests.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/status/tests.rs index 8b33401e..efc883b6 100644 --- a/crates/j2k-cuda-runtime/src/htj2k_decode/status/tests.rs +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/status/tests.rs @@ -16,6 +16,61 @@ fn release_error() -> CudaError { } } +#[test] +fn ht_cleanup_status_preserves_the_failing_descriptor_index() { + let statuses = [ + CudaHtj2kStatus::default(), + CudaHtj2kStatus { + code: 7, + detail: 11, + ..CudaHtj2kStatus::default() + }, + ]; + assert!(matches!( + first_status_error(&statuses, "test_kernel"), + Some(CudaError::KernelJobStatus { + kernel: "test_kernel", + job_index: 1, + code: 7, + detail: 11, + }) + )); +} + +#[test] +fn grouped_ht_cleanup_status_preserves_the_global_descriptor_index_and_kernel() { + let statuses = [ + CudaHtj2kStatus::default(), + CudaHtj2kStatus::default(), + CudaHtj2kStatus { + code: 9, + detail: 17, + ..CudaHtj2kStatus::default() + }, + ]; + let spans = [ + CudaHtj2kStatusSpan { + start: 0, + count: 1, + kernel: "cleanup_only", + }, + CudaHtj2kStatusSpan { + start: 1, + count: 2, + kernel: "refinement", + }, + ]; + assert!(matches!( + first_group_status_error(&statuses, &spans), + Some(CudaError::KernelJobStatus { + kernel: "refinement", + job_index: 2, + code: 9, + detail: 17, + }) + )); +} + #[test] fn kernel_and_release_failures_are_both_preserved() { let error = select_status_release_result( diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/status_group.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/status_group.rs new file mode 100644 index 00000000..1517739c --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/status_group.rs @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + allocation::HostPhaseBudget, + bytes::{htj2k_statuses_as_bytes_mut, htj2k_statuses_byte_len}, + context::CudaContext, + error::{select_resource_release_error, CudaError}, + execution::CudaExecutionStats, + memory::{pooled_device_buffer, CudaBufferPool, CudaDeviceBuffer, CudaPooledDeviceBuffer}, +}; + +use super::{ + queued::CudaQueuedHtj2kCleanup, + status::{first_group_status_error, CudaHtj2kStatusSpan}, + CudaHtj2kStatus, +}; + +/// One group-owned status allocation shared by all bounded HTJ2K launches. +#[doc(hidden)] +#[derive(Debug)] +pub struct CudaQueuedHtj2kCleanupGroup { + context: CudaContext, + status_buffer: Option, + status_count: usize, + cleanups: Vec, + finished: bool, +} + +impl CudaQueuedHtj2kCleanupGroup { + /// Number of descriptor statuses downloaded by group completion. + #[doc(hidden)] + pub const fn status_count(&self) -> usize { + self.status_count + } + + /// Allocate one status arena for every HTJ2K descriptor in a homogeneous group. + #[doc(hidden)] + pub fn new( + context: &CudaContext, + pool: &CudaBufferPool, + status_count: usize, + ) -> Result { + if !pool.is_owned_by(context) { + return Err(CudaError::InvalidArgument { + message: "HTJ2K status group pool must belong to the decode context".to_string(), + }); + } + let status_buffer = (status_count != 0) + .then(|| pool.take(htj2k_statuses_byte_len(status_count)?)) + .transpose()?; + Ok(Self { + context: context.clone(), + status_buffer, + status_count, + cleanups: Vec::new(), + finished: false, + }) + } + + pub(crate) fn status_destination( + &self, + offset: usize, + count: usize, + ) -> Result<(&CudaDeviceBuffer, usize), CudaError> { + let end = offset + .checked_add(count) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if end > self.status_count { + return Err(CudaError::OutputTooSmall { + required: end, + have: self.status_count, + }); + } + let buffer = self + .status_buffer + .as_ref() + .ok_or_else(|| CudaError::InvalidArgument { + message: "non-empty HTJ2K launch requires a group status allocation".to_string(), + })?; + let byte_offset = offset + .checked_mul(core::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: offset })?; + Ok((pooled_device_buffer(buffer)?, byte_offset)) + } + + /// Retain one launch guard whose statuses were written into this group's arena. + #[doc(hidden)] + pub fn retain(&mut self, cleanup: CudaQueuedHtj2kCleanup) -> Result<(), CudaError> { + if !self.context.is_same_context(&cleanup.context) { + return Err(CudaError::InvalidArgument { + message: "HTJ2K cleanup and status group must belong to the same context" + .to_string(), + }); + } + if cleanup.status_count != 0 && !cleanup.uses_external_status_group { + return Err(CudaError::InvalidArgument { + message: "HTJ2K status group can retain only group-status cleanup launches" + .to_string(), + }); + } + let end = cleanup + .status_offset + .checked_add(cleanup.status_count) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if end > self.status_count { + return Err(CudaError::OutputTooSmall { + required: end, + have: self.status_count, + }); + } + if let Some(previous) = self.cleanups.last() { + let previous_end = previous + .status_offset + .checked_add(previous.status_count) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if cleanup.status_offset < previous_end { + return Err(CudaError::InvalidArgument { + message: "HTJ2K group status spans must be retained in non-overlapping order" + .to_string(), + }); + } + } + self.cleanups + .try_reserve(1) + .map_err(|_| CudaError::HostAllocationFailed { + bytes: self + .cleanups + .len() + .saturating_add(1) + .saturating_mul(core::mem::size_of::()), + })?; + self.cleanups.push(cleanup); + Ok(()) + } + + /// Download and validate all launch statuses with one group-level transfer. + pub fn finish(mut self) -> Result { + let result = self.finish_inner(); + self.finished = true; + result + } + + fn finish_inner(&mut self) -> Result { + let mut execution = CudaExecutionStats::default(); + let retained_live_bytes = self + .cleanups + .iter() + .map(|cleanup| cleanup.finish_host_live_bytes) + .max() + .unwrap_or(0); + let statuses_result = HostPhaseBudget::with_live_bytes( + "CUDA grouped HTJ2K cleanup status readback", + retained_live_bytes, + ) + .and_then(|mut budget| { + budget.try_vec_filled(self.status_count, CudaHtj2kStatus::default()) + }); + let mut statuses = match statuses_result { + Ok(statuses) => statuses, + Err(error) => return Err(self.synchronize_release_all(error)), + }; + if let Some(status_buffer) = self.status_buffer.as_ref() { + if let Err(error) = + status_buffer.copy_to_host(htj2k_statuses_as_bytes_mut(&mut statuses)) + { + return Err(self.synchronize_release_all(error)); + } + self.context.record_status_device_to_host_copy( + self.status_count + .saturating_mul(core::mem::size_of::()), + ); + } + + let status_error = self.cleanups.iter().find_map(|cleanup| { + first_group_status_error( + &statuses, + &[CudaHtj2kStatusSpan { + start: cleanup.status_offset, + count: cleanup.status_count, + kernel: cleanup.kernel_name, + }], + ) + }); + let release_result = self.release_all_after_stream_completion(&mut execution); + match (status_error, release_result) { + (Some(primary), Err(release)) => Err(select_resource_release_error(primary, release)), + (Some(error), Ok(())) | (None, Err(error)) => Err(error), + (None, Ok(())) => Ok(execution), + } + } + + fn release_all_after_stream_completion( + &mut self, + execution: &mut CudaExecutionStats, + ) -> Result<(), CudaError> { + let mut release_error = None; + for cleanup in &mut self.cleanups { + accumulate_execution(execution, cleanup.execution); + if let Err(error) = cleanup.release_after_stream_completion() { + release_error = Some(match release_error { + None => error, + Some(primary) => select_resource_release_error(primary, error), + }); + } + } + self.cleanups.clear(); + release_error.map_or(Ok(()), Err) + } + + fn synchronize_release_all(&mut self, primary: CudaError) -> CudaError { + let mut error = primary; + for cleanup in &mut self.cleanups { + if let Err(release) = cleanup.synchronize_and_release() { + error = select_resource_release_error(error, release); + } + } + self.cleanups.clear(); + error + } +} + +impl Drop for CudaQueuedHtj2kCleanupGroup { + fn drop(&mut self) { + if !self.finished { + let _ = self.finish_inner(); + } + } +} + +fn accumulate_execution(total: &mut CudaExecutionStats, next: CudaExecutionStats) { + total.kernel_dispatches = total + .kernel_dispatches + .saturating_add(next.kernel_dispatches); + total.copy_kernel_dispatches = total + .copy_kernel_dispatches + .saturating_add(next.copy_kernel_dispatches); + total.decode_kernel_dispatches = total + .decode_kernel_dispatches + .saturating_add(next.decode_kernel_dispatches); + total.hardware_decode |= next.hardware_decode; +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode/types.rs b/crates/j2k-cuda-runtime/src/htj2k_decode/types.rs index d2f0eb38..5dbf161d 100644 --- a/crates/j2k-cuda-runtime/src/htj2k_decode/types.rs +++ b/crates/j2k-cuda-runtime/src/htj2k_decode/types.rs @@ -93,6 +93,13 @@ pub(crate) struct CudaHtj2kCleanupMultiKernelJob { pub(crate) reserved_tail: u32, } +/// Device descriptor bytes used by one multi-target HTJ2K cleanup job. +#[doc(hidden)] +#[must_use] +pub const fn htj2k_cleanup_multi_descriptor_bytes() -> usize { + core::mem::size_of::() +} + /// One output buffer and its code-block jobs for batched HTJ2K dequantization. #[doc(hidden)] #[derive(Clone, Copy, Debug)] @@ -310,6 +317,7 @@ pub(super) struct Htj2kDecodeCodeblocksMultiLaunch<'a> { pub(super) jobs: &'a CudaDeviceBuffer, pub(super) tables: Htj2kDecodeKernelTables<'a>, pub(super) statuses: &'a CudaDeviceBuffer, + pub(super) status_byte_offset: usize, pub(super) job_count: usize, pub(super) mode: CudaLaunchMode, } diff --git a/crates/j2k-cuda-runtime/src/j2k_decode.rs b/crates/j2k-cuda-runtime/src/j2k_decode.rs index e37e616a..9180f526 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode.rs @@ -29,18 +29,26 @@ pub(crate) struct CudaJ2kIdwtBatchTraceRow { pub(crate) elapsed_us: u128, } +pub use self::store::CudaQueuedJ2kStoreBatch; #[cfg(test)] pub(crate) use self::trace::idwt_batch_uses_cooperative_53; pub use self::types::{ CudaJ2kIdwtJob, CudaJ2kIdwtTarget, CudaJ2kInverseMctJob, CudaJ2kRect, CudaJ2kStoreGray16Job, - CudaJ2kStoreGray8Job, CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, - CudaJ2kStoreRgb8MctJob, CudaJ2kStoreRgb8MctTarget, CudaJ2kStridedInterleavedPixels, + CudaJ2kStoreGray16Target, CudaJ2kStoreGray8Job, CudaJ2kStoreGray8Target, + CudaJ2kStoreGrayI16Target, CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, + CudaJ2kStoreRgb8MctJob, CudaJ2kStoreRgb8MctTarget, CudaJ2kStoreRgbNativeJob, + CudaJ2kStoreRgbNativeTarget, CudaJ2kStoreRgbaNativeJob, CudaJ2kStoreRgbaNativeTarget, + CudaJ2kStridedInterleavedPixels, }; #[cfg(test)] pub(crate) use self::validation::checked_f32_words_byte_len; pub(crate) use self::{ trace::{format_idwt_batch_trace_row, idwt_batch_kernel_mode, idwt_batch_trace_row}, - types::{CudaJ2kIdwtMultiKernelJob, CudaJ2kStoreRgb8MctBatchJob}, + types::{ + CudaJ2kIdwtMultiKernelJob, CudaJ2kStoreGray16BatchJob, CudaJ2kStoreGray8BatchJob, + CudaJ2kStoreGrayI16BatchJob, CudaJ2kStoreRgb8MctBatchJob, CudaJ2kStoreRgbNativeBatchJob, + CudaJ2kStoreRgbaNativeBatchJob, + }, validation::{ active_dwt53_buffers, append_j2k_idwt_multi_kernel_jobs, j2k_idwt_multi_kernel_jobs, }, diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store.rs index 6b6ca9a2..8da8ce6f 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/store.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store.rs @@ -18,7 +18,12 @@ use super::types::{ }; mod batch; +mod color_native_batch; +mod color_native_rgba_batch; mod destination; +mod destination_groups; +mod grayscale_batch; +pub use grayscale_batch::CudaQueuedJ2kStoreBatch; mod validation; use destination::{validate_store_destination, zero_unwritten_store_output}; diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/batch.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/batch.rs index e1ee52f0..bd5f9107 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/store/batch.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/batch.rs @@ -16,6 +16,8 @@ use super::{ }; use crate::j2k_decode::types::{CudaJ2kStoreRgb8MctBatchJob, CudaJ2kStoreRgb8MctTarget}; +mod external; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct Rgb8MctTargetPlan { output_bytes: usize, @@ -215,7 +217,16 @@ impl CudaContext { plan.active_job_count, "J2K store batch active-job count mismatch", )?; - self.launch_j2k_store_rgb8_mct_batch(&jobs_buffer, plan.max_pixels, plan.active_job_count)?; + // SAFETY: this owned path retains every plane, output, and uploaded + // job buffer through the immediate context completion boundary. + unsafe { + self.launch_j2k_store_rgb8_mct_batch_enqueue( + &jobs_buffer, + plan.max_pixels, + plan.active_job_count, + )?; + } + self.synchronize()?; Ok(CudaKernelBatchOutput { outputs, execution: CudaExecutionStats { @@ -311,7 +322,16 @@ impl CudaContext { if initialize_output()? { self.synchronize()?; } - self.launch_j2k_store_rgb8_mct_batch(&jobs_buffer, plan.max_pixels, plan.active_job_count)?; + // SAFETY: this owned path retains every plane, output, and uploaded + // job buffer through the immediate context completion boundary. + unsafe { + self.launch_j2k_store_rgb8_mct_batch_enqueue( + &jobs_buffer, + plan.max_pixels, + plan.active_job_count, + )?; + } + self.synchronize()?; Ok(CudaKernelContiguousBatchOutput { output, ranges, @@ -326,18 +346,4 @@ impl CudaContext { } #[cfg(test)] -mod tests { - use super::ensure_internal_count; - use crate::error::CudaError; - - #[test] - fn batch_materialization_mismatch_is_a_typed_error() { - assert!(ensure_internal_count(3, 3, "matching counts").is_ok()); - assert!(matches!( - ensure_internal_count(2, 3, "fixture mismatch"), - Err(CudaError::InternalInvariant { - what: "fixture mismatch" - }) - )); - } -} +mod tests; diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/batch/external.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/batch/external.rs new file mode 100644 index 00000000..ee115bff --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/batch/external.rs @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + allocation::HostPhaseBudget, + bytes::store_rgb8_mct_batch_jobs_as_bytes, + context::CudaContext, + error::CudaError, + execution::CudaExecutionStats, + memory::{CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut}, +}; + +use super::{ensure_internal_count, validate_rgb8_mct_targets, Rgb8MctBatchPlan}; +use crate::j2k_decode::{ + store::grayscale_batch::CudaQueuedJ2kStoreBatch, + types::{CudaJ2kStoreRgb8MctBatchJob, CudaJ2kStoreRgb8MctTarget}, +}; + +fn materialize_external_rgb_batch( + targets: &[CudaJ2kStoreRgb8MctTarget<'_>], + plan: &Rgb8MctBatchPlan, + base_ptr: u64, +) -> Result<(Vec, Vec), CudaError> { + let mut budget = HostPhaseBudget::new("CUDA external J2K RGB store batch metadata"); + budget.account_vec(&plan.targets)?; + let mut ranges = budget.try_vec_with_capacity(plan.targets.len())?; + let mut offset = 0usize; + for target_plan in &plan.targets { + ranges.push(CudaDeviceBufferRange { + offset, + len: target_plan.output_bytes, + }); + offset = offset + .checked_add(target_plan.output_bytes) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + } + ensure_internal_count( + offset, + plan.total_bytes, + "external J2K RGB store output range mismatch", + )?; + + let mut kernel_jobs = budget.try_vec_with_capacity(plan.active_job_count)?; + for ((target, range), target_plan) in targets.iter().zip(&ranges).zip(&plan.targets) { + if !target_plan.active { + continue; + } + let range_offset = u64::try_from(range.offset) + .map_err(|_| CudaError::LengthTooLarge { len: range.offset })?; + let output_ptr = base_ptr + .checked_add(range_offset) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + kernel_jobs.push(CudaJ2kStoreRgb8MctBatchJob { + plane0_ptr: target.plane0.device_ptr(), + plane1_ptr: target.plane1.device_ptr(), + plane2_ptr: target.plane2.device_ptr(), + output_ptr, + job: target.job, + reserved_tail: 0, + }); + } + ensure_internal_count( + kernel_jobs.len(), + plan.active_job_count, + "external J2K RGB store active-job count mismatch", + )?; + Ok((ranges, kernel_jobs)) +} + +impl CudaContext { + /// Enqueue inverse RCT/ICT and native RGB8/RGBA8 stores directly into a + /// validated caller-owned CUDA range. + /// + /// # Safety + /// + /// Every component plane and the destination allocation must remain live + /// and unavailable for mutation or reuse until the returned guard finishes + /// or drops after confirmed CUDA completion. If completion cannot be + /// proven, the caller must quarantine those allocations. Decoded pixels + /// must not be exposed as valid until codec status validation succeeds. + #[doc(hidden)] + pub unsafe fn j2k_store_rgb8_mct_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgb8MctTarget<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + let plan = validate_rgb8_mct_targets(self, targets, 0)?; + if !self.is_same_context(destination.context()) { + return Err(CudaError::InvalidArgument { + message: "external RGB destination belongs to a different CUDA context".to_string(), + }); + } + if destination.byte_len() < plan.total_bytes { + return Err(CudaError::OutputTooSmall { + required: plan.total_bytes, + have: destination.byte_len(), + }); + } + if plan.requires_zero_fill { + return Err(CudaError::InvalidArgument { + message: "external RGB batch destination requires full output coverage".to_string(), + }); + } + + let (ranges, kernel_jobs) = + materialize_external_rgb_batch(targets, &plan, destination.device_ptr())?; + if plan.active_job_count == 0 { + return Ok(( + ranges, + CudaQueuedJ2kStoreBatch { + completion: None, + jobs: None, + execution: CudaExecutionStats::default(), + }, + )); + } + + let jobs_buffer = self.upload(store_rgb8_mct_batch_jobs_as_bytes(&kernel_jobs))?; + // SAFETY: the returned completion guard retains `jobs_buffer`; the + // caller's unsafe contract retains every plane and destination range. + if let Err(error) = unsafe { + self.launch_j2k_store_rgb8_mct_batch_enqueue( + &jobs_buffer, + plan.max_pixels, + plan.active_job_count, + ) + } { + return self.synchronize_then_error(error); + } + let completion = self.create_event().and_then(|event| { + event.record_default_stream()?; + Ok(event) + }); + let completion = match completion { + Ok(completion) => completion, + Err(error) => return self.synchronize_then_error(error), + }; + Ok(( + ranges, + CudaQueuedJ2kStoreBatch { + completion: Some(completion), + jobs: Some(jobs_buffer), + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }, + )) + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/batch/tests.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/batch/tests.rs new file mode 100644 index 00000000..91472f1d --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/batch/tests.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::ensure_internal_count; +use crate::error::CudaError; + +#[test] +fn batch_materialization_mismatch_is_a_typed_error() { + assert!(ensure_internal_count(3, 3, "matching counts").is_ok()); + assert!(matches!( + ensure_internal_count(2, 3, "fixture mismatch"), + Err(CudaError::InternalInvariant { + what: "fixture mismatch" + }) + )); +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_batch.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_batch.rs new file mode 100644 index 00000000..91da674b --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_batch.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + allocation::HostPhaseBudget, + bytes::store_rgb_native_batch_jobs_as_bytes, + context::CudaContext, + error::CudaError, + execution::{CudaExecutionStats, CudaKernelContiguousBatchOutput}, + memory::{CudaDeviceBuffer, CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut}, +}; + +use super::CudaQueuedJ2kStoreBatch; +use crate::j2k_decode::types::{CudaJ2kStoreRgbNativeBatchJob, CudaJ2kStoreRgbNativeTarget}; + +mod plan; +use plan::{ + validate_external_destination, validate_native_rgb_targets, NativeRgbBatchPlan, + NativeRgbStorage, +}; + +impl CudaContext { + /// Store an exact-native RGB U8 batch into one J2K-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgb8_native_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + ) -> Result { + self.store_rgb_native_batch_contiguous_device(targets, NativeRgbStorage::U8) + } + + /// Enqueue exact-native RGB U8 output into one J2K-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgb8_native_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + self.store_rgb_native_batch_contiguous_device_enqueue(targets, NativeRgbStorage::U8) + } + + /// Store an exact-native RGB U16 batch into one J2K-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgb16_native_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + ) -> Result { + self.store_rgb_native_batch_contiguous_device(targets, NativeRgbStorage::U16) + } + + /// Enqueue exact-native RGB U16 output into one J2K-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgb16_native_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + self.store_rgb_native_batch_contiguous_device_enqueue(targets, NativeRgbStorage::U16) + } + + /// Store an exact-native signed RGB I16 batch into one J2K-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgbi16_native_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + ) -> Result { + self.store_rgb_native_batch_contiguous_device(targets, NativeRgbStorage::I16) + } + + /// Enqueue exact-native signed RGB I16 output into one J2K-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgbi16_native_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + self.store_rgb_native_batch_contiguous_device_enqueue(targets, NativeRgbStorage::I16) + } + + fn store_rgb_native_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + storage: NativeRgbStorage, + ) -> Result { + let (output, ranges, queued) = + self.store_rgb_native_batch_contiguous_device_enqueue(targets, storage)?; + let execution = match queued.finish() { + Ok(execution) => execution, + Err(error) => { + if error.completion_is_uncertain() { + core::mem::forget(output); + } + return Err(error); + } + }; + Ok(CudaKernelContiguousBatchOutput { + output, + ranges, + execution, + }) + } + + fn store_rgb_native_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + storage: NativeRgbStorage, + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + let plan = validate_native_rgb_targets(self, targets, storage)?; + let output = self.allocate(plan.total_bytes)?; + // SAFETY: the returned owners retain the destination and every caller + // retains the source planes until the queued guard is retired. + let queued = + unsafe { self.enqueue_rgb_native_batch(targets, &plan, output.device_ptr(), storage)? }; + let ranges = plan.ranges; + Ok((output, ranges, queued)) + } + + /// Enqueue exact-native RGB U8 output into caller-owned CUDA storage. + /// + /// # Safety + /// + /// All source planes and the external destination must remain live and + /// inaccessible until the returned completion owner is retired. + #[doc(hidden)] + pub unsafe fn j2k_store_rgb8_native_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + // SAFETY: forwarded from the caller after validating the destination. + unsafe { + self.store_rgb_native_batch_into_external_device_enqueue( + targets, + destination, + NativeRgbStorage::U8, + ) + } + } + + /// Enqueue exact-native RGB U16 output into caller-owned CUDA storage. + /// + /// # Safety + /// + /// All source planes and the external destination must remain live and + /// inaccessible until the returned completion owner is retired. + #[doc(hidden)] + pub unsafe fn j2k_store_rgb16_native_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + // SAFETY: forwarded from the caller after validating the destination. + unsafe { + self.store_rgb_native_batch_into_external_device_enqueue( + targets, + destination, + NativeRgbStorage::U16, + ) + } + } + + /// Enqueue exact-native signed RGB I16 output into caller-owned CUDA storage. + /// + /// # Safety + /// + /// All source planes and the external destination must remain live and + /// inaccessible until the returned completion owner is retired. + #[doc(hidden)] + pub unsafe fn j2k_store_rgbi16_native_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + // SAFETY: forwarded from the caller after validating the destination. + unsafe { + self.store_rgb_native_batch_into_external_device_enqueue( + targets, + destination, + NativeRgbStorage::I16, + ) + } + } + + unsafe fn store_rgb_native_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + storage: NativeRgbStorage, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + let plan = validate_native_rgb_targets(self, targets, storage)?; + let output_base = validate_external_destination(self, destination, &plan, storage)?; + // SAFETY: caller retains every source and destination owner through + // the returned completion guard. + let queued = + unsafe { self.enqueue_rgb_native_batch(targets, &plan, output_base, storage)? }; + Ok((plan.ranges, queued)) + } + + unsafe fn enqueue_rgb_native_batch( + &self, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + plan: &NativeRgbBatchPlan, + output_base: u64, + storage: NativeRgbStorage, + ) -> Result { + if plan.active_count == 0 { + return Ok(CudaQueuedJ2kStoreBatch { + completion: None, + jobs: None, + execution: CudaExecutionStats::default(), + }); + } + let mut budget = HostPhaseBudget::new("CUDA exact-native RGB batch job metadata"); + let mut jobs = budget.try_vec_with_capacity(plan.active_count)?; + for (target, item) in targets.iter().zip(&plan.items) { + if !item.active { + continue; + } + let range = plan.ranges[item.range_index]; + let offset = u64::try_from(range.offset) + .map_err(|_| CudaError::LengthTooLarge { len: range.offset })?; + jobs.push(CudaJ2kStoreRgbNativeBatchJob { + plane0_ptr: target.plane0.device_ptr(), + plane1_ptr: target.plane1.device_ptr(), + plane2_ptr: target.plane2.device_ptr(), + output_ptr: output_base + .checked_add(offset) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?, + job: target.job, + }); + } + let jobs_buffer = self.upload(store_rgb_native_batch_jobs_as_bytes(&jobs))?; + let launch = match storage { + NativeRgbStorage::U8 => { + // SAFETY: `jobs_buffer` is retained below through the event. + unsafe { + self.launch_j2k_store_rgb8_native_batch_enqueue( + &jobs_buffer, + plan.max_pixels, + plan.active_count, + ) + } + } + NativeRgbStorage::U16 => { + // SAFETY: `jobs_buffer` is retained below through the event. + unsafe { + self.launch_j2k_store_rgb16_native_batch_enqueue( + &jobs_buffer, + plan.max_pixels, + plan.active_count, + ) + } + } + NativeRgbStorage::I16 => { + // SAFETY: `jobs_buffer` is retained below through the event. + unsafe { + self.launch_j2k_store_rgbi16_native_batch_enqueue( + &jobs_buffer, + plan.max_pixels, + plan.active_count, + ) + } + } + }; + if let Err(error) = launch { + return self.synchronize_then_error(error); + } + let completion = self.create_event().and_then(|event| { + event.record_default_stream()?; + Ok(event) + }); + let completion = match completion { + Ok(completion) => completion, + Err(error) => return self.synchronize_then_error(error), + }; + Ok(CudaQueuedJ2kStoreBatch { + completion: Some(completion), + jobs: Some(jobs_buffer), + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } +} + +#[cfg(test)] +mod tests { + use super::plan::{NativeRgbStorage, RGB_LAYOUT_NCHW, RGB_LAYOUT_NHWC}; + + #[test] + fn exact_native_storage_and_layout_contract_is_not_display_scaled() { + assert_eq!(NativeRgbStorage::U8.max_precision(), 8); + assert_eq!(NativeRgbStorage::U16.max_precision(), 16); + assert_eq!(NativeRgbStorage::I16.max_precision(), 16); + assert_eq!((RGB_LAYOUT_NHWC, RGB_LAYOUT_NCHW), (0, 1)); + + let exports = include_str!("../../cuda_oxide_j2k_decode_store/simt/src/exports.rs"); + let native_color = + include_str!("../../cuda_oxide_j2k_decode_store/simt/src/native_color.rs"); + for entrypoint in [ + "j2k_store_rgb8_native_batch", + "j2k_store_rgb16_native_batch", + "j2k_store_rgbi16_native_batch", + "j2k_store_rgba8_native_batch", + "j2k_store_rgba16_native_batch", + "j2k_store_rgbai16_native_batch", + ] { + assert!(exports.contains(entrypoint)); + } + assert!(native_color.contains("sample_as_native_u8(samples.0")); + assert!(native_color.contains("sample_as_native_u16(samples.0")); + assert!(native_color.contains("sample_as_native_i16(samples.0")); + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_batch/plan.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_batch/plan.rs new file mode 100644 index 00000000..6be48b22 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_batch/plan.rs @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + allocation::HostPhaseBudget, + context::CudaContext, + error::CudaError, + j2k_decode::types::CudaJ2kStoreRgbNativeTarget, + kernels::j2k_store_batch_launch_geometry, + memory::{checked_image_words, CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut}, +}; + +use super::super::{ + destination_groups::{plan_dense_destination_regions, DenseDestinationRegion}, + validation::{ + validate_rgb_tile_compatibility, validate_store_buffer_context, validate_store_plane, + }, +}; + +const RGB_CHANNELS: usize = 3; +pub(super) const RGB_LAYOUT_NHWC: u32 = 0; +pub(super) const RGB_LAYOUT_NCHW: u32 = 1; +const RGB_TRANSFORM_NONE: u32 = 0; +const RGB_TRANSFORM_ICT: u32 = 2; + +#[derive(Clone, Copy)] +pub(super) enum NativeRgbStorage { + U8, + U16, + I16, +} + +impl NativeRgbStorage { + const fn bytes_per_sample(self) -> usize { + match self { + Self::U8 => std::mem::size_of::(), + Self::U16 => std::mem::size_of::(), + Self::I16 => std::mem::size_of::(), + } + } + + pub(super) const fn max_precision(self) -> u32 { + match self { + Self::U8 => 8, + Self::U16 | Self::I16 => 16, + } + } + + const fn alignment(self) -> usize { + self.bytes_per_sample() + } +} + +pub(super) struct NativeRgbBatchItemPlan { + pub(super) range_index: usize, + pub(super) active: bool, +} + +pub(super) struct NativeRgbBatchPlan { + pub(super) items: Vec, + pub(super) ranges: Vec, + pub(super) total_bytes: usize, + pub(super) max_pixels: usize, + pub(super) active_count: usize, +} + +pub(super) fn validate_native_rgb_targets( + context: &CudaContext, + targets: &[CudaJ2kStoreRgbNativeTarget<'_>], + storage: NativeRgbStorage, +) -> Result { + validate_store_buffer_context( + context, + targets + .iter() + .flat_map(|target| [target.plane0, target.plane1, target.plane2]), + )?; + let mut budget = HostPhaseBudget::new("CUDA exact-native RGB store batch plan"); + let mut regions = budget.try_vec_with_capacity(targets.len())?; + let mut active = budget.try_vec_with_capacity(targets.len())?; + let mut max_pixels = 0usize; + let mut active_count = 0usize; + + for (index, target) in targets.iter().enumerate() { + let pixels = validate_native_rgb_target(target, storage)?; + if pixels != 0 { + active_count = active_count + .checked_add(1) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + max_pixels = max_pixels.max(pixels); + } + if index != 0 && target.output_index == targets[index - 1].output_index { + validate_rgb_tile_compatibility(&targets[index - 1].job, &target.job)?; + } + let job = target.job; + regions.push(DenseDestinationRegion { + output_index: target.output_index, + output_width: job.output_width, + output_height: job.output_height, + output_x: job.output_x, + output_y: job.output_y, + copy_width: job.copy_width, + copy_height: job.copy_height, + }); + active.push(pixels != 0); + } + let destinations = plan_dense_destination_regions( + ®ions, + RGB_CHANNELS, + storage.bytes_per_sample(), + &mut budget, + )?; + if destinations.requires_zero_fill { + return Err(CudaError::InvalidArgument { + message: "exact-native RGB tile stores must cover each dense destination exactly" + .to_string(), + }); + } + let mut items = budget.try_vec_with_capacity(targets.len())?; + items.extend( + destinations + .item_range_indices + .iter() + .copied() + .zip(active) + .map(|(range_index, active)| NativeRgbBatchItemPlan { + range_index, + active, + }), + ); + if active_count != 0 && j2k_store_batch_launch_geometry(max_pixels, active_count).is_none() { + return Err(CudaError::LengthTooLarge { len: active_count }); + } + Ok(NativeRgbBatchPlan { + items, + ranges: destinations.ranges, + total_bytes: destinations.total_bytes, + max_pixels, + active_count, + }) +} + +fn validate_native_rgb_target( + target: &CudaJ2kStoreRgbNativeTarget<'_>, + storage: NativeRgbStorage, +) -> Result { + let job = target.job; + if !matches!(job.layout, RGB_LAYOUT_NHWC | RGB_LAYOUT_NCHW) { + return Err(CudaError::InvalidArgument { + message: "exact-native RGB layout must be NHWC or NCHW".to_string(), + }); + } + if !(RGB_TRANSFORM_NONE..=RGB_TRANSFORM_ICT).contains(&job.transform) { + return Err(CudaError::InvalidArgument { + message: "exact-native RGB transform selector is invalid".to_string(), + }); + } + if job.reserved != 0 { + return Err(CudaError::InvalidArgument { + message: "exact-native RGB reserved field must be zero".to_string(), + }); + } + let max_precision = storage.max_precision(); + if [job.bit_depth0, job.bit_depth1, job.bit_depth2] + .into_iter() + .any(|precision| precision == 0 || precision > max_precision) + { + return Err(CudaError::InvalidArgument { + message: format!("exact-native RGB precision must fit in {max_precision}-bit storage"), + }); + } + let pixels = checked_image_words(job.copy_width, job.copy_height, 1)?; + if pixels != 0 { + for (plane, input_width, source_x, source_y) in [ + ( + target.plane0, + job.input_width0, + job.source_x0, + job.source_y0, + ), + ( + target.plane1, + job.input_width1, + job.source_x1, + job.source_y1, + ), + ( + target.plane2, + job.input_width2, + job.source_x2, + job.source_y2, + ), + ] { + validate_store_plane( + plane, + input_width, + source_x, + source_y, + job.copy_width, + job.copy_height, + )?; + } + } + Ok(pixels) +} + +pub(super) fn validate_external_destination( + context: &CudaContext, + destination: &CudaExternalDeviceBufferViewMut<'_>, + plan: &NativeRgbBatchPlan, + storage: NativeRgbStorage, +) -> Result { + if !context.is_same_context(destination.context()) { + return Err(CudaError::InvalidArgument { + message: "external exact-native RGB destination belongs to a different CUDA context" + .to_string(), + }); + } + if destination.byte_len() < plan.total_bytes { + return Err(CudaError::OutputTooSmall { + required: plan.total_bytes, + have: destination.byte_len(), + }); + } + if !destination + .device_ptr() + .is_multiple_of(storage.alignment() as u64) + { + return Err(CudaError::InvalidArgument { + message: format!( + "external exact-native RGB destination is not {}-byte aligned", + storage.alignment() + ), + }); + } + Ok(destination.device_ptr()) +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_rgba_batch.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_rgba_batch.rs new file mode 100644 index 00000000..16fe5ce1 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_rgba_batch.rs @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +mod validation; + +use validation::{validate_external, validate_targets, NativeRgbaBatchPlan}; + +use crate::{ + allocation::HostPhaseBudget, + bytes::store_rgba_native_batch_jobs_as_bytes, + context::CudaContext, + error::CudaError, + execution::{CudaExecutionStats, CudaKernelContiguousBatchOutput}, + memory::{CudaDeviceBuffer, CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut}, +}; + +use super::CudaQueuedJ2kStoreBatch; +use crate::j2k_decode::types::{CudaJ2kStoreRgbaNativeBatchJob, CudaJ2kStoreRgbaNativeTarget}; + +const RGBA_CHANNELS: usize = 4; + +#[derive(Clone, Copy)] +enum NativeRgbaStorage { + U8, + U16, + I16, +} + +impl NativeRgbaStorage { + const fn bytes_per_sample(self) -> usize { + match self { + Self::U8 => std::mem::size_of::(), + Self::U16 => std::mem::size_of::(), + Self::I16 => std::mem::size_of::(), + } + } + + const fn max_precision(self) -> u32 { + match self { + Self::U8 => 8, + Self::U16 | Self::I16 => 16, + } + } +} + +impl CudaContext { + /// Store exact-native RGBA U8 into one codec-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgba8_native_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + ) -> Result { + self.store_rgba_native_batch(targets, NativeRgbaStorage::U8) + } + + /// Store exact-native RGBA U16 into one codec-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgba16_native_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + ) -> Result { + self.store_rgba_native_batch(targets, NativeRgbaStorage::U16) + } + + /// Store exact-native signed RGBA I16 into one codec-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgbai16_native_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + ) -> Result { + self.store_rgba_native_batch(targets, NativeRgbaStorage::I16) + } + + /// Enqueue exact-native RGBA U8 into one codec-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgba8_native_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + self.enqueue_owned_rgba_native_batch(targets, NativeRgbaStorage::U8) + } + + /// Enqueue exact-native RGBA U16 into one codec-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgba16_native_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + self.enqueue_owned_rgba_native_batch(targets, NativeRgbaStorage::U16) + } + + /// Enqueue exact-native signed RGBA I16 into one codec-owned allocation. + #[doc(hidden)] + pub fn j2k_store_rgbai16_native_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + self.enqueue_owned_rgba_native_batch(targets, NativeRgbaStorage::I16) + } + + fn store_rgba_native_batch( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + storage: NativeRgbaStorage, + ) -> Result { + let (output, ranges, queued) = self.enqueue_owned_rgba_native_batch(targets, storage)?; + let execution = match queued.finish() { + Ok(execution) => execution, + Err(error) => { + if error.completion_is_uncertain() { + core::mem::forget(output); + } + return Err(error); + } + }; + Ok(CudaKernelContiguousBatchOutput { + output, + ranges, + execution, + }) + } + + fn enqueue_owned_rgba_native_batch( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + storage: NativeRgbaStorage, + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + let plan = validate_targets(self, targets, storage)?; + let output = self.allocate(plan.total_bytes)?; + // SAFETY: returned owners retain output and the caller retains planes. + let queued = + unsafe { self.enqueue_rgba_native(targets, &plan, output.device_ptr(), storage)? }; + Ok((output, plan.ranges, queued)) + } + + /// Enqueue RGBA U8 into validated caller-owned CUDA storage. + #[doc(hidden)] + pub unsafe fn j2k_store_rgba8_native_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + // SAFETY: forwarded lifetime and exclusivity contract. + unsafe { self.enqueue_external_rgba_native(targets, destination, NativeRgbaStorage::U8) } + } + + /// Enqueue RGBA U16 into validated caller-owned CUDA storage. + #[doc(hidden)] + pub unsafe fn j2k_store_rgba16_native_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + // SAFETY: forwarded lifetime and exclusivity contract. + unsafe { self.enqueue_external_rgba_native(targets, destination, NativeRgbaStorage::U16) } + } + + /// Enqueue signed RGBA I16 into validated caller-owned CUDA storage. + #[doc(hidden)] + pub unsafe fn j2k_store_rgbai16_native_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + // SAFETY: forwarded lifetime and exclusivity contract. + unsafe { self.enqueue_external_rgba_native(targets, destination, NativeRgbaStorage::I16) } + } + + unsafe fn enqueue_external_rgba_native( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + storage: NativeRgbaStorage, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + let plan = validate_targets(self, targets, storage)?; + let output_base = validate_external(self, destination, &plan, storage)?; + // SAFETY: caller retains and excludes all source/destination storage. + let queued = unsafe { self.enqueue_rgba_native(targets, &plan, output_base, storage)? }; + Ok((plan.ranges, queued)) + } + + unsafe fn enqueue_rgba_native( + &self, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + plan: &NativeRgbaBatchPlan, + output_base: u64, + storage: NativeRgbaStorage, + ) -> Result { + if plan.active_count == 0 { + return Ok(CudaQueuedJ2kStoreBatch { + completion: None, + jobs: None, + execution: CudaExecutionStats::default(), + }); + } + let mut budget = HostPhaseBudget::new("CUDA exact-native RGBA batch job metadata"); + let mut jobs = budget.try_vec_with_capacity(plan.active_count)?; + for (target, item) in targets.iter().zip(&plan.items) { + if !item.active { + continue; + } + let range = plan.ranges[item.range_index]; + let offset = u64::try_from(range.offset) + .map_err(|_| CudaError::LengthTooLarge { len: range.offset })?; + jobs.push(CudaJ2kStoreRgbaNativeBatchJob { + plane0_ptr: target.plane0.device_ptr(), + plane1_ptr: target.plane1.device_ptr(), + plane2_ptr: target.plane2.device_ptr(), + plane3_ptr: target.plane3.device_ptr(), + output_ptr: output_base + .checked_add(offset) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?, + job: target.job, + reserved_tail: 0, + }); + } + let jobs = self.upload(store_rgba_native_batch_jobs_as_bytes(&jobs))?; + let launch = match storage { + // SAFETY: validated targets and the uploaded descriptor owner stay + // live until the recorded completion event is retired below. + NativeRgbaStorage::U8 => unsafe { + self.launch_j2k_store_rgba8_native_batch_enqueue( + &jobs, + plan.max_pixels, + plan.active_count, + ) + }, + // SAFETY: the same validated ownership and completion contract + // applies to the two-byte unsigned store entrypoint. + NativeRgbaStorage::U16 => unsafe { + self.launch_j2k_store_rgba16_native_batch_enqueue( + &jobs, + plan.max_pixels, + plan.active_count, + ) + }, + // SAFETY: the same validated ownership and completion contract + // applies to the two-byte signed store entrypoint. + NativeRgbaStorage::I16 => unsafe { + self.launch_j2k_store_rgbai16_native_batch_enqueue( + &jobs, + plan.max_pixels, + plan.active_count, + ) + }, + }; + if let Err(error) = launch { + return self.synchronize_then_error(error); + } + let completion = match self.create_event().and_then(|event| { + event.record_default_stream()?; + Ok(event) + }) { + Ok(completion) => completion, + Err(error) => return self.synchronize_then_error(error), + }; + Ok(CudaQueuedJ2kStoreBatch { + completion: Some(completion), + jobs: Some(jobs), + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_rgba_batch/validation.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_rgba_batch/validation.rs new file mode 100644 index 00000000..8f2dca37 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_rgba_batch/validation.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{NativeRgbaStorage, RGBA_CHANNELS}; +use crate::{ + allocation::HostPhaseBudget, + context::CudaContext, + error::CudaError, + j2k_decode::{ + store::{ + destination_groups::{plan_dense_destination_regions, DenseDestinationRegion}, + validation::{validate_store_buffer_context, validate_store_plane}, + }, + types::{CudaJ2kStoreRgbaNativeJob, CudaJ2kStoreRgbaNativeTarget}, + }, + kernels::j2k_store_batch_launch_geometry, + memory::{checked_image_words, CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut}, +}; + +pub(super) struct NativeRgbaBatchItemPlan { + pub(super) range_index: usize, + pub(super) active: bool, +} + +pub(super) struct NativeRgbaBatchPlan { + pub(super) items: Vec, + pub(super) ranges: Vec, + pub(super) total_bytes: usize, + pub(super) max_pixels: usize, + pub(super) active_count: usize, +} + +pub(super) fn validate_targets( + context: &CudaContext, + targets: &[CudaJ2kStoreRgbaNativeTarget<'_>], + storage: NativeRgbaStorage, +) -> Result { + validate_store_buffer_context( + context, + targets + .iter() + .flat_map(|target| [target.plane0, target.plane1, target.plane2, target.plane3]), + )?; + let mut budget = HostPhaseBudget::new("CUDA exact-native RGBA store batch plan"); + let mut regions = budget.try_vec_with_capacity(targets.len())?; + let mut active = budget.try_vec_with_capacity(targets.len())?; + let mut max_pixels = 0usize; + let mut active_count = 0usize; + for (index, target) in targets.iter().enumerate() { + let pixels = validate_target(target, storage)?; + if pixels != 0 { + active_count = active_count + .checked_add(1) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + max_pixels = max_pixels.max(pixels); + } + if index != 0 && target.output_index == targets[index - 1].output_index { + validate_tile_compatibility(&targets[index - 1].job, &target.job)?; + } + let job = target.job; + regions.push(DenseDestinationRegion { + output_index: target.output_index, + output_width: job.output_width, + output_height: job.output_height, + output_x: job.output_x, + output_y: job.output_y, + copy_width: job.copy_width, + copy_height: job.copy_height, + }); + active.push(pixels != 0); + } + let destinations = plan_dense_destination_regions( + ®ions, + RGBA_CHANNELS, + storage.bytes_per_sample(), + &mut budget, + )?; + if destinations.requires_zero_fill { + return Err(CudaError::InvalidArgument { + message: "exact-native RGBA tile stores must cover each dense destination exactly" + .to_string(), + }); + } + let mut items = budget.try_vec_with_capacity(targets.len())?; + items.extend( + destinations + .item_range_indices + .iter() + .copied() + .zip(active) + .map(|(range_index, active)| NativeRgbaBatchItemPlan { + range_index, + active, + }), + ); + if active_count != 0 && j2k_store_batch_launch_geometry(max_pixels, active_count).is_none() { + return Err(CudaError::LengthTooLarge { len: active_count }); + } + Ok(NativeRgbaBatchPlan { + items, + ranges: destinations.ranges, + total_bytes: destinations.total_bytes, + max_pixels, + active_count, + }) +} + +fn validate_target( + target: &CudaJ2kStoreRgbaNativeTarget<'_>, + storage: NativeRgbaStorage, +) -> Result { + let job = target.job; + if !matches!(job.layout, 0 | 1) { + return Err(CudaError::InvalidArgument { + message: "exact-native RGBA layout must be NHWC or NCHW".to_string(), + }); + } + if job.transform > 2 || job.reserved != 0 { + return Err(CudaError::InvalidArgument { + message: "exact-native RGBA transform or reserved field is invalid".to_string(), + }); + } + if [ + job.bit_depth0, + job.bit_depth1, + job.bit_depth2, + job.bit_depth3, + ] + .into_iter() + .any(|precision| precision == 0 || precision > storage.max_precision()) + { + return Err(CudaError::InvalidArgument { + message: format!( + "exact-native RGBA precision must fit in {}-bit storage", + storage.max_precision() + ), + }); + } + let pixels = checked_image_words(job.copy_width, job.copy_height, 1)?; + if pixels != 0 { + for (plane, width, x, y) in [ + ( + target.plane0, + job.input_width0, + job.source_x0, + job.source_y0, + ), + ( + target.plane1, + job.input_width1, + job.source_x1, + job.source_y1, + ), + ( + target.plane2, + job.input_width2, + job.source_x2, + job.source_y2, + ), + ( + target.plane3, + job.input_width3, + job.source_x3, + job.source_y3, + ), + ] { + validate_store_plane(plane, width, x, y, job.copy_width, job.copy_height)?; + } + } + Ok(pixels) +} + +fn validate_tile_compatibility( + previous: &CudaJ2kStoreRgbaNativeJob, + current: &CudaJ2kStoreRgbaNativeJob, +) -> Result<(), CudaError> { + if previous.output_width != current.output_width + || previous.output_height != current.output_height + || previous.bit_depth0 != current.bit_depth0 + || previous.bit_depth1 != current.bit_depth1 + || previous.bit_depth2 != current.bit_depth2 + || previous.bit_depth3 != current.bit_depth3 + || previous.layout != current.layout + || previous.transform != current.transform + { + return Err(CudaError::InvalidArgument { + message: "tile stores for one exact-native RGBA output have incompatible metadata" + .to_string(), + }); + } + Ok(()) +} + +pub(super) fn validate_external( + context: &CudaContext, + destination: &CudaExternalDeviceBufferViewMut<'_>, + plan: &NativeRgbaBatchPlan, + storage: NativeRgbaStorage, +) -> Result { + if !context.is_same_context(destination.context()) { + return Err(CudaError::InvalidArgument { + message: "external exact-native RGBA destination has a different CUDA context" + .to_string(), + }); + } + if destination.byte_len() < plan.total_bytes { + return Err(CudaError::OutputTooSmall { + required: plan.total_bytes, + have: destination.byte_len(), + }); + } + if !destination + .device_ptr() + .is_multiple_of(storage.bytes_per_sample() as u64) + { + return Err(CudaError::InvalidArgument { + message: "external exact-native RGBA destination is misaligned".to_string(), + }); + } + Ok(destination.device_ptr()) +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/destination_groups.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/destination_groups.rs new file mode 100644 index 00000000..e9291172 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/destination_groups.rs @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + allocation::HostPhaseBudget, + error::CudaError, + memory::{checked_image_words, CudaDeviceBufferRange}, +}; + +use super::destination::validate_store_destination; + +#[derive(Clone, Copy, Debug)] +pub(super) struct DenseDestinationRegion { + pub(super) output_index: usize, + pub(super) output_width: u32, + pub(super) output_height: u32, + pub(super) output_x: u32, + pub(super) output_y: u32, + pub(super) copy_width: u32, + pub(super) copy_height: u32, +} + +pub(super) struct DenseDestinationPlan { + pub(super) ranges: Vec, + pub(super) item_range_indices: Vec, + pub(super) total_bytes: usize, + pub(super) requires_zero_fill: bool, +} + +pub(super) fn plan_dense_destination_regions( + regions: &[DenseDestinationRegion], + channels: usize, + bytes_per_sample: usize, + budget: &mut HostPhaseBudget, +) -> Result { + if channels == 0 || bytes_per_sample == 0 { + return Err(CudaError::InvalidArgument { + message: "dense J2K destination requires nonzero channels and sample size".to_string(), + }); + } + let mut ranges = budget.try_vec_with_capacity(regions.len())?; + let mut item_range_indices = budget.try_vec_with_capacity(regions.len())?; + let mut total_bytes = 0usize; + let mut requires_zero_fill = false; + let mut group_start = 0usize; + let mut group_area = 0usize; + let mut group_pixels = 0usize; + + for (item_index, region) in regions.iter().copied().enumerate() { + let new_group = + item_index == 0 || regions[item_index - 1].output_index != region.output_index; + if new_group { + if item_index != 0 { + requires_zero_fill |= group_area != group_pixels; + } + if region.output_index != ranges.len() { + return Err(CudaError::InvalidArgument { + message: "dense J2K destination output indices must be contiguous and grouped" + .to_string(), + }); + } + group_start = item_index; + group_area = 0; + group_pixels = checked_image_words(region.output_width, region.output_height, 1)?; + let output_samples = group_pixels + .checked_mul(channels) + .ok_or(CudaError::LengthTooLarge { len: group_pixels })?; + let output_bytes = + output_samples + .checked_mul(bytes_per_sample) + .ok_or(CudaError::LengthTooLarge { + len: output_samples, + })?; + ranges.push(CudaDeviceBufferRange { + offset: total_bytes, + len: output_bytes, + }); + total_bytes = total_bytes + .checked_add(output_bytes) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + } else { + let first = regions[group_start]; + if (region.output_width, region.output_height) + != (first.output_width, first.output_height) + { + return Err(CudaError::InvalidArgument { + message: "tile stores for one J2K output disagree on destination dimensions" + .to_string(), + }); + } + } + + validate_store_destination( + region.output_width, + region.output_height, + region.output_x, + region.output_y, + region.copy_width, + region.copy_height, + u32::try_from(channels).map_err(|_| CudaError::LengthTooLarge { len: channels })?, + )?; + for prior in ®ions[group_start..item_index] { + if rectangles_overlap(*prior, region) { + return Err(CudaError::InvalidArgument { + message: format!("tile stores for J2K output {} overlap", region.output_index), + }); + } + } + let area = checked_image_words(region.copy_width, region.copy_height, 1)?; + group_area = group_area + .checked_add(area) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if group_area > group_pixels { + return Err(CudaError::InvalidArgument { + message: format!( + "tile stores for J2K output {} exceed its destination area", + region.output_index + ), + }); + } + item_range_indices.push(region.output_index); + } + if !regions.is_empty() { + requires_zero_fill |= group_area != group_pixels; + } + + Ok(DenseDestinationPlan { + ranges, + item_range_indices, + total_bytes, + requires_zero_fill, + }) +} + +fn rectangles_overlap(a: DenseDestinationRegion, b: DenseDestinationRegion) -> bool { + if a.copy_width == 0 || a.copy_height == 0 || b.copy_width == 0 || b.copy_height == 0 { + return false; + } + let a_end_x = a.output_x + a.copy_width; + let a_end_y = a.output_y + a.copy_height; + let b_end_x = b.output_x + b.copy_width; + let b_end_y = b.output_y + b.copy_height; + a.output_x < b_end_x && b.output_x < a_end_x && a.output_y < b_end_y && b.output_y < a_end_y +} + +#[cfg(test)] +mod tests { + use super::*; + + fn region( + output_index: usize, + output_x: u32, + output_y: u32, + copy_width: u32, + copy_height: u32, + ) -> DenseDestinationRegion { + DenseDestinationRegion { + output_index, + output_width: 19, + output_height: 13, + output_x, + output_y, + copy_width, + copy_height, + } + } + + #[test] + fn four_tiles_share_one_dense_output_range() { + let regions = [ + region(0, 0, 0, 11, 7), + region(0, 11, 0, 8, 7), + region(0, 0, 7, 11, 6), + region(0, 11, 7, 8, 6), + ]; + let mut budget = HostPhaseBudget::new("dense destination test"); + let plan = plan_dense_destination_regions(®ions, 3, 2, &mut budget) + .expect("plan disjoint multi-tile destination"); + + assert_eq!(plan.ranges.len(), 1); + assert_eq!( + plan.ranges[0], + CudaDeviceBufferRange { + offset: 0, + len: 19 * 13 * 3 * 2 + } + ); + assert_eq!(plan.item_range_indices, [0, 0, 0, 0]); + assert!(!plan.requires_zero_fill); + } + + #[test] + fn overlapping_tiles_are_rejected() { + let regions = [region(0, 0, 0, 12, 13), region(0, 11, 0, 8, 13)]; + let mut budget = HostPhaseBudget::new("dense destination overlap test"); + let error = plan_dense_destination_regions(®ions, 1, 2, &mut budget) + .err() + .expect("overlap must fail"); + assert!(matches!( + error, + CudaError::InvalidArgument { message } if message.contains("overlap") + )); + } + + #[test] + fn output_indices_must_be_dense_and_grouped() { + let regions = [region(1, 0, 0, 19, 13)]; + let mut budget = HostPhaseBudget::new("dense destination index test"); + let error = plan_dense_destination_regions(®ions, 1, 1, &mut budget) + .err() + .expect("sparse output index must fail"); + assert!(matches!(error, CudaError::InvalidArgument { .. })); + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch.rs new file mode 100644 index 00000000..4337bc5f --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +mod api; +mod enqueue; +mod plan; + +use crate::{ + error::CudaError, + execution::{CudaEvent, CudaExecutionStats}, + memory::CudaDeviceBuffer, +}; + +/// Completion owner for an asynchronously enqueued grayscale final-store +/// batch. Dropping unfinished work waits before releasing uploaded job data. +#[doc(hidden)] +#[derive(Debug)] +#[must_use = "queued J2K final store must be retained or finished"] +pub struct CudaQueuedJ2kStoreBatch { + pub(super) completion: Option, + pub(super) jobs: Option, + pub(super) execution: CudaExecutionStats, +} + +impl CudaQueuedJ2kStoreBatch { + /// Query final-store completion without waiting on the host. + pub fn is_complete(&self) -> Result { + self.completion + .as_ref() + .map_or(Ok(true), CudaEvent::is_complete) + } + + /// Wait for final store completion and release retained job metadata. + pub fn finish(mut self) -> Result { + if let Some(completion) = self.completion.take() { + if let Err(error) = completion.synchronize() { + if let Some(jobs) = self.jobs.take() { + // The driver may still reference uploaded metadata. + std::mem::forget(jobs); + } + return Err(error); + } + } + self.jobs.take(); + Ok(self.execution) + } + + /// Release final-store metadata after another ordered operation has + /// already established completion of this context's default stream. + /// + /// # Safety + /// + /// The caller must prove that all default-stream work through this final + /// store has completed, for example with a later synchronous status + /// download on the same stream. + #[doc(hidden)] + pub unsafe fn release_after_stream_completion(mut self) -> CudaExecutionStats { + self.completion.take(); + self.jobs.take(); + self.execution + } + + /// Kernel dispatch counters for this queued final store. + #[must_use] + pub const fn execution(&self) -> CudaExecutionStats { + self.execution + } +} + +impl Drop for CudaQueuedJ2kStoreBatch { + fn drop(&mut self) { + let Some(completion) = self.completion.take() else { + return; + }; + if completion.synchronize().is_ok() { + self.jobs.take(); + } else if let Some(jobs) = self.jobs.take() { + // Completion is uncertain, so intentionally retain the allocation + // rather than free metadata still reachable by the driver. + std::mem::forget(jobs); + } + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/api.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/api.rs new file mode 100644 index 00000000..bbf6aacb --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/api.rs @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + plan::{ + external_destination_base, gray16_geometry, gray8_geometry, grayi16_geometry, + validate_gray_targets, + }, + CudaQueuedJ2kStoreBatch, +}; +use crate::{ + context::CudaContext, + error::CudaError, + execution::{CudaExecutionStats, CudaKernelContiguousBatchOutput}, + j2k_decode::{ + store::destination::zero_unwritten_store_output, + types::{CudaJ2kStoreGray16Target, CudaJ2kStoreGray8Target, CudaJ2kStoreGrayI16Target}, + }, + memory::{CudaDeviceBuffer, CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut}, +}; + +fn attach_zero_fill_completion( + context: &CudaContext, + mut queued: CudaQueuedJ2kStoreBatch, + zero_fill_enqueued: bool, +) -> Result { + if zero_fill_enqueued && queued.completion.is_none() { + let completion = context.create_event().and_then(|completion| { + completion.record_default_stream()?; + Ok(completion) + })?; + queued.completion = Some(completion); + } + Ok(queued) +} + +fn retire_failed_zero_fill( + context: &CudaContext, + error: CudaError, + zero_fill_enqueued: bool, +) -> CudaError { + if !zero_fill_enqueued { + return error; + } + match context.synchronize_then_error::<()>(error) { + Err(error) => error, + Ok(()) => unreachable!("synchronize_then_error always returns the primary error"), + } +} + +fn finish_owned_store( + output: CudaDeviceBuffer, + queued: CudaQueuedJ2kStoreBatch, +) -> Result<(CudaDeviceBuffer, CudaExecutionStats), CudaError> { + match queued.finish() { + Ok(execution) => Ok((output, execution)), + Err(error) => { + if error.completion_is_uncertain() { + core::mem::forget(output); + } + Err(error) + } + } +} + +impl CudaContext { + /// Store a Gray8 batch into one J2K-owned contiguous device allocation. + #[doc(hidden)] + pub fn j2k_store_gray8_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreGray8Target<'_>], + ) -> Result { + let (output, ranges, queued) = + self.j2k_store_gray8_batch_contiguous_device_enqueue(targets)?; + let (output, execution) = finish_owned_store(output, queued)?; + Ok(CudaKernelContiguousBatchOutput { + output, + ranges, + execution, + }) + } + + /// Enqueue a Gray8 batch into one J2K-owned contiguous allocation. + #[doc(hidden)] + pub fn j2k_store_gray8_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreGray8Target<'_>], + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + let plan = validate_gray_targets( + self, + targets, + 1, + |target| target.input, + |target| target.output_index, + gray8_geometry, + 0, + )?; + let output = self.allocate(plan.total_bytes)?; + let zero_fill_enqueued = plan.requires_zero_fill + && zero_unwritten_store_output(self, &output, plan.total_bytes, false)?; + let queued = self + .enqueue_gray8_batch(targets, &plan, output.device_ptr()) + .and_then(|queued| attach_zero_fill_completion(self, queued, zero_fill_enqueued)) + .map_err(|error| retire_failed_zero_fill(self, error, zero_fill_enqueued))?; + Ok((output, plan.ranges, queued)) + } + + /// Store a Gray16 batch into one J2K-owned contiguous device allocation. + #[doc(hidden)] + pub fn j2k_store_gray16_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreGray16Target<'_>], + ) -> Result { + let (output, ranges, queued) = + self.j2k_store_gray16_batch_contiguous_device_enqueue(targets)?; + let (output, execution) = finish_owned_store(output, queued)?; + Ok(CudaKernelContiguousBatchOutput { + output, + ranges, + execution, + }) + } + + /// Enqueue a Gray16 batch into one J2K-owned contiguous allocation. + #[doc(hidden)] + pub fn j2k_store_gray16_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreGray16Target<'_>], + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + let plan = validate_gray_targets( + self, + targets, + std::mem::size_of::(), + |target| target.input, + |target| target.output_index, + gray16_geometry, + 0, + )?; + let output = self.allocate(plan.total_bytes)?; + let zero_fill_enqueued = plan.requires_zero_fill + && zero_unwritten_store_output(self, &output, plan.total_bytes, false)?; + let queued = self + .enqueue_gray16_batch(targets, &plan, output.device_ptr()) + .and_then(|queued| attach_zero_fill_completion(self, queued, zero_fill_enqueued)) + .map_err(|error| retire_failed_zero_fill(self, error, zero_fill_enqueued))?; + Ok((output, plan.ranges, queued)) + } + + /// Store a signed GrayI16 batch into one J2K-owned contiguous device allocation. + #[doc(hidden)] + pub fn j2k_store_grayi16_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreGrayI16Target<'_>], + ) -> Result { + let (output, ranges, queued) = + self.j2k_store_grayi16_batch_contiguous_device_enqueue(targets)?; + let (output, execution) = finish_owned_store(output, queued)?; + Ok(CudaKernelContiguousBatchOutput { + output, + ranges, + execution, + }) + } + + /// Enqueue a signed GrayI16 batch into one J2K-owned contiguous allocation. + #[doc(hidden)] + pub fn j2k_store_grayi16_batch_contiguous_device_enqueue( + &self, + targets: &[CudaJ2kStoreGrayI16Target<'_>], + ) -> Result< + ( + CudaDeviceBuffer, + Vec, + CudaQueuedJ2kStoreBatch, + ), + CudaError, + > { + let plan = validate_gray_targets( + self, + targets, + std::mem::size_of::(), + |target| target.input, + |target| target.output_index, + grayi16_geometry, + 0, + )?; + let output = self.allocate(plan.total_bytes)?; + let zero_fill_enqueued = plan.requires_zero_fill + && zero_unwritten_store_output(self, &output, plan.total_bytes, false)?; + let queued = self + .enqueue_grayi16_batch(targets, &plan, output.device_ptr()) + .and_then(|queued| attach_zero_fill_completion(self, queued, zero_fill_enqueued)) + .map_err(|error| retire_failed_zero_fill(self, error, zero_fill_enqueued))?; + Ok((output, plan.ranges, queued)) + } + + /// Store Gray8 samples directly into a validated caller-owned CUDA range. + /// + /// # Safety + /// + /// The target inputs and destination must remain live until this method + /// returns success. If CUDA completion cannot be proven and an error is + /// returned, the caller must quarantine every referenced allocation. + #[doc(hidden)] + pub unsafe fn j2k_store_gray8_batch_into_external_device( + &self, + targets: &[CudaJ2kStoreGray8Target<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaExecutionStats), CudaError> { + // SAFETY: this wrapper retains both borrows and immediately waits for + // the returned completion guard before returning to safe Rust. + let (ranges, queued) = unsafe { + self.j2k_store_gray8_batch_into_external_device_enqueue(targets, destination)? + }; + let execution = queued.finish()?; + Ok((ranges, execution)) + } + + /// Enqueue Gray8 final store into a validated caller-owned CUDA range. + /// + /// The returned guard retains uploaded job metadata but cannot own the + /// caller's coefficient buffers or external destination. + /// + /// # Safety + /// + /// Every target input and the destination allocation must remain live and + /// unavailable for mutation or reuse until the returned guard finishes or + /// drops after confirmed CUDA completion. If completion cannot be proven, + /// the caller must quarantine those allocations rather than free them. + /// Stream ordering alone does not validate codec status; decoded pixels + /// must not be exposed as valid output until the owning codec guard has + /// successfully validated the group. + #[doc(hidden)] + pub unsafe fn j2k_store_gray8_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreGray8Target<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + let plan = validate_gray_targets( + self, + targets, + 1, + |target| target.input, + |target| target.output_index, + gray8_geometry, + 0, + )?; + let base = external_destination_base(self, destination, &plan, 1)?; + let queued = self.enqueue_gray8_batch(targets, &plan, base)?; + Ok((plan.ranges, queued)) + } + + /// Store Gray16 samples directly into a validated caller-owned CUDA range. + /// + /// # Safety + /// + /// The target inputs and destination must remain live until this method + /// returns success. If CUDA completion cannot be proven and an error is + /// returned, the caller must quarantine every referenced allocation. + #[doc(hidden)] + pub unsafe fn j2k_store_gray16_batch_into_external_device( + &self, + targets: &[CudaJ2kStoreGray16Target<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaExecutionStats), CudaError> { + // SAFETY: this wrapper retains both borrows and immediately waits for + // the returned completion guard before returning to safe Rust. + let (ranges, queued) = unsafe { + self.j2k_store_gray16_batch_into_external_device_enqueue(targets, destination)? + }; + let execution = queued.finish()?; + Ok((ranges, execution)) + } + + /// Enqueue Gray16 final store into a validated caller-owned CUDA range. + /// + /// # Safety + /// + /// Every target input and the destination allocation must remain live and + /// unavailable for mutation or reuse until the returned guard finishes or + /// drops after confirmed CUDA completion. If completion cannot be proven, + /// the caller must quarantine those allocations rather than free them. + /// Stream ordering alone does not validate codec status; decoded pixels + /// must not be exposed as valid output until the owning codec guard has + /// successfully validated the group. + #[doc(hidden)] + pub unsafe fn j2k_store_gray16_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreGray16Target<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + let plan = validate_gray_targets( + self, + targets, + std::mem::size_of::(), + |target| target.input, + |target| target.output_index, + gray16_geometry, + 0, + )?; + let base = + external_destination_base(self, destination, &plan, std::mem::align_of::())?; + let queued = self.enqueue_gray16_batch(targets, &plan, base)?; + Ok((plan.ranges, queued)) + } + + /// Store signed GrayI16 samples directly into a validated caller-owned CUDA range. + /// + /// # Safety + /// + /// The target inputs and destination must remain live until this method + /// returns success. If CUDA completion cannot be proven and an error is + /// returned, the caller must quarantine every referenced allocation. + #[doc(hidden)] + pub unsafe fn j2k_store_grayi16_batch_into_external_device( + &self, + targets: &[CudaJ2kStoreGrayI16Target<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaExecutionStats), CudaError> { + // SAFETY: this wrapper retains both borrows and immediately waits for + // the returned completion guard before returning to safe Rust. + let (ranges, queued) = unsafe { + self.j2k_store_grayi16_batch_into_external_device_enqueue(targets, destination)? + }; + let execution = queued.finish()?; + Ok((ranges, execution)) + } + + /// Enqueue signed GrayI16 final store into a validated caller-owned CUDA range. + /// + /// # Safety + /// + /// Every target input and the destination allocation must remain live and + /// unavailable for mutation or reuse until the returned guard finishes or + /// drops after confirmed CUDA completion. If completion cannot be proven, + /// the caller must quarantine those allocations rather than free them. + /// Stream ordering alone does not validate codec status; decoded pixels + /// must not be exposed as valid output until the owning codec guard has + /// successfully validated the group. + #[doc(hidden)] + pub unsafe fn j2k_store_grayi16_batch_into_external_device_enqueue( + &self, + targets: &[CudaJ2kStoreGrayI16Target<'_>], + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + ) -> Result<(Vec, CudaQueuedJ2kStoreBatch), CudaError> { + let plan = validate_gray_targets( + self, + targets, + std::mem::size_of::(), + |target| target.input, + |target| target.output_index, + grayi16_geometry, + 0, + )?; + let base = + external_destination_base(self, destination, &plan, std::mem::align_of::())?; + let queued = self.enqueue_grayi16_batch(targets, &plan, base)?; + Ok((plan.ranges, queued)) + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/enqueue.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/enqueue.rs new file mode 100644 index 00000000..834fb57c --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/enqueue.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{plan::GrayBatchPlan, CudaQueuedJ2kStoreBatch}; +use crate::{ + bytes::{ + store_gray16_batch_jobs_as_bytes, store_gray8_batch_jobs_as_bytes, + store_grayi16_batch_jobs_as_bytes, + }, + context::CudaContext, + error::CudaError, + execution::CudaExecutionStats, + j2k_decode::types::{ + CudaJ2kStoreGray16BatchJob, CudaJ2kStoreGray16Target, CudaJ2kStoreGray8BatchJob, + CudaJ2kStoreGray8Target, CudaJ2kStoreGrayI16BatchJob, CudaJ2kStoreGrayI16Target, + }, +}; + +impl CudaContext { + pub(super) fn enqueue_gray8_batch( + &self, + targets: &[CudaJ2kStoreGray8Target<'_>], + plan: &GrayBatchPlan, + output_base: u64, + ) -> Result { + if plan.active_count == 0 { + return Ok(CudaQueuedJ2kStoreBatch { + completion: None, + jobs: None, + execution: CudaExecutionStats::default(), + }); + } + let mut jobs = Vec::new(); + jobs.try_reserve_exact(plan.active_count) + .map_err(|_| CudaError::HostAllocationFailed { + bytes: plan + .active_count + .saturating_mul(std::mem::size_of::()), + })?; + for (target, item) in targets.iter().zip(&plan.items) { + if !item.active { + continue; + } + let range = plan.ranges[item.range_index]; + let offset = u64::try_from(range.offset) + .map_err(|_| CudaError::LengthTooLarge { len: range.offset })?; + jobs.push(CudaJ2kStoreGray8BatchJob { + input_ptr: target.input.device_ptr(), + output_ptr: output_base + .checked_add(offset) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?, + job: target.job, + reserved_tail: 0, + }); + } + let jobs_buffer = self.upload(store_gray8_batch_jobs_as_bytes(&jobs))?; + // SAFETY: the returned guard owns `jobs_buffer` until the recorded + // default-stream completion event has finished. + if let Err(error) = unsafe { + self.launch_j2k_store_gray8_batch_enqueue( + &jobs_buffer, + plan.max_pixels, + plan.active_count, + ) + } { + return self.synchronize_then_error(error); + } + let completion = self.create_event().and_then(|completion| { + completion.record_default_stream()?; + Ok(completion) + }); + let completion = match completion { + Ok(completion) => completion, + Err(error) => return self.synchronize_then_error(error), + }; + Ok(CudaQueuedJ2kStoreBatch { + completion: Some(completion), + jobs: Some(jobs_buffer), + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + pub(super) fn enqueue_gray16_batch( + &self, + targets: &[CudaJ2kStoreGray16Target<'_>], + plan: &GrayBatchPlan, + output_base: u64, + ) -> Result { + if plan.active_count == 0 { + return Ok(CudaQueuedJ2kStoreBatch { + completion: None, + jobs: None, + execution: CudaExecutionStats::default(), + }); + } + let mut jobs = Vec::new(); + jobs.try_reserve_exact(plan.active_count) + .map_err(|_| CudaError::HostAllocationFailed { + bytes: plan + .active_count + .saturating_mul(std::mem::size_of::()), + })?; + for (target, item) in targets.iter().zip(&plan.items) { + if !item.active { + continue; + } + let range = plan.ranges[item.range_index]; + let offset = u64::try_from(range.offset) + .map_err(|_| CudaError::LengthTooLarge { len: range.offset })?; + jobs.push(CudaJ2kStoreGray16BatchJob { + input_ptr: target.input.device_ptr(), + output_ptr: output_base + .checked_add(offset) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?, + job: target.job, + reserved_tail: 0, + }); + } + let jobs_buffer = self.upload(store_gray16_batch_jobs_as_bytes(&jobs))?; + // SAFETY: the returned guard owns `jobs_buffer` through completion. + if let Err(error) = unsafe { + self.launch_j2k_store_gray16_batch_enqueue( + &jobs_buffer, + plan.max_pixels, + plan.active_count, + ) + } { + return self.synchronize_then_error(error); + } + let completion = self.create_event().and_then(|completion| { + completion.record_default_stream()?; + Ok(completion) + }); + let completion = match completion { + Ok(completion) => completion, + Err(error) => return self.synchronize_then_error(error), + }; + Ok(CudaQueuedJ2kStoreBatch { + completion: Some(completion), + jobs: Some(jobs_buffer), + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + pub(super) fn enqueue_grayi16_batch( + &self, + targets: &[CudaJ2kStoreGrayI16Target<'_>], + plan: &GrayBatchPlan, + output_base: u64, + ) -> Result { + if plan.active_count == 0 { + return Ok(CudaQueuedJ2kStoreBatch { + completion: None, + jobs: None, + execution: CudaExecutionStats::default(), + }); + } + let mut jobs = Vec::new(); + jobs.try_reserve_exact(plan.active_count) + .map_err(|_| CudaError::HostAllocationFailed { + bytes: plan + .active_count + .saturating_mul(std::mem::size_of::()), + })?; + for (target, item) in targets.iter().zip(&plan.items) { + if !item.active { + continue; + } + let range = plan.ranges[item.range_index]; + let offset = u64::try_from(range.offset) + .map_err(|_| CudaError::LengthTooLarge { len: range.offset })?; + jobs.push(CudaJ2kStoreGrayI16BatchJob { + input_ptr: target.input.device_ptr(), + output_ptr: output_base + .checked_add(offset) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?, + job: target.job, + reserved_tail: 0, + }); + } + let jobs_buffer = self.upload(store_grayi16_batch_jobs_as_bytes(&jobs))?; + // SAFETY: the returned guard owns `jobs_buffer` through completion. + if let Err(error) = unsafe { + self.launch_j2k_store_grayi16_batch_enqueue( + &jobs_buffer, + plan.max_pixels, + plan.active_count, + ) + } { + return self.synchronize_then_error(error); + } + let completion = self.create_event().and_then(|completion| { + completion.record_default_stream()?; + Ok(completion) + }); + let completion = match completion { + Ok(completion) => completion, + Err(error) => return self.synchronize_then_error(error), + }; + Ok(CudaQueuedJ2kStoreBatch { + completion: Some(completion), + jobs: Some(jobs_buffer), + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/plan.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/plan.rs new file mode 100644 index 00000000..b5fcf045 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch/plan.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::{ + destination_groups::{plan_dense_destination_regions, DenseDestinationRegion}, + validation::{validate_store_buffer_context, validate_store_plane}, +}; +use crate::{ + allocation::HostPhaseBudget, + context::CudaContext, + error::CudaError, + j2k_decode::types::{ + CudaJ2kStoreGray16Target, CudaJ2kStoreGray8Target, CudaJ2kStoreGrayI16Target, + }, + kernels::j2k_store_batch_launch_geometry, + memory::{checked_image_words, CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut}, +}; + +pub(super) struct GrayBatchItemPlan { + pub(super) range_index: usize, + pub(super) active: bool, +} + +pub(super) struct GrayBatchPlan { + pub(super) items: Vec, + pub(super) ranges: Vec, + pub(super) total_bytes: usize, + pub(super) max_pixels: usize, + pub(super) active_count: usize, + pub(super) requires_zero_fill: bool, +} + +pub(super) fn validate_gray_targets<'a, T>( + context: &CudaContext, + targets: &'a [T], + bytes_per_sample: usize, + input: impl Fn(&'a T) -> &'a crate::memory::CudaDeviceBuffer, + output_index: impl Fn(&T) -> usize, + geometry: impl Fn(&T) -> (u32, u32, u32, u32, u32, u32, u32, u32, u32), + live_host_bytes: usize, +) -> Result { + validate_store_buffer_context(context, targets.iter().map(&input))?; + let mut budget = + HostPhaseBudget::with_live_bytes("CUDA J2K grayscale store batch plan", live_host_bytes)?; + let mut regions = budget.try_vec_with_capacity(targets.len())?; + let mut active = budget.try_vec_with_capacity(targets.len())?; + let mut max_pixels = 0usize; + let mut active_count = 0usize; + + for target in targets { + let ( + input_width, + source_x, + source_y, + copy_width, + copy_height, + output_width, + output_height, + output_x, + output_y, + ) = geometry(target); + let pixels = checked_image_words(copy_width, copy_height, 1)?; + if pixels != 0 { + validate_store_plane( + input(target), + input_width, + source_x, + source_y, + copy_width, + copy_height, + )?; + active_count = active_count + .checked_add(1) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + max_pixels = max_pixels.max(pixels); + } + regions.push(DenseDestinationRegion { + output_index: output_index(target), + output_width, + output_height, + output_x, + output_y, + copy_width, + copy_height, + }); + active.push(pixels != 0); + } + + let destinations = plan_dense_destination_regions(®ions, 1, bytes_per_sample, &mut budget)?; + let mut items = budget.try_vec_with_capacity(targets.len())?; + items.extend( + destinations + .item_range_indices + .iter() + .copied() + .zip(active) + .map(|(range_index, active)| GrayBatchItemPlan { + range_index, + active, + }), + ); + + if active_count != 0 && j2k_store_batch_launch_geometry(max_pixels, active_count).is_none() { + return Err(CudaError::LengthTooLarge { len: active_count }); + } + + Ok(GrayBatchPlan { + items, + ranges: destinations.ranges, + total_bytes: destinations.total_bytes, + max_pixels, + active_count, + requires_zero_fill: destinations.requires_zero_fill, + }) +} + +pub(super) fn gray8_geometry( + target: &CudaJ2kStoreGray8Target<'_>, +) -> (u32, u32, u32, u32, u32, u32, u32, u32, u32) { + let job = target.job; + ( + job.input_width, + job.source_x, + job.source_y, + job.copy_width, + job.copy_height, + job.output_width, + job.output_height, + job.output_x, + job.output_y, + ) +} + +pub(super) fn gray16_geometry( + target: &CudaJ2kStoreGray16Target<'_>, +) -> (u32, u32, u32, u32, u32, u32, u32, u32, u32) { + let job = target.job; + ( + job.input_width, + job.source_x, + job.source_y, + job.copy_width, + job.copy_height, + job.output_width, + job.output_height, + job.output_x, + job.output_y, + ) +} + +pub(super) fn grayi16_geometry( + target: &CudaJ2kStoreGrayI16Target<'_>, +) -> (u32, u32, u32, u32, u32, u32, u32, u32, u32) { + let job = target.job; + ( + job.input_width, + job.source_x, + job.source_y, + job.copy_width, + job.copy_height, + job.output_width, + job.output_height, + job.output_x, + job.output_y, + ) +} + +pub(super) fn external_destination_base( + context: &CudaContext, + destination: &CudaExternalDeviceBufferViewMut<'_>, + plan: &GrayBatchPlan, + alignment: usize, +) -> Result { + if !context.is_same_context(destination.context()) { + return Err(CudaError::InvalidArgument { + message: "external grayscale destination belongs to a different CUDA context" + .to_string(), + }); + } + if destination.byte_len() < plan.total_bytes { + return Err(CudaError::OutputTooSmall { + required: plan.total_bytes, + have: destination.byte_len(), + }); + } + if !destination.device_ptr().is_multiple_of(alignment as u64) { + return Err(CudaError::InvalidArgument { + message: format!("external grayscale destination is not {alignment}-byte aligned"), + }); + } + if plan.requires_zero_fill { + return Err(CudaError::InvalidArgument { + message: "external grayscale batch destination requires full output coverage" + .to_string(), + }); + } + Ok(destination.device_ptr()) +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/tests.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/tests.rs index 895b20bf..4b93dc56 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/store/tests.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/tests.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +mod color_native; mod zero_init; use super::{ diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/tests/color_native.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/tests/color_native.rs new file mode 100644 index 00000000..8a4d8d5f --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/tests/color_native.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{CudaContext, CudaJ2kStoreRgbNativeJob, CudaJ2kStoreRgbNativeTarget}; + +#[test] +fn exact_native_rgb_batch_preserves_subnative_codes_and_layout_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let plane0 = context.upload_f32(&[1.0, 127.0]).expect("plane 0"); + let plane1 = context.upload_f32(&[3.0, 126.0]).expect("plane 1"); + let plane2 = context.upload_f32(&[5.0, 125.0]).expect("plane 2"); + for (layout, expected) in [ + (0, vec![1_u8, 3, 5, 127, 126, 125]), + (1, vec![1_u8, 127, 3, 126, 5, 125]), + ] { + let output = context + .j2k_store_rgb8_native_batch_contiguous_device(&[CudaJ2kStoreRgbNativeTarget { + output_index: 0, + plane0: &plane0, + plane1: &plane1, + plane2: &plane2, + job: exact_native_rgb_job(7, layout), + }]) + .expect("exact RGB U8 batch store"); + let mut actual = vec![0_u8; expected.len()]; + output + .output() + .copy_to_host(&mut actual) + .expect("download exact RGB U8"); + assert_eq!(actual, expected); + } + + let plane0 = context.upload_f32(&[1.0, 4095.0]).expect("plane 0"); + let plane1 = context.upload_f32(&[3.0, 4094.0]).expect("plane 1"); + let plane2 = context.upload_f32(&[5.0, 4093.0]).expect("plane 2"); + for (layout, expected) in [ + (0, vec![1_u16, 3, 5, 4095, 4094, 4093]), + (1, vec![1_u16, 4095, 3, 4094, 5, 4093]), + ] { + let output = context + .j2k_store_rgb16_native_batch_contiguous_device(&[CudaJ2kStoreRgbNativeTarget { + output_index: 0, + plane0: &plane0, + plane1: &plane1, + plane2: &plane2, + job: exact_native_rgb_job(12, layout), + }]) + .expect("exact RGB U16 batch store"); + let mut bytes = vec![0_u8; expected.len() * std::mem::size_of::()]; + output + .output() + .copy_to_host(&mut bytes) + .expect("download exact RGB U16"); + let actual = bytes + .chunks_exact(2) + .map(|bytes| u16::from_ne_bytes([bytes[0], bytes[1]])) + .collect::>(); + assert_eq!(actual, expected); + } + + let plane0 = context.upload_f32(&[-2048.0, 2047.0]).expect("plane 0"); + let plane1 = context.upload_f32(&[-3.0, 2.0]).expect("plane 1"); + let plane2 = context.upload_f32(&[17.0, -19.0]).expect("plane 2"); + for (layout, expected) in [ + (0, vec![-2048_i16, -3, 17, 2047, 2, -19]), + (1, vec![-2048_i16, 2047, -3, 2, 17, -19]), + ] { + let output = context + .j2k_store_rgbi16_native_batch_contiguous_device(&[CudaJ2kStoreRgbNativeTarget { + output_index: 0, + plane0: &plane0, + plane1: &plane1, + plane2: &plane2, + job: exact_native_rgb_job(12, layout), + }]) + .expect("exact RGB I16 batch store"); + let mut bytes = vec![0_u8; expected.len() * std::mem::size_of::()]; + output + .output() + .copy_to_host(&mut bytes) + .expect("download exact RGB I16"); + let actual = bytes + .chunks_exact(2) + .map(|bytes| i16::from_ne_bytes([bytes[0], bytes[1]])) + .collect::>(); + assert_eq!(actual, expected); + } +} + +fn exact_native_rgb_job(bit_depth: u32, layout: u32) -> CudaJ2kStoreRgbNativeJob { + CudaJ2kStoreRgbNativeJob { + input_width0: 2, + input_width1: 2, + input_width2: 2, + source_x0: 0, + source_y0: 0, + source_x1: 0, + source_y1: 0, + source_x2: 0, + source_y2: 0, + copy_width: 2, + copy_height: 1, + output_width: 2, + output_height: 1, + output_x: 0, + output_y: 0, + addend0: 0.0, + addend1: 0.0, + addend2: 0.0, + bit_depth0: bit_depth, + bit_depth1: bit_depth, + bit_depth2: bit_depth, + layout, + transform: 0, + reserved: 0, + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store/validation.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store/validation.rs index 9d842442..16cba5ca 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/store/validation.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store/validation.rs @@ -3,7 +3,7 @@ use crate::{ context::{ensure_context_ownership, CudaContext}, error::CudaError, - j2k_decode::types::CudaJ2kStoreRgb8MctTarget, + j2k_decode::types::{CudaJ2kStoreRgb8MctTarget, CudaJ2kStoreRgbNativeJob}, memory::CudaDeviceBuffer, }; @@ -62,6 +62,26 @@ pub(super) fn validate_inverse_mct_planes_disjoint( ]) } +pub(super) fn validate_rgb_tile_compatibility( + previous: &CudaJ2kStoreRgbNativeJob, + current: &CudaJ2kStoreRgbNativeJob, +) -> Result<(), CudaError> { + if previous.output_width != current.output_width + || previous.output_height != current.output_height + || previous.bit_depth0 != current.bit_depth0 + || previous.bit_depth1 != current.bit_depth1 + || previous.bit_depth2 != current.bit_depth2 + || previous.layout != current.layout + || previous.transform != current.transform + { + return Err(CudaError::InvalidArgument { + message: "tile stores for one exact-native RGB output have incompatible metadata" + .to_string(), + }); + } + Ok(()) +} + pub(super) fn validate_store_plane( plane: &CudaDeviceBuffer, input_width: u32, diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store_launch.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store_launch.rs index 44f23e0a..19207f1a 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/store_launch.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store_launch.rs @@ -1,5 +1,8 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +mod color_native; +mod color_native_rgba; + use crate::{ context::CudaContext, error::CudaError, @@ -43,6 +46,48 @@ impl CudaContext { self.launch_kernel(function, geometry, &mut params) } + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_gray8_batch_enqueue( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreGray8Batch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_gray16_batch_enqueue( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreGray16Batch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_grayi16_batch_enqueue( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreGrayI16Batch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel_async(function, geometry, &mut params) + } + pub(in crate::j2k_decode) fn launch_j2k_inverse_mct( &self, plane0: &CudaDeviceBuffer, @@ -106,7 +151,7 @@ impl CudaContext { self.launch_kernel(function, geometry, &mut params) } - pub(in crate::j2k_decode) fn launch_j2k_store_rgb8_mct_batch( + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_rgb8_mct_batch_enqueue( &self, jobs: &CudaDeviceBuffer, max_pixels: usize, @@ -117,7 +162,7 @@ impl CudaContext { let mut params = cuda_kernel_params!(jobs_ptr); let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; - self.launch_kernel(function, geometry, &mut params) + self.launch_kernel_async(function, geometry, &mut params) } pub(in crate::j2k_decode) fn launch_j2k_store_rgb16_mct( @@ -150,3 +195,30 @@ impl CudaContext { .cuda_oxide_j2k_decode_store_kernel_function(kernel) } } + +#[cfg(test)] +mod tests { + #[test] + fn grayscale_batch_final_stores_enqueue_without_context_synchronization() { + let source = include_str!("store_launch.rs") + .split("#[cfg(test)]") + .next() + .expect("production store launch source"); + for name in [ + "launch_j2k_store_gray8_batch_enqueue", + "launch_j2k_store_gray16_batch_enqueue", + "launch_j2k_store_grayi16_batch_enqueue", + ] { + let function = source + .split(name) + .nth(1) + .unwrap_or_else(|| panic!("missing {name}")) + .split("\n }") + .next() + .expect("batch store function"); + assert!(function.contains("launch_kernel_async"), "{name}"); + assert!(!function.contains("launch_kernel(function"), "{name}"); + assert!(!function.contains("synchronize"), "{name}"); + } + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native.rs new file mode 100644 index 00000000..22a2b89a --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + context::CudaContext, + error::CudaError, + execution::cuda_kernel_param, + kernels::{j2k_store_batch_launch_geometry, CudaKernel}, + memory::CudaDeviceBuffer, +}; + +impl CudaContext { + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_rgb8_native_batch_enqueue( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = + self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgb8NativeBatch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_rgb16_native_batch_enqueue( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = + self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgb16NativeBatch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_rgbi16_native_batch_enqueue( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = + self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgbI16NativeBatch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel_async(function, geometry, &mut params) + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native_rgba.rs b/crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native_rgba.rs new file mode 100644 index 00000000..6d85dd8c --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native_rgba.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{ + context::CudaContext, + error::CudaError, + execution::cuda_kernel_param, + kernels::{j2k_store_batch_launch_geometry, CudaKernel}, + memory::CudaDeviceBuffer, +}; + +impl CudaContext { + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_rgba8_native_batch_enqueue( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = + self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgba8NativeBatch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_rgba16_native_batch_enqueue( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = + self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgba16NativeBatch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + pub(in crate::j2k_decode) unsafe fn launch_j2k_store_rgbai16_native_batch_enqueue( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = + self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgbaI16NativeBatch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel_async(function, geometry, &mut params) + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/structure_tests.rs b/crates/j2k-cuda-runtime/src/j2k_decode/structure_tests.rs index ca8a4bc7..97ad5ead 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/structure_tests.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/structure_tests.rs @@ -54,12 +54,37 @@ const MODULES: &[(&str, &str, usize)] = &[ include_str!("store/batch.rs"), 350, ), + ( + "j2k_decode/store/batch/external.rs", + include_str!("store/batch/external.rs"), + 175, + ), + ( + "j2k_decode/store/batch/tests.rs", + include_str!("store/batch/tests.rs"), + 50, + ), ( "j2k_decode/store/destination.rs", include_str!("store/destination.rs"), 125, ), + ( + "j2k_decode/store/color_native_batch.rs", + include_str!("store/color_native_batch.rs"), + 450, + ), + ( + "j2k_decode/store/color_native_batch/plan.rs", + include_str!("store/color_native_batch/plan.rs"), + 250, + ), ("j2k_decode/store/tests.rs", STORE_TESTS, 500), + ( + "j2k_decode/store/tests/color_native.rs", + include_str!("store/tests/color_native.rs"), + 125, + ), ( "j2k_decode/store/tests/zero_init.rs", include_str!("store/tests/zero_init.rs"), @@ -75,8 +100,23 @@ const MODULES: &[(&str, &str, usize)] = &[ include_str!("store_launch.rs"), 225, ), + ( + "j2k_decode/store_launch/color_native.rs", + include_str!("store_launch/color_native.rs"), + 75, + ), + ( + "j2k_decode/store_launch/color_native_rgba.rs", + include_str!("store_launch/color_native_rgba.rs"), + 75, + ), ("j2k_decode/trace.rs", include_str!("trace.rs"), 175), - ("j2k_decode/types.rs", include_str!("types.rs"), 375), + ("j2k_decode/types.rs", include_str!("types.rs"), 385), + ( + "j2k_decode/types/color_native.rs", + include_str!("types/color_native.rs"), + 125, + ), ( "j2k_decode/validation.rs", include_str!("validation.rs"), @@ -122,7 +162,9 @@ fn cuda_j2k_decode_uses_focused_real_modules() { "IDWT output allocation must have a public runtime preflight" ); assert!( - STORE.contains("mod batch;") && STORE.contains("mod destination;"), + STORE.contains("mod batch;") + && STORE.contains("mod color_native_batch;") + && STORE.contains("mod destination;"), "j2k_decode/store.rs must delegate batch planning and destination validation" ); assert!( @@ -130,8 +172,12 @@ fn cuda_j2k_decode_uses_focused_real_modules() { "j2k_decode/store.rs must delegate store and MCT validation" ); assert!( - STORE_TESTS.contains("mod zero_init;"), - "store tests must delegate zero-initialization coverage" + STORE_TESTS.contains("mod color_native;") && STORE_TESTS.contains("mod zero_init;"), + "store tests must delegate exact-color and zero-initialization coverage" + ); + assert!( + include_str!("store/color_native_batch.rs").contains("mod plan;"), + "exact-native RGB store must delegate target planning and validation" ); assert!(!ROOT.contains(&include_macro)); diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/types.rs b/crates/j2k-cuda-runtime/src/j2k_decode/types.rs index 3214d697..ad6dcd3d 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/types.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/types.rs @@ -1,5 +1,13 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +mod color_native; +mod color_native_rgba; + +pub(crate) use color_native::CudaJ2kStoreRgbNativeBatchJob; +pub use color_native::{CudaJ2kStoreRgbNativeJob, CudaJ2kStoreRgbNativeTarget}; +pub(crate) use color_native_rgba::CudaJ2kStoreRgbaNativeBatchJob; +pub use color_native_rgba::{CudaJ2kStoreRgbaNativeJob, CudaJ2kStoreRgbaNativeTarget}; + use crate::{driver::CuDevicePtr, memory::CudaDeviceBuffer}; /// CUDA-side integer rectangle for JPEG 2000 direct-plan kernels. @@ -124,6 +132,69 @@ pub struct CudaJ2kStoreGray16Job { pub bit_depth: u32, } +/// One Gray8 store item for a batched dispatch. +#[derive(Clone, Copy, Debug)] +#[doc(hidden)] +pub struct CudaJ2kStoreGray8Target<'a> { + /// Dense output index; consecutive tile stores may write disjoint rectangles. + pub output_index: usize, + /// Source reconstructed component plane. + pub input: &'a CudaDeviceBuffer, + /// Store geometry, level shift, and precision. + pub job: CudaJ2kStoreGray8Job, +} + +/// One Gray16 store item for a batched dispatch. +#[derive(Clone, Copy, Debug)] +#[doc(hidden)] +pub struct CudaJ2kStoreGray16Target<'a> { + /// Dense output image receiving this store. + pub output_index: usize, + /// Source reconstructed component plane. + pub input: &'a CudaDeviceBuffer, + /// Store geometry, level shift, and precision. + pub job: CudaJ2kStoreGray16Job, +} + +/// One signed GrayI16 store item for a batched dispatch. +#[derive(Clone, Copy, Debug)] +#[doc(hidden)] +pub struct CudaJ2kStoreGrayI16Target<'a> { + /// Dense output image receiving this store. + pub output_index: usize, + /// Source reconstructed component plane. + pub input: &'a CudaDeviceBuffer, + /// Store geometry, zero signed level shift, and precision. + pub job: CudaJ2kStoreGray16Job, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct CudaJ2kStoreGray8BatchJob { + pub(crate) input_ptr: CuDevicePtr, + pub(crate) output_ptr: CuDevicePtr, + pub(crate) job: CudaJ2kStoreGray8Job, + pub(crate) reserved_tail: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct CudaJ2kStoreGray16BatchJob { + pub(crate) input_ptr: CuDevicePtr, + pub(crate) output_ptr: CuDevicePtr, + pub(crate) job: CudaJ2kStoreGray16Job, + pub(crate) reserved_tail: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct CudaJ2kStoreGrayI16BatchJob { + pub(crate) input_ptr: CuDevicePtr, + pub(crate) output_ptr: CuDevicePtr, + pub(crate) job: CudaJ2kStoreGray16Job, + pub(crate) reserved_tail: u32, +} + /// In-place inverse MCT dispatch over three device f32 component planes. #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq)] diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/types/color_native.rs b/crates/j2k-cuda-runtime/src/j2k_decode/types/color_native.rs new file mode 100644 index 00000000..dea6ef58 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/types/color_native.rs @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{driver::CuDevicePtr, memory::CudaDeviceBuffer}; + +/// Exact-native RGB store geometry shared by the U8 and U16 batch kernels. +/// +/// Unlike the legacy display-oriented RGB stores, these jobs clamp to the +/// declared component precision without scaling samples to fill the storage +/// type. `layout` is zero for NHWC and one for NCHW. `transform` is zero for +/// no color transform, one for reversible RCT, and two for irreversible ICT. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +#[doc(hidden)] +pub struct CudaJ2kStoreRgbNativeJob { + /// Source width for component 0. + pub input_width0: u32, + /// Source width for component 1. + pub input_width1: u32, + /// Source width for component 2. + pub input_width2: u32, + /// Source x offset for component 0. + pub source_x0: u32, + /// Source y offset for component 0. + pub source_y0: u32, + /// Source x offset for component 1. + pub source_x1: u32, + /// Source y offset for component 1. + pub source_y1: u32, + /// Source x offset for component 2. + pub source_x2: u32, + /// Source y offset for component 2. + pub source_y2: u32, + /// Number of pixels copied per row. + pub copy_width: u32, + /// Number of rows copied. + pub copy_height: u32, + /// Destination output width in pixels. + pub output_width: u32, + /// Destination output height in rows. + pub output_height: u32, + /// Destination x offset. + pub output_x: u32, + /// Destination y offset. + pub output_y: u32, + /// Addend applied to component 0 after an optional inverse color transform. + pub addend0: f32, + /// Addend applied to component 1 after an optional inverse color transform. + pub addend1: f32, + /// Addend applied to component 2 after an optional inverse color transform. + pub addend2: f32, + /// Declared precision for component 0. + pub bit_depth0: u32, + /// Declared precision for component 1. + pub bit_depth1: u32, + /// Declared precision for component 2. + pub bit_depth2: u32, + /// Zero for NHWC, one for NCHW. + pub layout: u32, + /// Zero for none, one for reversible RCT, two for irreversible ICT. + pub transform: u32, + /// Reserved and initialized to zero. + pub reserved: u32, +} + +/// One exact-native RGB store item for a batched dispatch. +#[derive(Clone, Copy, Debug)] +#[doc(hidden)] +pub struct CudaJ2kStoreRgbNativeTarget<'a> { + /// Dense output image receiving this store. Tile stores for one image use + /// the same index and disjoint destination rectangles. + pub output_index: usize, + /// Source component plane 0. + pub plane0: &'a CudaDeviceBuffer, + /// Source component plane 1. + pub plane1: &'a CudaDeviceBuffer, + /// Source component plane 2. + pub plane2: &'a CudaDeviceBuffer, + /// Exact-native store geometry, precision, transform, and layout. + pub job: CudaJ2kStoreRgbNativeJob, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct CudaJ2kStoreRgbNativeBatchJob { + pub(crate) plane0_ptr: CuDevicePtr, + pub(crate) plane1_ptr: CuDevicePtr, + pub(crate) plane2_ptr: CuDevicePtr, + pub(crate) output_ptr: CuDevicePtr, + pub(crate) job: CudaJ2kStoreRgbNativeJob, +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/types/color_native_rgba.rs b/crates/j2k-cuda-runtime/src/j2k_decode/types/color_native_rgba.rs new file mode 100644 index 00000000..63a7af28 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode/types/color_native_rgba.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{driver::CuDevicePtr, memory::CudaDeviceBuffer}; + +/// Exact-native RGBA store geometry shared by the U8, U16, and I16 kernels. +/// +/// The optional inverse color transform applies only to planes zero through +/// two. Plane three is always stored independently so an encoded alpha channel +/// is preserved exactly. `layout` is zero for NHWC and one for NCHW. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +#[doc(hidden)] +pub struct CudaJ2kStoreRgbaNativeJob { + /// Source width for component 0. + pub input_width0: u32, + /// Source width for component 1. + pub input_width1: u32, + /// Source width for component 2. + pub input_width2: u32, + /// Source width for component 3. + pub input_width3: u32, + /// Source x offset for component 0. + pub source_x0: u32, + /// Source y offset for component 0. + pub source_y0: u32, + /// Source x offset for component 1. + pub source_x1: u32, + /// Source y offset for component 1. + pub source_y1: u32, + /// Source x offset for component 2. + pub source_x2: u32, + /// Source y offset for component 2. + pub source_y2: u32, + /// Source x offset for component 3. + pub source_x3: u32, + /// Source y offset for component 3. + pub source_y3: u32, + /// Number of pixels copied per row. + pub copy_width: u32, + /// Number of rows copied. + pub copy_height: u32, + /// Destination output width in pixels. + pub output_width: u32, + /// Destination output height in rows. + pub output_height: u32, + /// Destination x offset. + pub output_x: u32, + /// Destination y offset. + pub output_y: u32, + /// Addend for component 0 after an optional inverse color transform. + pub addend0: f32, + /// Addend for component 1 after an optional inverse color transform. + pub addend1: f32, + /// Addend for component 2 after an optional inverse color transform. + pub addend2: f32, + /// Addend for the independent alpha component. + pub addend3: f32, + /// Declared precision for component 0. + pub bit_depth0: u32, + /// Declared precision for component 1. + pub bit_depth1: u32, + /// Declared precision for component 2. + pub bit_depth2: u32, + /// Declared precision for component 3. + pub bit_depth3: u32, + /// Zero for NHWC, one for NCHW. + pub layout: u32, + /// Zero for none, one for reversible RCT, two for irreversible ICT. + pub transform: u32, + /// Reserved and initialized to zero. + pub reserved: u32, +} + +/// One exact-native RGBA store item for a batched dispatch. +#[derive(Clone, Copy, Debug)] +#[doc(hidden)] +pub struct CudaJ2kStoreRgbaNativeTarget<'a> { + /// Dense output image receiving this store. Tile stores for one image use + /// the same index and disjoint destination rectangles. + pub output_index: usize, + /// Source component plane 0. + pub plane0: &'a CudaDeviceBuffer, + /// Source component plane 1. + pub plane1: &'a CudaDeviceBuffer, + /// Source component plane 2. + pub plane2: &'a CudaDeviceBuffer, + /// Source alpha component plane. + pub plane3: &'a CudaDeviceBuffer, + /// Exact-native store geometry, precision, transform, and layout. + pub job: CudaJ2kStoreRgbaNativeJob, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct CudaJ2kStoreRgbaNativeBatchJob { + pub(crate) plane0_ptr: CuDevicePtr, + pub(crate) plane1_ptr: CuDevicePtr, + pub(crate) plane2_ptr: CuDevicePtr, + pub(crate) plane3_ptr: CuDevicePtr, + pub(crate) output_ptr: CuDevicePtr, + pub(crate) job: CudaJ2kStoreRgbaNativeJob, + pub(crate) reserved_tail: u32, +} diff --git a/crates/j2k-cuda-runtime/src/kernels.rs b/crates/j2k-cuda-runtime/src/kernels.rs index 770aab96..3a4827f0 100644 --- a/crates/j2k-cuda-runtime/src/kernels.rs +++ b/crates/j2k-cuda-runtime/src/kernels.rs @@ -169,11 +169,20 @@ pub(crate) enum CudaKernel { JpegEncodeBaselineEntropyBatch, J2kInverseMct, J2kStoreGray16, + J2kStoreGray16Batch, + J2kStoreGrayI16Batch, J2kStoreGray8, + J2kStoreGray8Batch, J2kStoreRgb16, J2kStoreRgb16Mct, J2kStoreRgb8, J2kStoreRgb8MctBatch, + J2kStoreRgb8NativeBatch, + J2kStoreRgb16NativeBatch, + J2kStoreRgbI16NativeBatch, + J2kStoreRgba8NativeBatch, + J2kStoreRgba16NativeBatch, + J2kStoreRgbaI16NativeBatch, // Coefficient-domain JPEG->HTJ2K transcode (j2k-transcode-cuda). TranscodeReversible53Idct, TranscodeReversible53VerticalLow, @@ -264,11 +273,20 @@ impl CudaKernel { Self::JpegEncodeBaselineEntropyBatch => b"j2k_jpeg_encode_baseline_entropy_batch\0", Self::J2kInverseMct => b"j2k_inverse_mct\0", Self::J2kStoreGray16 => b"j2k_store_gray16\0", + Self::J2kStoreGray16Batch => b"j2k_store_gray16_batch\0", + Self::J2kStoreGrayI16Batch => b"j2k_store_grayi16_batch\0", Self::J2kStoreGray8 => b"j2k_store_gray8\0", + Self::J2kStoreGray8Batch => b"j2k_store_gray8_batch\0", Self::J2kStoreRgb16 => b"j2k_store_rgb16\0", Self::J2kStoreRgb16Mct => b"j2k_store_rgb16_mct\0", Self::J2kStoreRgb8 => b"j2k_store_rgb8\0", Self::J2kStoreRgb8MctBatch => b"j2k_store_rgb8_mct_batch\0", + Self::J2kStoreRgb8NativeBatch => b"j2k_store_rgb8_native_batch\0", + Self::J2kStoreRgb16NativeBatch => b"j2k_store_rgb16_native_batch\0", + Self::J2kStoreRgbI16NativeBatch => b"j2k_store_rgbi16_native_batch\0", + Self::J2kStoreRgba8NativeBatch => b"j2k_store_rgba8_native_batch\0", + Self::J2kStoreRgba16NativeBatch => b"j2k_store_rgba16_native_batch\0", + Self::J2kStoreRgbaI16NativeBatch => b"j2k_store_rgbai16_native_batch\0", Self::TranscodeReversible53Idct => b"transcode_reversible53_idct\0", Self::TranscodeReversible53VerticalLow => b"transcode_reversible53_vertical_low\0", Self::TranscodeReversible53VerticalHigh => b"transcode_reversible53_vertical_high\0", diff --git a/crates/j2k-cuda-runtime/src/kernels/j2k.rs b/crates/j2k-cuda-runtime/src/kernels/j2k.rs index 83633540..e0e8b379 100644 --- a/crates/j2k-cuda-runtime/src/kernels/j2k.rs +++ b/crates/j2k-cuda-runtime/src/kernels/j2k.rs @@ -56,11 +56,20 @@ impl CudaKernel { self, Self::J2kInverseMct | Self::J2kStoreGray16 + | Self::J2kStoreGray16Batch + | Self::J2kStoreGrayI16Batch | Self::J2kStoreGray8 + | Self::J2kStoreGray8Batch | Self::J2kStoreRgb16 + | Self::J2kStoreRgb16NativeBatch + | Self::J2kStoreRgbI16NativeBatch | Self::J2kStoreRgb16Mct | Self::J2kStoreRgb8 | Self::J2kStoreRgb8MctBatch + | Self::J2kStoreRgb8NativeBatch + | Self::J2kStoreRgba8NativeBatch + | Self::J2kStoreRgba16NativeBatch + | Self::J2kStoreRgbaI16NativeBatch ) } diff --git a/crates/j2k-cuda-runtime/src/kernels/tests.rs b/crates/j2k-cuda-runtime/src/kernels/tests.rs index a7d83d38..1ee6c393 100644 --- a/crates/j2k-cuda-runtime/src/kernels/tests.rs +++ b/crates/j2k-cuda-runtime/src/kernels/tests.rs @@ -164,6 +164,94 @@ fn htj2k_cleanup_decode_geometry_packs_large_batches_into_warps() { assert_eq!(large_geometry.block(), (32, 1, 1)); } +#[test] +fn htj2k_sigprop_forward_reader_discards_a_set_stuffed_overlap_bit() { + let data = [0xFF_u8, 0x80, 0x00, 0x00, 0x00]; + let mut tmp = 0_u64; + let mut bits = 0_u32; + let mut unstuff = false; + for byte in data { + let valid_bits = 8 - u32::from(unstuff); + let next_unstuff = byte == 0xFF; + let byte = if unstuff { byte & 0x7F } else { byte }; + tmp |= u64::from(byte) << bits; + bits += valid_bits; + unstuff = next_unstuff; + } + assert_eq!( + u32::try_from(tmp).expect("low CUDA reservoir word"), + 0x0000_00FF + ); + + let device = include_str!("../cuda_oxide_htj2k_decode/simt/src/main.rs"); + let fill = device + .split("fn forward_reader_fill") + .nth(1) + .expect("CUDA HT forward-reader fill") + .split("fn forward_reader_fetch") + .next() + .expect("CUDA HT forward-reader fill body"); + assert!( + fill.contains("let byte = if reader.unstuff { byte & 0x7f } else { byte };") + || fill.contains("let byte = if reader.unstuff { byte & 0x7F } else { byte };") + ); + assert!(fill.contains("reader.unstuff = next_unstuff;")); +} + +#[test] +fn htj2k_sigprop_quad_preserves_the_next_above_stripe_context() { + let mut previous_row = [0x0000_u16, 0xA5A5]; + let new_sig = 0x0000_0088_u32; + let cleanup_sig_pair = 0xF00F_0011_u32; + let combined_sig = new_sig | (cleanup_sig_pair & 0xFFFF); + previous_row[0] = u16::try_from(combined_sig).expect("low significance half"); + assert_eq!(previous_row, [0x0099, 0xA5A5]); + + let device = include_str!("../cuda_oxide_htj2k_decode/simt/src/main.rs"); + let sigprop = device + .split("fn apply_significance_propagation") + .nth(1) + .expect("CUDA HT SigProp phase") + .split("fn apply_magnitude_refinement") + .next() + .expect("CUDA HT SigProp phase body"); + assert!(sigprop.contains("let combined_sig = new_sig | (cs & 0xffff);")); + assert!(!sigprop.contains("prev_row_sig[idx as usize + 1] =")); +} + +#[test] +fn htj2k_magref_reverse_reader_discards_a_set_stuffed_overlap_bit() { + let data = [0x00_u8, 0x00, 0x00, 0x00, 0xFF]; + let mut tmp = 0_u64; + let mut bits = 0_u32; + let mut unstuff = true; + for raw in data.into_iter().rev() { + let stuffed = unstuff && (raw & 0x7F) == 0x7F; + let valid_bits = 8 - u32::from(stuffed); + let next_unstuff = raw > 0x8F; + let byte = if stuffed { raw & 0x7F } else { raw }; + tmp |= u64::from(byte) << bits; + bits += valid_bits; + unstuff = next_unstuff; + } + assert_eq!( + u32::try_from(tmp).expect("low CUDA reservoir word"), + 0x0000_007F + ); + + let device = include_str!("../cuda_oxide_htj2k_decode/simt/src/main.rs"); + let fill = device + .split("fn reverse_reader_fill") + .nth(1) + .expect("CUDA HT reverse-reader fill") + .split("fn reverse_reader_fetch") + .next() + .expect("CUDA HT reverse-reader fill body"); + assert!(fill.contains("let stuffed = reader.unstuff && (byte & 0x7f) == 0x7f;")); + assert!(fill.contains("let byte = if stuffed { byte & 0x7f } else { byte };")); + assert!(fill.contains("reader.unstuff = next_unstuff;")); +} + #[test] fn classic_decode_geometry_and_device_stride_share_a_classic_owned_constant() { let geometry = j2k_classic_codeblock_launch_geometry(3).expect("classic geometry"); @@ -306,6 +394,12 @@ fn cuda_oxide_j2k_decode_store_kernel_metadata_matches_generated_ptx() { CudaKernel::J2kStoreGray16, CudaKernel::J2kStoreRgb8, CudaKernel::J2kStoreRgb8MctBatch, + CudaKernel::J2kStoreRgb8NativeBatch, + CudaKernel::J2kStoreRgb16NativeBatch, + CudaKernel::J2kStoreRgbI16NativeBatch, + CudaKernel::J2kStoreRgba8NativeBatch, + CudaKernel::J2kStoreRgba16NativeBatch, + CudaKernel::J2kStoreRgbaI16NativeBatch, CudaKernel::J2kStoreRgb16, CudaKernel::J2kStoreRgb16Mct, ]; diff --git a/crates/j2k-cuda-runtime/src/lib.rs b/crates/j2k-cuda-runtime/src/lib.rs index 533c812d..915214b4 100644 --- a/crates/j2k-cuda-runtime/src/lib.rs +++ b/crates/j2k-cuda-runtime/src/lib.rs @@ -74,13 +74,13 @@ mod transcode; pub use build_flags::transcode_kernels_built; pub use classic_decode::{ - CudaClassicCodeBlockJob, CudaClassicDecodeStageTimings, CudaClassicDecodeTarget, - CudaClassicSegment, CudaClassicStatus, + CudaClassicCodeBlockJob, CudaClassicDecodeStageTimings, CudaClassicDecodeTableResources, + CudaClassicDecodeTarget, CudaClassicSegment, CudaClassicStatus, CudaQueuedClassicDecode, }; #[cfg(test)] pub(crate) use context::CudaKernelName; pub use context::{ - CudaContext, CudaExternalHostOwner, CudaExternalHostReservation, + CudaContext, CudaContextDiagnostics, CudaExternalHostOwner, CudaExternalHostReservation, CudaHtj2kCompactEncodedCodeBlock, CudaHtj2kCompactEncodedCodeBlocks, }; pub use error::CudaError; @@ -91,10 +91,11 @@ pub use execution::{ CudaPooledKernelOutput, CudaQueuedExecution, }; pub use htj2k_decode::{ - CudaHtj2kCleanupTarget, CudaHtj2kCodeBlockJob, CudaHtj2kDecodeOutput, CudaHtj2kDecodeResources, - CudaHtj2kDecodeStageTimings, CudaHtj2kDecodeTableResources, CudaHtj2kDecodeTables, - CudaHtj2kDequantizeTarget, CudaHtj2kStatus, CudaPooledHtj2kDecodeOutput, - CudaQueuedHtj2kCleanup, + htj2k_cleanup_multi_descriptor_bytes, CudaHtj2kCleanupTarget, CudaHtj2kCodeBlockJob, + CudaHtj2kDecodeOutput, CudaHtj2kDecodeResources, CudaHtj2kDecodeStageTimings, + CudaHtj2kDecodeTableResources, CudaHtj2kDecodeTables, CudaHtj2kDequantizeTarget, + CudaHtj2kStatus, CudaPooledHtj2kDecodeOutput, CudaQueuedHtj2kCleanup, + CudaQueuedHtj2kCleanupGroup, }; pub use htj2k_encode::{ CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeCodeBlockRegionJob, CudaHtj2kEncodeResidentTarget, @@ -109,8 +110,11 @@ pub use htj2k_packetize::{ }; pub use j2k_decode::{ CudaJ2kIdwtJob, CudaJ2kIdwtTarget, CudaJ2kInverseMctJob, CudaJ2kRect, CudaJ2kStoreGray16Job, - CudaJ2kStoreGray8Job, CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, - CudaJ2kStoreRgb8MctJob, CudaJ2kStoreRgb8MctTarget, CudaJ2kStridedInterleavedPixels, + CudaJ2kStoreGray16Target, CudaJ2kStoreGray8Job, CudaJ2kStoreGray8Target, + CudaJ2kStoreGrayI16Target, CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, + CudaJ2kStoreRgb8MctJob, CudaJ2kStoreRgb8MctTarget, CudaJ2kStoreRgbNativeJob, + CudaJ2kStoreRgbNativeTarget, CudaJ2kStoreRgbaNativeJob, CudaJ2kStoreRgbaNativeTarget, + CudaJ2kStridedInterleavedPixels, CudaQueuedJ2kStoreBatch, }; pub use j2k_encode::{ CudaDwt53LevelShape, CudaDwt53Output, CudaDwt97BatchStageTimings, CudaDwt97Output, diff --git a/crates/j2k-cuda-runtime/src/memory.rs b/crates/j2k-cuda-runtime/src/memory.rs index 639cf41e..5a093faa 100644 --- a/crates/j2k-cuda-runtime/src/memory.rs +++ b/crates/j2k-cuda-runtime/src/memory.rs @@ -49,6 +49,7 @@ impl CudaContext { })?; crate::context::validate_device_allocation(ptr, bytes.len()) })?; + self.record_device_allocation(bytes.len()); CudaDeviceBuffer { context: self.clone(), @@ -69,6 +70,7 @@ impl CudaContext { ) }) })?; + self.record_host_to_device_copy(bytes.len()); } Ok(buffer) @@ -91,6 +93,7 @@ impl CudaContext { })?; crate::context::validate_device_allocation(ptr, len) })?; + self.record_device_allocation(len); } else { self.inner.set_current()?; } @@ -423,7 +426,9 @@ impl CudaDeviceBuffer { self.len, ) }) - }) + })?; + self.context.record_device_to_host_copy(self.len); + Ok(()) } /// Copy a byte range from this device buffer into caller-owned host output. @@ -478,7 +483,9 @@ impl CudaDeviceBuffer { byte_len, ) }) - }) + })?; + self.context.record_device_to_host_copy(byte_len); + Ok(()) } } @@ -491,7 +498,9 @@ impl Drop for CudaDeviceBuffer { let status = unsafe { (self.context.inner.driver.cu_mem_free)(self.ptr) }; self.context.inner.driver.check("cuMemFree_v2", status) }); - if free_result.is_err() { + if free_result.is_ok() { + self.context.record_device_free(self.len); + } else { // Retain the context so neither this allocation nor any // potentially in-flight work is torn down after completion // became uncertain. diff --git a/crates/j2k-cuda-runtime/src/memory/pool.rs b/crates/j2k-cuda-runtime/src/memory/pool.rs index 7e6563f7..5cd2f1b4 100644 --- a/crates/j2k-cuda-runtime/src/memory/pool.rs +++ b/crates/j2k-cuda-runtime/src/memory/pool.rs @@ -307,6 +307,7 @@ impl CudaBufferPool { .driver .check("cuMemcpyHtoD_v2", result) })?; + self.inner.context.record_host_to_device_copy(bytes.len()); } Ok(buffer) } @@ -343,6 +344,9 @@ impl CudaBufferPool { .driver .check("cuMemcpyHtoD_v2", result) }); + if upload_result.is_ok() { + self.inner.context.record_host_to_device_copy(bytes.len()); + } let recycle_result = staging.recycle(); select_pinned_upload_result(upload_result.map(|()| buffer), recycle_result) } @@ -532,6 +536,9 @@ impl CudaPooledDeviceBuffer { }; buffer.context.inner.driver.check("cuMemcpyDtoH_v2", result) })?; + buffer + .context + .record_device_to_host_copy(self.requested_len); Ok(()) } } diff --git a/crates/j2k-cuda-runtime/src/tests.rs b/crates/j2k-cuda-runtime/src/tests.rs index 9d9dca08..d41e7c85 100644 --- a/crates/j2k-cuda-runtime/src/tests.rs +++ b/crates/j2k-cuda-runtime/src/tests.rs @@ -1,134 +1,25 @@ +mod context_diagnostics; +mod context_external; +mod grayscale_external; + use super::{ checked_f32_words_byte_len, f32_slice_as_bytes_mut, format_idwt_batch_trace_row, idwt_batch_kernel_mode, idwt_batch_trace_row, idwt_batch_uses_cooperative_53, jpeg_entropy_overflow_count, pool_fit_buffer_index_by_len, validate_dct_block_grid, CudaContext, CudaDwt97BatchGeometry, CudaError, CudaExecutionStats, - CudaExternalDeviceBufferViewMut, CudaHtj2k97CodeblockBatchWithPoolRequest, - CudaHtj2kCleanupMultiKernelJob, CudaHtj2kCleanupTarget, CudaHtj2kCodeBlockJob, - CudaHtj2kDecodeTables, CudaHtj2kDequantizeTarget, CudaHtj2kEncodeCodeBlockJob, - CudaHtj2kEncodeCodeBlockRegionJob, CudaHtj2kEncodeResidentTarget, CudaHtj2kEncodeTables, - CudaJ2kIdwtBatchKernelMode, CudaJ2kIdwtJob, CudaJ2kIdwtMultiKernelJob, CudaJ2kIdwtTarget, - CudaJ2kQuantizeJob, CudaJ2kQuantizeSubbandRegionJob, CudaJ2kRect, CudaJpegChunkedEntropyConfig, + CudaHtj2k97CodeblockBatchWithPoolRequest, CudaHtj2kCleanupMultiKernelJob, + CudaHtj2kCleanupTarget, CudaHtj2kCodeBlockJob, CudaHtj2kDecodeTables, + CudaHtj2kDequantizeTarget, CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeCodeBlockRegionJob, + CudaHtj2kEncodeResidentTarget, CudaHtj2kEncodeTables, CudaJ2kIdwtBatchKernelMode, + CudaJ2kIdwtJob, CudaJ2kIdwtMultiKernelJob, CudaJ2kIdwtTarget, CudaJ2kQuantizeJob, + CudaJ2kQuantizeSubbandRegionJob, CudaJ2kRect, CudaJpegChunkedEntropyConfig, CudaJpegChunkedEntropyPlan, CudaJpegChunkedEntropyReport, CudaJpegEntropyOverflowState, CudaJpegEntropySyncState, CudaJpegHuffmanTable, CudaKernelName, CudaQueuedHtj2kCleanup, }; -#[cfg(feature = "cuda-oxide-j2k-ml")] -use super::{CudaJ2kMlKernelConfig, CudaJ2kMlLayout, CudaJ2kMlNormalization, CudaJ2kMlSample}; - fn cuda_runtime_gate() -> bool { j2k_test_support::cuda_runtime_gate(module_path!()) } -#[test] -fn cuda_context_identity_distinguishes_clones_from_independent_contexts_when_required() { - if !cuda_runtime_gate() { - return; - } - - let context = CudaContext::system_default().expect("CUDA context"); - let cloned = context.clone(); - let independent = CudaContext::system_default().expect("independent CUDA context"); - - assert!(context.is_same_context(&cloned)); - assert!(!context.is_same_context(&independent)); -} - -#[test] -fn retained_primary_context_identity_and_release_are_balanced_when_required() { - if !cuda_runtime_gate() { - return; - } - - let first = CudaContext::retain_primary(0).expect("retain primary context"); - let second = CudaContext::retain_primary(0).expect("retain primary context again"); - let owned = CudaContext::system_default().expect("independent owned context"); - - assert!(first.is_same_context(&second)); - assert!(!first.is_same_context(&owned)); - assert_eq!(first.device_ordinal(), 0); - - drop(first); - drop(second); - let retained_again = CudaContext::retain_primary(0).expect("retain primary after release"); - assert_eq!(retained_again.device_ordinal(), 0); -} - -#[test] -fn external_cuda_view_rejects_foreign_context_and_never_owns_memory_when_required() { - if !cuda_runtime_gate() { - return; - } - - let context = CudaContext::system_default().expect("CUDA context"); - let foreign = CudaContext::system_default().expect("foreign CUDA context"); - let mut allocation = context.allocate(16).expect("device allocation"); - let ptr = allocation.device_ptr(); - let len = allocation.byte_len(); - - // SAFETY: `allocation` owns the live range and is exclusively borrowed by - // the view for the duration of this scope. - let view = unsafe { - CudaExternalDeviceBufferViewMut::from_raw_parts(&context, ptr, len, 4, &mut allocation) - } - .expect("external view"); - assert_eq!(view.device_ptr(), ptr); - assert_eq!(view.byte_len(), 16); - drop(view); - - // The external view has no ownership: the original allocation remains - // live and is still responsible for freeing its memory. - assert_eq!(allocation.device_ptr(), ptr); - assert_eq!(allocation.byte_len(), 16); - - // SAFETY: the allocation remains live and exclusively borrowed, but the - // deliberately foreign context must reject its pointer identity. - let error = unsafe { - CudaExternalDeviceBufferViewMut::from_raw_parts(&foreign, ptr, len, 4, &mut allocation) - } - .expect_err("foreign context must fail"); - assert!(matches!(error, CudaError::InvalidArgument { .. })); -} - -#[cfg(feature = "cuda-oxide-j2k-ml")] -#[test] -fn j2k_ml_external_destination_checks_batch_offsets_before_launch_when_required() { - if !cuda_runtime_gate() { - return; - } - - let context = CudaContext::system_default().expect("CUDA context"); - let source = context.upload(&[1, 2, 3, 4]).expect("source upload"); - let mut allocation = context.allocate(4).expect("destination allocation"); - let ptr = allocation.device_ptr(); - let len = allocation.byte_len(); - // SAFETY: `allocation` owns this live four-byte range and the view holds - // its exclusive borrow until validation returns. - let mut destination = unsafe { - CudaExternalDeviceBufferViewMut::from_raw_parts(&context, ptr, len, 1, &mut allocation) - } - .expect("external view"); - - let error = context - .j2k_ml_convert_into_external( - source.device_ptr(), - source.byte_len(), - &mut destination, - CudaJ2kMlKernelConfig { - width: 2, - height: 2, - channels: 1, - sample: CudaJ2kMlSample::U8, - layout: CudaJ2kMlLayout::ChannelsFirst, - destination_offset_elements: 1, - normalization: CudaJ2kMlNormalization::Integer, - }, - ) - .expect_err("offset must exceed destination bounds"); - assert!(matches!(error, CudaError::OutputTooSmall { .. })); - drop(destination); - assert_eq!(allocation.device_ptr(), ptr); -} - #[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] fn cuda_transcode_kernel_gate() -> bool { if super::transcode_kernels_built() { @@ -1861,6 +1752,10 @@ fn typed_device_view_reports_element_count_when_required() { } #[test] +#[expect( + clippy::too_many_lines, + reason = "the table intentionally inventories every decode and encode kernel entry point" +)] fn kernel_module_names_cover_htj2k_decode_and_encode_stages() { let cases = [ ( @@ -1916,12 +1811,45 @@ fn kernel_module_names_cover_htj2k_decode_and_encode_stages() { (CudaKernelName::J2kIdwtVertical97, "j2k_idwt_vertical_97"), (CudaKernelName::J2kInverseMct, "j2k_inverse_mct"), (CudaKernelName::J2kStoreGray8, "j2k_store_gray8"), + (CudaKernelName::J2kStoreGray8Batch, "j2k_store_gray8_batch"), (CudaKernelName::J2kStoreGray16, "j2k_store_gray16"), + ( + CudaKernelName::J2kStoreGray16Batch, + "j2k_store_gray16_batch", + ), + ( + CudaKernelName::J2kStoreGrayI16Batch, + "j2k_store_grayi16_batch", + ), (CudaKernelName::J2kStoreRgb8, "j2k_store_rgb8"), ( CudaKernelName::J2kStoreRgb8MctBatch, "j2k_store_rgb8_mct_batch", ), + ( + CudaKernelName::J2kStoreRgb8NativeBatch, + "j2k_store_rgb8_native_batch", + ), + ( + CudaKernelName::J2kStoreRgb16NativeBatch, + "j2k_store_rgb16_native_batch", + ), + ( + CudaKernelName::J2kStoreRgbI16NativeBatch, + "j2k_store_rgbi16_native_batch", + ), + ( + CudaKernelName::J2kStoreRgba8NativeBatch, + "j2k_store_rgba8_native_batch", + ), + ( + CudaKernelName::J2kStoreRgba16NativeBatch, + "j2k_store_rgba16_native_batch", + ), + ( + CudaKernelName::J2kStoreRgbaI16NativeBatch, + "j2k_store_rgbai16_native_batch", + ), (CudaKernelName::J2kStoreRgb16, "j2k_store_rgb16"), (CudaKernelName::J2kStoreRgb16Mct, "j2k_store_rgb16_mct"), ( diff --git a/crates/j2k-cuda-runtime/src/tests/context_diagnostics.rs b/crates/j2k-cuda-runtime/src/tests/context_diagnostics.rs new file mode 100644 index 00000000..072a2973 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/tests/context_diagnostics.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::cuda_runtime_gate; +use crate::CudaContext; + +#[test] +fn runtime_diagnostics_count_device_to_host_transfers_when_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let before = context.diagnostics().expect("initial runtime diagnostics"); + + let owned = context.upload(&[1_u8, 2, 3, 4]).expect("owned upload"); + let mut owned_out = [0_u8; 4]; + owned + .copy_to_host(&mut owned_out) + .expect("owned device-to-host copy"); + + let pool = context.buffer_pool(); + let pooled = pool.upload(&[5_u8, 6, 7]).expect("pooled upload"); + let mut pooled_out = [0_u8; 3]; + pooled + .copy_to_host(&mut pooled_out) + .expect("pooled device-to-host copy"); + + let after = context.diagnostics().expect("final runtime diagnostics"); + assert_eq!( + after.device_to_host_operations - before.device_to_host_operations, + 2 + ); + assert_eq!(after.device_to_host_bytes - before.device_to_host_bytes, 7); +} + +#[test] +fn completed_cuda_events_are_reused_by_the_context_when_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let before = context.diagnostics().expect("initial runtime diagnostics"); + { + let event = context.create_event().expect("first event"); + event.record_default_stream().expect("record first event"); + event.synchronize().expect("complete first event"); + } + let after_first = context.diagnostics().expect("first event diagnostics"); + { + let event = context.create_event().expect("reused event"); + event.record_default_stream().expect("record reused event"); + event.synchronize().expect("complete reused event"); + } + let after_second = context.diagnostics().expect("second event diagnostics"); + + assert_eq!( + after_first.event_driver_allocations - before.event_driver_allocations, + 1 + ); + assert_eq!( + after_second.event_driver_allocations, + after_first.event_driver_allocations + ); + assert_eq!(after_second.event_reuses - after_first.event_reuses, 1); + assert_eq!(after_second.cached_events, 1); +} diff --git a/crates/j2k-cuda-runtime/src/tests/context_external.rs b/crates/j2k-cuda-runtime/src/tests/context_external.rs new file mode 100644 index 00000000..8ba62be5 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/tests/context_external.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::cuda_runtime_gate; +use crate::{CudaContext, CudaError, CudaExternalDeviceBufferViewMut}; +#[cfg(feature = "cuda-oxide-j2k-ml")] +use crate::{CudaJ2kMlKernelConfig, CudaJ2kMlLayout, CudaJ2kMlNormalization, CudaJ2kMlSample}; + +#[test] +fn cuda_context_identity_distinguishes_clones_from_independent_contexts_when_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let cloned = context.clone(); + let independent = CudaContext::system_default().expect("independent CUDA context"); + + assert!(context.is_same_context(&cloned)); + assert!(!context.is_same_context(&independent)); +} + +#[test] +fn retained_primary_context_identity_and_release_are_balanced_when_required() { + if !cuda_runtime_gate() { + return; + } + + let first = CudaContext::retain_primary(0).expect("retain primary context"); + let second = CudaContext::retain_primary(0).expect("retain primary context again"); + let owned = CudaContext::system_default().expect("independent owned context"); + + assert!(first.is_same_context(&second)); + assert!(!first.is_same_context(&owned)); + assert_eq!(first.device_ordinal(), 0); + + drop(first); + drop(second); + let retained_again = CudaContext::retain_primary(0).expect("retain primary after release"); + assert_eq!(retained_again.device_ordinal(), 0); +} + +#[test] +fn context_diagnostics_track_runtime_owned_allocations_and_h2d_when_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let before = context + .diagnostics() + .expect("diagnostics before allocations"); + let uploaded = context.upload(&[1_u8, 2, 3, 4]).expect("upload"); + let allocated = context.allocate(16).expect("allocation"); + let live = context + .diagnostics() + .expect("diagnostics with live buffers"); + + assert_eq!( + live.host_to_device_operations + .saturating_sub(before.host_to_device_operations), + 1 + ); + assert_eq!( + live.host_to_device_bytes + .saturating_sub(before.host_to_device_bytes), + 4 + ); + assert_eq!( + live.device_allocation_operations + .saturating_sub(before.device_allocation_operations), + 2 + ); + assert_eq!( + live.device_allocation_bytes + .saturating_sub(before.device_allocation_bytes), + 20 + ); + assert_eq!( + live.live_device_allocations + .saturating_sub(before.live_device_allocations), + 2 + ); + assert_eq!( + live.live_device_bytes + .saturating_sub(before.live_device_bytes), + 20 + ); + assert!(live.peak_live_device_allocations >= live.live_device_allocations); + assert!(live.peak_live_device_bytes >= live.live_device_bytes); + + drop(uploaded); + drop(allocated); + let released = context + .diagnostics() + .expect("diagnostics after releasing buffers"); + assert_eq!( + released.live_device_allocations, + before.live_device_allocations + ); + assert_eq!(released.live_device_bytes, before.live_device_bytes); +} + +#[cfg(all(feature = "cuda-oxide-copy-u8", j2k_cuda_oxide_copy_u8_built))] +#[test] +fn context_diagnostics_track_successful_kernel_submissions_when_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let before = context.diagnostics().expect("diagnostics before copy"); + let output = context + .copy_with_kernel(&[1_u8, 2, 3, 4]) + .expect("kernel copy"); + let after = context.diagnostics().expect("diagnostics after copy"); + + assert_eq!( + after.kernel_launches.saturating_sub(before.kernel_launches), + 1 + ); + assert_eq!( + after + .context_host_synchronizations + .saturating_sub(before.context_host_synchronizations), + 1 + ); + drop(output); +} + +#[test] +fn external_cuda_view_rejects_foreign_context_and_never_owns_memory_when_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let foreign = CudaContext::system_default().expect("foreign CUDA context"); + let mut allocation = context.allocate(16).expect("device allocation"); + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + + // SAFETY: `allocation` owns the live range and is exclusively borrowed by + // the view for the duration of this scope. + let view = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts(&context, ptr, len, 4, &mut allocation) + } + .expect("external view"); + assert_eq!(view.device_ptr(), ptr); + assert_eq!(view.byte_len(), 16); + drop(view); + + // The external view has no ownership: the original allocation remains + // live and is still responsible for freeing its memory. + assert_eq!(allocation.device_ptr(), ptr); + assert_eq!(allocation.byte_len(), 16); + + // SAFETY: the allocation remains live and exclusively borrowed, but the + // deliberately foreign context must reject its pointer identity. + let error = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts(&foreign, ptr, len, 4, &mut allocation) + } + .expect_err("foreign context must fail"); + assert!(matches!(error, CudaError::InvalidArgument { .. })); +} + +#[cfg(feature = "cuda-oxide-j2k-ml")] +#[test] +fn j2k_ml_external_destination_checks_batch_offsets_before_launch_when_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let source = context.upload(&[1, 2, 3, 4]).expect("source upload"); + let mut allocation = context.allocate(4).expect("destination allocation"); + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: `allocation` owns this live four-byte range and the view holds + // its exclusive borrow until validation returns. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts(&context, ptr, len, 1, &mut allocation) + } + .expect("external view"); + + let error = context + .j2k_ml_convert_into_external( + source.device_ptr(), + source.byte_len(), + &mut destination, + CudaJ2kMlKernelConfig { + width: 2, + height: 2, + channels: 1, + sample: CudaJ2kMlSample::U8, + layout: CudaJ2kMlLayout::ChannelsFirst, + destination_offset_elements: 1, + normalization: CudaJ2kMlNormalization::Integer, + }, + ) + .expect_err("offset must exceed destination bounds"); + assert!(matches!(error, CudaError::OutputTooSmall { .. })); + drop(destination); + assert_eq!(allocation.device_ptr(), ptr); +} diff --git a/crates/j2k-cuda-runtime/src/tests/grayscale_external.rs b/crates/j2k-cuda-runtime/src/tests/grayscale_external.rs new file mode 100644 index 00000000..a9803bd8 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/tests/grayscale_external.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::cuda_runtime_gate; +use crate::{ + CudaContext, CudaError, CudaExternalDeviceBufferViewMut, CudaJ2kStoreGray16Job, + CudaJ2kStoreGray16Target, CudaJ2kStoreGray8Job, CudaJ2kStoreGray8Target, +}; + +#[test] +fn external_grayscale_batch_store_rejects_bounds_and_device_identity_before_launch_when_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let foreign = CudaContext::system_default().expect("foreign CUDA context"); + let input = context.upload(&[0_u8; 4 * 4]).expect("f32 source bytes"); + let job = CudaJ2kStoreGray16Job { + input_width: 2, + source_x: 0, + source_y: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend: 0.0, + bit_depth: 12, + }; + let target = CudaJ2kStoreGray16Target { + output_index: 0, + input: &input, + job, + }; + + let mut too_small = context.allocate(6).expect("undersized destination"); + let ptr = too_small.device_ptr(); + // SAFETY: `too_small` owns this live allocation and remains exclusively + // borrowed until validation returns. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + &context, + ptr, + 6, + std::mem::align_of::(), + &mut too_small, + ) + } + .expect("undersized external view"); + // SAFETY: preflight rejects the undersized range before launch; both + // referenced allocations remain live for the complete call. + let error = + unsafe { context.j2k_store_gray16_batch_into_external_device(&[target], &mut destination) } + .expect_err("destination bounds must fail before launch"); + assert!(matches!(error, CudaError::OutputTooSmall { .. })); + drop(destination); + + let mut foreign_allocation = foreign.allocate(8).expect("foreign destination"); + let foreign_ptr = foreign_allocation.device_ptr(); + // SAFETY: the foreign allocation is live and exclusively borrowed; store + // validation must reject its distinct CUDA context. + let mut foreign_destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + &foreign, + foreign_ptr, + 8, + std::mem::align_of::(), + &mut foreign_allocation, + ) + } + .expect("foreign external view"); + // SAFETY: context validation rejects the foreign destination before + // launch; both referenced allocations remain live for the complete call. + let error = unsafe { + context.j2k_store_gray16_batch_into_external_device(&[target], &mut foreign_destination) + } + .expect_err("foreign destination context must fail before launch"); + assert!(matches!(error, CudaError::InvalidArgument { .. })); +} + +#[test] +fn external_grayscale_batch_store_honors_suballocation_offsets_when_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let input = context + .upload_f32(&[0.0, 1.0, 2.0, 3.0]) + .expect("grayscale source"); + let target = CudaJ2kStoreGray8Target { + output_index: 0, + input: &input, + job: CudaJ2kStoreGray8Job { + input_width: 2, + source_x: 0, + source_y: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend: 0.0, + bit_depth: 8, + }, + }; + let mut allocation = context + .upload(&[0xA5; 12]) + .expect("initialized destination allocation"); + let suballocation_ptr = allocation + .device_ptr() + .checked_add(4) + .expect("suballocation pointer"); + // SAFETY: bytes 4..8 are a live subrange of `allocation` and the owner is + // exclusively borrowed for the lifetime of this destination view. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + &context, + suballocation_ptr, + 4, + 1, + &mut allocation, + ) + } + .expect("external suballocation view"); + // SAFETY: source and destination allocations remain live and exclusively + // borrowed through the synchronous completion boundary. + let (ranges, _) = + unsafe { context.j2k_store_gray8_batch_into_external_device(&[target], &mut destination) } + .expect("direct store into suballocation"); + assert_eq!(ranges, [crate::CudaDeviceBufferRange { offset: 0, len: 4 }]); + drop(destination); + + let mut actual = [0_u8; 12]; + allocation + .copy_to_host(&mut actual) + .expect("download full allocation"); + assert_eq!(&actual[..4], &[0xA5; 4]); + assert_eq!(&actual[4..8], &[0, 1, 2, 3]); + assert_eq!(&actual[8..], &[0xA5; 4]); +} + +#[test] +fn external_batch_enqueue_api_requires_an_unsafe_lifetime_contract() { + let source = include_str!("../j2k_decode/store/grayscale_batch/api.rs"); + for name in [ + "j2k_store_gray8_batch_into_external_device", + "j2k_store_gray8_batch_into_external_device_enqueue", + "j2k_store_gray16_batch_into_external_device", + "j2k_store_gray16_batch_into_external_device_enqueue", + "j2k_store_grayi16_batch_into_external_device", + "j2k_store_grayi16_batch_into_external_device_enqueue", + ] { + assert!( + source.contains(&format!("pub unsafe fn {name}")), + "{name} must not let safe Rust detach borrowed CUDA allocations from pending work" + ); + } +} diff --git a/crates/j2k-cuda-runtime/src/tests/pipeline.rs b/crates/j2k-cuda-runtime/src/tests/pipeline.rs index 8d737347..4ad2694b 100644 --- a/crates/j2k-cuda-runtime/src/tests/pipeline.rs +++ b/crates/j2k-cuda-runtime/src/tests/pipeline.rs @@ -2,6 +2,11 @@ use super::*; +mod forward_reference; +mod native_store; + +use forward_reference::{cpu_forward_dwt53_buffer, cpu_forward_dwt97_buffer}; + #[test] fn idwt_cooperative_53_selection_requires_large_reversible_batches() { let mut kernel_job = CudaJ2kIdwtMultiKernelJob { @@ -54,7 +59,6 @@ fn idwt_cooperative_53_selection_requires_large_reversible_batches() { kernel_job.job.irreversible97 = 1; assert!(!idwt_batch_uses_cooperative_53(&[kernel_job], 128, 128)); } - #[test] fn idwt_cooperative_97_selection_requires_large_irreversible_batches() { let mut kernel_job = CudaJ2kIdwtMultiKernelJob { @@ -1537,6 +1541,8 @@ fn queued_cleanup_metadata_dequantizes_without_second_job_upload_when_runtime_re resources: vec![jobs_buffer], status_buffer: None, status_count: jobs.len(), + status_offset: 0, + uses_external_status_group: false, kernel_name: "j2k_htj2k_decode_codeblocks_multi", execution: CudaExecutionStats::default(), pool_reuse_guard: None, @@ -2114,171 +2120,3 @@ fn j2k_quantize_strided_resident_subband_matches_contiguous_when_runtime_require expected.coefficients() ); } - -fn cpu_forward_dwt53_buffer(samples: &[f32], width: usize, height: usize, levels: u8) -> Vec { - let mut buffer = samples.to_vec(); - let mut current_width = width; - let mut current_height = height; - - for _ in 0..levels { - if current_width < 2 && current_height < 2 { - break; - } - if current_height >= 2 { - let low_height = current_height.div_ceil(2); - let mut col = vec![0.0; current_height]; - for x in 0..current_width { - for y in 0..current_height { - col[y] = buffer[y * width + x]; - } - forward_lift_53(&mut col); - for y in 0..low_height { - buffer[y * width + x] = col[y * 2]; - } - for y in 0..current_height / 2 { - buffer[(low_height + y) * width + x] = col[y * 2 + 1]; - } - } - } - if current_width >= 2 { - let mut row = vec![0.0; current_width]; - for y in 0..current_height { - let row_start = y * width; - row.copy_from_slice(&buffer[row_start..row_start + current_width]); - forward_lift_53(&mut row); - let low_width = current_width.div_ceil(2); - for x in 0..low_width { - buffer[row_start + x] = row[x * 2]; - } - for x in 0..current_width / 2 { - buffer[row_start + low_width + x] = row[x * 2 + 1]; - } - } - } - current_width = current_width.div_ceil(2); - current_height = current_height.div_ceil(2); - } - - buffer -} - -fn cpu_forward_dwt97_buffer(samples: &[f32], width: usize, height: usize, levels: u8) -> Vec { - let mut buffer = samples.to_vec(); - let mut current_width = width; - let mut current_height = height; - - for _ in 0..levels { - if current_width < 2 && current_height < 2 { - break; - } - if current_height >= 2 { - let low_height = current_height.div_ceil(2); - let mut col = vec![0.0; current_height]; - for x in 0..current_width { - for y in 0..current_height { - col[y] = buffer[y * width + x]; - } - forward_lift_97(&mut col); - for y in 0..low_height { - buffer[y * width + x] = col[y * 2]; - } - for y in 0..current_height / 2 { - buffer[(low_height + y) * width + x] = col[y * 2 + 1]; - } - } - } - if current_width >= 2 { - let mut row = vec![0.0; current_width]; - for y in 0..current_height { - let row_start = y * width; - row.copy_from_slice(&buffer[row_start..row_start + current_width]); - forward_lift_97(&mut row); - let low_width = current_width.div_ceil(2); - for x in 0..low_width { - buffer[row_start + x] = row[x * 2]; - } - for x in 0..current_width / 2 { - buffer[row_start + low_width + x] = row[x * 2 + 1]; - } - } - } - current_width = current_width.div_ceil(2); - current_height = current_height.div_ceil(2); - } - - buffer -} - -fn forward_lift_53(data: &mut [f32]) { - let n = data.len(); - if n < 2 { - return; - } - - let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; - for i in (1..n).step_by(2) { - let left = data[i - 1]; - let right = if i + 1 < n { - data[i + 1] - } else { - data[last_even] - }; - data[i] -= ((left + right) * 0.5).floor(); - } - - for i in (0..n).step_by(2) { - let left = if i > 0 { data[i - 1] } else { data[1] }; - let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] += ((left + right) * 0.25 + 0.5).floor(); - } -} - -fn forward_lift_97(data: &mut [f32]) { - const ALPHA: f32 = -1.586_134_3; - const BETA: f32 = -0.052_980_117; - const GAMMA: f32 = 0.882_911_1; - const DELTA: f32 = 0.443_506_87; - const KAPPA: f32 = 1.230_174_1; - const INV_KAPPA: f32 = 1.0 / KAPPA; - - let n = data.len(); - if n < 2 { - return; - } - - let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; - for i in (1..n).step_by(2) { - let left = data[i - 1]; - let right = if i + 1 < n { - data[i + 1] - } else { - data[last_even] - }; - data[i] += ALPHA * (left + right); - } - for i in (0..n).step_by(2) { - let left = if i > 0 { data[i - 1] } else { data[1] }; - let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] += BETA * (left + right); - } - for i in (1..n).step_by(2) { - let left = data[i - 1]; - let right = if i + 1 < n { - data[i + 1] - } else { - data[last_even] - }; - data[i] += GAMMA * (left + right); - } - for i in (0..n).step_by(2) { - let left = if i > 0 { data[i - 1] } else { data[1] }; - let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] += DELTA * (left + right); - } - for i in (0..n).step_by(2) { - data[i] *= INV_KAPPA; - } - for i in (1..n).step_by(2) { - data[i] *= KAPPA; - } -} diff --git a/crates/j2k-cuda-runtime/src/tests/pipeline/forward_reference.rs b/crates/j2k-cuda-runtime/src/tests/pipeline/forward_reference.rs new file mode 100644 index 00000000..6b594314 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/tests/pipeline/forward_reference.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +pub(super) fn cpu_forward_dwt53_buffer( + samples: &[f32], + width: usize, + height: usize, + levels: u8, +) -> Vec { + let mut buffer = samples.to_vec(); + let mut current_width = width; + let mut current_height = height; + + for _ in 0..levels { + if current_width < 2 && current_height < 2 { + break; + } + if current_height >= 2 { + let low_height = current_height.div_ceil(2); + let mut col = vec![0.0; current_height]; + for x in 0..current_width { + for y in 0..current_height { + col[y] = buffer[y * width + x]; + } + forward_lift_53(&mut col); + for y in 0..low_height { + buffer[y * width + x] = col[y * 2]; + } + for y in 0..current_height / 2 { + buffer[(low_height + y) * width + x] = col[y * 2 + 1]; + } + } + } + if current_width >= 2 { + let mut row = vec![0.0; current_width]; + for y in 0..current_height { + let row_start = y * width; + row.copy_from_slice(&buffer[row_start..row_start + current_width]); + forward_lift_53(&mut row); + let low_width = current_width.div_ceil(2); + for x in 0..low_width { + buffer[row_start + x] = row[x * 2]; + } + for x in 0..current_width / 2 { + buffer[row_start + low_width + x] = row[x * 2 + 1]; + } + } + } + current_width = current_width.div_ceil(2); + current_height = current_height.div_ceil(2); + } + + buffer +} + +pub(super) fn cpu_forward_dwt97_buffer( + samples: &[f32], + width: usize, + height: usize, + levels: u8, +) -> Vec { + let mut buffer = samples.to_vec(); + let mut current_width = width; + let mut current_height = height; + + for _ in 0..levels { + if current_width < 2 && current_height < 2 { + break; + } + if current_height >= 2 { + let low_height = current_height.div_ceil(2); + let mut col = vec![0.0; current_height]; + for x in 0..current_width { + for y in 0..current_height { + col[y] = buffer[y * width + x]; + } + forward_lift_97(&mut col); + for y in 0..low_height { + buffer[y * width + x] = col[y * 2]; + } + for y in 0..current_height / 2 { + buffer[(low_height + y) * width + x] = col[y * 2 + 1]; + } + } + } + if current_width >= 2 { + let mut row = vec![0.0; current_width]; + for y in 0..current_height { + let row_start = y * width; + row.copy_from_slice(&buffer[row_start..row_start + current_width]); + forward_lift_97(&mut row); + let low_width = current_width.div_ceil(2); + for x in 0..low_width { + buffer[row_start + x] = row[x * 2]; + } + for x in 0..current_width / 2 { + buffer[row_start + low_width + x] = row[x * 2 + 1]; + } + } + } + current_width = current_width.div_ceil(2); + current_height = current_height.div_ceil(2); + } + + buffer +} + +fn forward_lift_53(data: &mut [f32]) { + let n = data.len(); + if n < 2 { + return; + } + + let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; + for i in (1..n).step_by(2) { + let left = data[i - 1]; + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] -= ((left + right) * 0.5).floor(); + } + + for i in (0..n).step_by(2) { + let left = if i > 0 { data[i - 1] } else { data[1] }; + let right = if i + 1 < n { data[i + 1] } else { left }; + data[i] += ((left + right) * 0.25 + 0.5).floor(); + } +} + +fn forward_lift_97(data: &mut [f32]) { + const ALPHA: f32 = -1.586_134_3; + const BETA: f32 = -0.052_980_117; + const GAMMA: f32 = 0.882_911_1; + const DELTA: f32 = 0.443_506_87; + const KAPPA: f32 = 1.230_174_1; + const INV_KAPPA: f32 = 1.0 / KAPPA; + + let n = data.len(); + if n < 2 { + return; + } + + let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; + for i in (1..n).step_by(2) { + let left = data[i - 1]; + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] += ALPHA * (left + right); + } + for i in (0..n).step_by(2) { + let left = if i > 0 { data[i - 1] } else { data[1] }; + let right = if i + 1 < n { data[i + 1] } else { left }; + data[i] += BETA * (left + right); + } + for i in (1..n).step_by(2) { + let left = data[i - 1]; + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] += GAMMA * (left + right); + } + for i in (0..n).step_by(2) { + let left = if i > 0 { data[i - 1] } else { data[1] }; + let right = if i + 1 < n { data[i + 1] } else { left }; + data[i] += DELTA * (left + right); + } + for i in (0..n).step_by(2) { + data[i] *= INV_KAPPA; + } + for i in (1..n).step_by(2) { + data[i] *= KAPPA; + } +} diff --git a/crates/j2k-cuda-runtime/src/tests/pipeline/native_store.rs b/crates/j2k-cuda-runtime/src/tests/pipeline/native_store.rs new file mode 100644 index 00000000..71a8b3fc --- /dev/null +++ b/crates/j2k-cuda-runtime/src/tests/pipeline/native_store.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::cuda_runtime_gate; +use crate::{ + f32_slice_as_bytes, CudaContext, CudaExternalDeviceBufferViewMut, CudaJ2kStoreGray16Job, + CudaJ2kStoreGray16Target, CudaJ2kStoreGrayI16Target, CudaJ2kStoreRgb8Job, + CudaJ2kStoreRgb8MctJob, CudaJ2kStoreRgb8MctTarget, +}; + +#[test] +fn j2k_store_rgb8_mct_batch_writes_external_suballocation_when_runtime_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let plane0 = context + .upload(f32_slice_as_bytes(&[16.0, 18.0, 21.0, 24.0])) + .expect("upload external RGB plane 0"); + let plane1 = context + .upload(f32_slice_as_bytes(&[-3.0, 4.0, 5.0, -6.0])) + .expect("upload external RGB plane 1"); + let plane2 = context + .upload(f32_slice_as_bytes(&[2.0, -1.0, 7.0, 3.0])) + .expect("upload external RGB plane 2"); + let job = CudaJ2kStoreRgb8MctJob { + store: CudaJ2kStoreRgb8Job { + input_width0: 2, + input_width1: 2, + input_width2: 2, + source_x0: 0, + source_y0: 0, + source_x1: 0, + source_y1: 0, + source_x2: 0, + source_y2: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend0: 128.0, + addend1: 128.0, + addend2: 128.0, + bit_depth0: 8, + bit_depth1: 8, + bit_depth2: 8, + rgba: 0, + }, + irreversible97: 0, + }; + let target = CudaJ2kStoreRgb8MctTarget { + plane0: &plane0, + plane1: &plane1, + plane2: &plane2, + job, + }; + let expected = context + .j2k_store_rgb8_mct_device(&plane0, &plane1, &plane2, job) + .expect("owned RGB oracle"); + let mut expected_bytes = vec![0_u8; 12]; + expected + .buffer() + .copy_to_host(&mut expected_bytes) + .expect("download owned RGB oracle"); + + let mut allocation = context + .upload(&[0xA5_u8; 16]) + .expect("external RGB destination"); + let pointer = allocation + .device_ptr() + .checked_add(2) + .expect("external RGB suballocation pointer"); + // SAFETY: bytes 2..14 are a live disjoint subrange and the allocation is + // retained until the asynchronous completion guard is finished. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + &context, + pointer, + expected_bytes.len(), + 1, + &mut allocation, + ) + } + .expect("external RGB view"); + // SAFETY: all three source planes and the external destination remain live + // and unavailable for reuse until `queued.finish()` succeeds below. + let (ranges, queued) = unsafe { + context.j2k_store_rgb8_mct_batch_into_external_device_enqueue(&[target], &mut destination) + } + .expect("enqueue external RGB store"); + assert_eq!( + ranges, + [crate::CudaDeviceBufferRange { offset: 0, len: 12 }] + ); + queued.finish().expect("finish external RGB store"); + drop(destination); + + let mut actual = [0_u8; 16]; + allocation + .copy_to_host(&mut actual) + .expect("download external RGB allocation"); + assert_eq!(&actual[..2], &[0xA5; 2]); + assert_eq!(&actual[2..14], expected_bytes); + assert_eq!(&actual[14..], &[0xA5; 2]); +} + +#[test] +fn j2k_native_grayscale_batch_store_preserves_unsigned_and_signed_samples_when_runtime_required() { + if !cuda_runtime_gate() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let unsigned = [0.0_f32, 1.0, 2047.0, 4095.0]; + let signed = [-2048.0_f32, -1.0, 0.0, 2047.0]; + let unsigned = context + .upload(f32_slice_as_bytes(&unsigned)) + .expect("upload unsigned plane"); + let signed = context + .upload(f32_slice_as_bytes(&signed)) + .expect("upload signed plane"); + let job = CudaJ2kStoreGray16Job { + input_width: 2, + source_x: 0, + source_y: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend: 0.0, + bit_depth: 12, + }; + + let unsigned_output = context + .j2k_store_gray16_batch_contiguous_device(&[CudaJ2kStoreGray16Target { + output_index: 0, + input: &unsigned, + job, + }]) + .expect("native Gray16 batch store"); + let signed_output = context + .j2k_store_grayi16_batch_contiguous_device(&[CudaJ2kStoreGrayI16Target { + output_index: 0, + input: &signed, + job, + }]) + .expect("native GrayI16 batch store"); + + let mut unsigned_bytes = vec![0_u8; 8]; + unsigned_output + .output() + .copy_to_host(&mut unsigned_bytes) + .expect("download unsigned output"); + let unsigned_samples = unsigned_bytes + .chunks_exact(2) + .map(|sample| u16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + assert_eq!(unsigned_samples, [0, 1, 2047, 4095]); + + let mut signed_bytes = vec![0_u8; 8]; + signed_output + .output() + .copy_to_host(&mut signed_bytes) + .expect("download signed output"); + let signed_samples = signed_bytes + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + assert_eq!(signed_samples, [-2048, -1, 0, 2047]); +} diff --git a/crates/j2k-cuda/src/batch.rs b/crates/j2k-cuda/src/batch.rs new file mode 100644 index 00000000..29dd1833 --- /dev/null +++ b/crates/j2k-cuda/src/batch.rs @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::{ + prepare_batch, prepare_batch_from_images, BatchDecodeOptions, BatchDecoder, BatchGroupInfo, + EncodedImage, IndexedBatchError, J2kDecodeWarning, PreparedBatch, PreparedBatchGroup, + PreparedImage, +}; +#[cfg(feature = "cuda-runtime")] +use j2k::{BatchColor, BatchLayout}; +#[cfg(feature = "cuda-runtime")] +use j2k_core::{BackendKind, PixelFormat}; +use j2k_core::{BatchInfrastructureError, Rect}; + +#[cfg(feature = "cuda-runtime")] +use std::sync::Arc; + +#[cfg(feature = "cuda-runtime")] +use crate::surface::cuda_range_storage; +use crate::{CudaSession, Error, Surface}; +#[cfg(feature = "cuda-runtime")] +use crate::{CudaSurfaceStats, SurfaceResidency}; + +mod decoder; +#[cfg(feature = "cuda-runtime")] +mod external; +#[cfg(feature = "cuda-runtime")] +mod input; +#[cfg(feature = "cuda-runtime")] +mod resident_submission; +mod types; +pub use self::decoder::CudaBatchDecoder; +#[cfg(feature = "cuda-runtime")] +pub use self::external::{ + CudaExternalBatchGroup, CudaExternalBatchTryFinish, SubmittedCudaExternalBatch, +}; +#[cfg(feature = "cuda-runtime")] +pub use self::resident_submission::SubmittedCudaResidentBatch; +#[cfg(feature = "cuda-runtime")] +pub use self::types::CudaResidentBatchBuffer; +pub use self::types::{CudaBatchDecodeResult, CudaBatchError, CudaBatchGroup, CudaBatchGroupError}; + +#[cfg(feature = "cuda-runtime")] +use self::external::SubmittedCudaCodecBatch; +#[cfg(feature = "cuda-runtime")] +use self::input::{ + decode_warnings, group_pixel_format, native_color_group_storage, native_color_inputs, + native_decode_settings, native_referenced_classic_plan, native_referenced_htj2k_plan, + validate_layout, +}; + +#[cfg(all(test, feature = "cuda-runtime"))] +mod tests { + use std::sync::Arc; + + use j2k::{ + prepare_batch, BatchAlpha, BatchCodecRoute, BatchColor, BatchDecodeOptions, BatchGroupInfo, + BatchLayout, BatchWaveletTransform, EncodedImage, NativeSampleType, + }; + use j2k_core::{Colorspace, CompressedPayloadKind, CompressedTransferSyntax, PixelFormat}; + + use super::{group_pixel_format, native_color_inputs, validate_layout}; + + #[test] + fn signed_grayscale_group_selects_native_i16_cuda_store() { + let info = BatchGroupInfo { + dimensions: (8, 8), + color: BatchColor::Gray, + alpha: BatchAlpha::None, + precision: 12, + signed: true, + sample_type: NativeSampleType::I16, + layout: BatchLayout::Nchw, + colorspace: Colorspace::Grayscale, + route: BatchCodecRoute::Htj2k, + transform: BatchWaveletTransform::Reversible53, + transfer_syntax: CompressedTransferSyntax::HtJpeg2000Lossless, + payload_kind: CompressedPayloadKind::Jpeg2000Codestream, + }; + + assert_eq!( + group_pixel_format(&info).expect("signed grayscale CUDA format"), + PixelFormat::GrayI16 + ); + } + + #[test] + fn native_rgb_batch_selects_exact_unsigned_cuda_stores_for_both_layouts() { + for (sample_type, precision, expected) in [ + (NativeSampleType::U8, 7, PixelFormat::Rgb8), + (NativeSampleType::U16, 12, PixelFormat::Rgb16), + ] { + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + let info = BatchGroupInfo { + dimensions: (8, 8), + color: BatchColor::Rgb, + alpha: BatchAlpha::None, + precision, + signed: false, + sample_type, + layout, + colorspace: Colorspace::SRgb, + route: BatchCodecRoute::Htj2k, + transform: BatchWaveletTransform::Reversible53, + transfer_syntax: CompressedTransferSyntax::HtJpeg2000Lossless, + payload_kind: CompressedPayloadKind::Jpeg2000Codestream, + }; + + assert_eq!(group_pixel_format(&info).unwrap(), expected); + validate_layout(&info).expect("exact CUDA RGB layout"); + } + } + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn classic_rgb_group_enters_the_exact_native_color_pipeline_without_an_ht_plan() { + let pixels = (0_u8..8 * 8 * 3).collect::>(); + let encoded = j2k_native::encode( + &pixels, + 8, + 8, + 3, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode classic RGB fixture"); + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(encoded))], + BatchDecodeOptions::default(), + ) + .expect("prepare classic RGB fixture"); + let [group] = prepared.groups() else { + panic!("expected one classic RGB group") + }; + assert!(group.images()[0].htj2k_plan().is_none()); + + native_color_inputs(group).expect("classic RGB exact CUDA input"); + } + + #[test] + fn native_color_batch_selects_signed_rgb_and_exact_rgba_formats() { + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + let info = BatchGroupInfo { + dimensions: (8, 8), + color: BatchColor::Rgb, + alpha: BatchAlpha::None, + precision: 12, + signed: true, + sample_type: NativeSampleType::I16, + layout, + colorspace: Colorspace::SRgb, + route: BatchCodecRoute::Htj2k, + transform: BatchWaveletTransform::Reversible53, + transfer_syntax: CompressedTransferSyntax::HtJpeg2000Lossless, + payload_kind: CompressedPayloadKind::Jpeg2000Codestream, + }; + + assert_eq!( + group_pixel_format(&info).expect("signed RGB CUDA format"), + PixelFormat::RgbI16 + ); + validate_layout(&info).expect("signed exact CUDA RGB layout"); + } + + for (sample_type, precision, expected) in [ + (NativeSampleType::U8, 8, PixelFormat::Rgba8), + (NativeSampleType::U16, 12, PixelFormat::Rgba16), + (NativeSampleType::I16, 12, PixelFormat::RgbaI16), + ] { + let info = BatchGroupInfo { + dimensions: (8, 8), + color: BatchColor::Rgba, + alpha: BatchAlpha::Straight, + precision, + signed: sample_type == NativeSampleType::I16, + sample_type, + layout: BatchLayout::Nhwc, + colorspace: Colorspace::SRgb, + route: BatchCodecRoute::Htj2k, + transform: BatchWaveletTransform::Reversible53, + transfer_syntax: CompressedTransferSyntax::HtJpeg2000Lossless, + payload_kind: CompressedPayloadKind::Jpeg2000Codestream, + }; + + assert_eq!(group_pixel_format(&info).unwrap(), expected); + validate_layout(&info).expect("exact CUDA RGBA layout"); + } + } +} diff --git a/crates/j2k-cuda/src/batch/decoder.rs b/crates/j2k-cuda/src/batch/decoder.rs new file mode 100644 index 00000000..1d2b5eaa --- /dev/null +++ b/crates/j2k-cuda/src/batch/decoder.rs @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Persistent CUDA batch decoder facade. + +#[cfg(feature = "cuda-runtime")] +use super::{ + decode_warnings, group_pixel_format, native_color_inputs, native_decode_settings, + native_referenced_classic_plan, native_referenced_htj2k_plan, validate_layout, + CudaExternalBatchGroup, PixelFormat, PreparedBatchGroup, SubmittedCudaCodecBatch, + SubmittedCudaExternalBatch, +}; +use super::{ + prepare_batch, prepare_batch_from_images, BatchDecodeOptions, BatchDecoder, + BatchInfrastructureError, CudaBatchDecodeResult, CudaBatchError, CudaSession, EncodedImage, + Error, PreparedBatch, PreparedImage, +}; + +/// Persistent CUDA batch decoder that reuses one [`CudaSession`]. +#[derive(Clone, Debug, Default)] +pub struct CudaBatchDecoder { + pub(super) session: CudaSession, + pub(super) options: BatchDecodeOptions, +} + +impl CudaBatchDecoder { + /// Create a decoder with a lazily initialized CUDA session and strict + /// shared batch options. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Create a decoder with explicit shared preparation options. + #[must_use] + pub fn with_options(options: BatchDecodeOptions) -> Self { + Self { + session: CudaSession::default(), + options, + } + } + + /// Create a decoder around an existing CUDA session. + #[must_use] + pub fn with_session(session: CudaSession) -> Self { + Self { + session, + options: BatchDecodeOptions::default(), + } + } + + /// Create a decoder around an existing session and preparation policy. + #[must_use] + pub const fn with_session_and_options( + session: CudaSession, + options: BatchDecodeOptions, + ) -> Self { + Self { session, options } + } + + /// Shared preparation options used by [`Self::prepare`] and + /// [`Self::decode_batch`]. + #[must_use] + pub const fn options(&self) -> BatchDecodeOptions { + self.options + } + + /// Borrow the persistent session for diagnostics. + #[must_use] + pub const fn session(&self) -> &CudaSession { + &self.session + } + + /// Snapshot the persistent session's two private decode buffer pools. + #[cfg(feature = "cuda-runtime")] + pub fn decode_pool_diagnostics(&self) -> Result { + self.session.decode_pool_diagnostics() + } + + /// Snapshot CUDA transfer/event counters and retained decode-pool memory. + #[cfg(feature = "cuda-runtime")] + pub fn diagnostics(&self) -> Result { + self.session.diagnostics() + } + + /// Mutably borrow the persistent session for advanced configuration. + #[must_use] + pub fn session_mut(&mut self) -> &mut CudaSession { + &mut self.session + } + + /// Inspect and group owned inputs without copying their compressed bytes. + pub fn prepare( + &self, + inputs: Vec, + ) -> Result { + prepare_batch(inputs, self.options) + } + + /// Regroup caller-supplied prepared images without reparsing codestream bytes. + /// + /// Returned source indices are positions in `images`; each image retains + /// its original [`PreparedImage::source_index`] for provenance. + pub fn prepare_prepared_images( + &self, + images: Vec, + ) -> Result { + prepare_batch_from_images(images, self.options) + } + + /// Prepare and strictly decode one owned batch to CUDA-resident groups. + pub fn decode_batch( + &mut self, + inputs: Vec, + ) -> Result { + let prepared = self.prepare(inputs)?; + self.decode_prepared(&prepared) + } + + /// Regroup and decode prepared images without reparsing their encoded bytes. + pub fn decode_prepared_images( + &mut self, + images: Vec, + ) -> Result { + let prepared = self.prepare_prepared_images(images)?; + self.decode_prepared(&prepared) + } + + /// Strictly decode a reusable shared codec batch to CUDA-resident groups. + /// + /// Explicit CUDA execution never falls back to CPU-decoded pixels. A CUDA + /// execution error discards the entire affected dense group. + pub fn decode_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result { + #[cfg(feature = "cuda-runtime")] + { + self.submit_prepared(prepared)?.wait() + } + #[cfg(not(feature = "cuda-runtime"))] + { + if let Some(group) = prepared.groups().first() { + return Err(CudaBatchError::group(group, Error::CudaUnavailable)); + } + Ok(CudaBatchDecodeResult { + groups: Vec::new(), + errors: prepared.errors().to_vec(), + group_errors: Vec::new(), + }) + } + } + + /// Decode one prepared exact-native Gray/RGB/RGBA group directly into a + /// validated caller-owned CUDA allocation range. + /// + /// The destination must belong to this decoder's CUDA context and cover + /// the tightly concatenated group output. Decoded pixels are never staged + /// through host memory or copied through an intermediate device output. + /// + /// # Safety + /// + /// The destination allocation must remain live until this method returns + /// success. If CUDA completion cannot be proven and an error is returned, + /// the caller must quarantine the allocation rather than free or reuse it. + #[cfg(feature = "cuda-runtime")] + pub unsafe fn decode_batch_into( + &mut self, + group: &PreparedBatchGroup, + destination: &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>, + ) -> Result { + // SAFETY: this synchronous convenience immediately waits while the + // caller's external view and exclusive managed-owner borrow are live. + unsafe { self.submit_batch_into(group, destination) }?.wait() + } + + /// Submit one prepared exact-native Gray/RGB/RGBA group into CUDA storage + /// without a host completion wait. + /// + /// Callers integrating another CUDA runtime must use + /// [`j2k_cuda_runtime::CudaContext::with_primary_stream_ordering`] around + /// this submission, then retain the returned value alongside the tensor so + /// codec-internal resources outlive the ordered final store. + /// + /// # Safety + /// + /// The destination allocation must remain live and may only be consumed on + /// a CUDA stream ordered after codec completion until this value is waited + /// or dropped. No unordered host or device access may overlap the decode. + /// Stream completion alone does not validate entropy status: callers must + /// not expose the destination as decoded pixels until + /// [`SubmittedCudaExternalBatch::wait`] succeeds. If CUDA completion + /// cannot be proven, the external allocation must be quarantined rather + /// than freed or reused. + #[cfg(feature = "cuda-runtime")] + pub unsafe fn submit_batch_into( + &mut self, + group: &PreparedBatchGroup, + destination: &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>, + ) -> Result { + let fmt = group_pixel_format(group.info()) + .and_then(|fmt| { + validate_layout(group.info())?; + Ok(fmt) + }) + .map_err(|source| CudaBatchError::group(group, source))?; + let pending = if matches!( + fmt, + PixelFormat::Rgb8 + | PixelFormat::Rgb16 + | PixelFormat::RgbI16 + | PixelFormat::Rgba8 + | PixelFormat::Rgba16 + | PixelFormat::RgbaI16 + ) { + let inputs = native_color_inputs(group) + .map_err(|source| CudaBatchError::group(group, source))?; + SubmittedCudaCodecBatch::Color( + crate::decoder::submit_native_color_resident_prepared_batch_into( + &inputs, + &mut self.session, + fmt, + group.info().layout, + destination, + ) + .map_err(|source| CudaBatchError::group(group, source))?, + ) + } else if matches!( + fmt, + PixelFormat::Gray8 | PixelFormat::Gray16 | PixelFormat::GrayI16 + ) { + let inputs = group + .images() + .iter() + .zip(group.source_indices().iter().copied()) + .map(|(image, source_index)| { + let referenced_plan = image + .htj2k_plan() + .map(native_referenced_htj2k_plan) + .transpose()?; + let referenced_classic_plan = image + .classic_plan() + .map(native_referenced_classic_plan) + .transpose()?; + Ok(crate::decoder::grayscale_batch::GrayscaleBatchInput { + source_index, + bytes: image.bytes().as_ref(), + device_plan: Some(image.plan()), + referenced_plan, + referenced_classic_plan, + }) + }) + .collect::, Error>>() + .map_err(|source| CudaBatchError::group(group, source))?; + SubmittedCudaCodecBatch::Grayscale( + crate::decoder::grayscale_batch::submit_grayscale_cuda_resident_prepared_batch_into( + &inputs, + native_decode_settings(group.options().settings), + &mut self.session, + fmt, + destination, + ) + .map_err(|source| CudaBatchError::group(group, source))?, + ) + } else { + return Err(CudaBatchError::group( + group, + Error::UnsupportedCudaRequest { + reason: + "direct external CUDA batch decode requires exact Gray, RGB, or RGBA output", + }, + )); + }; + let external_group = CudaExternalBatchGroup { + info: group.info().clone(), + source_indices: group.source_indices().to_vec(), + decoded_rects: group + .images() + .iter() + .map(|image| image.plan().output_rect()) + .collect(), + warnings: decode_warnings(group.options(), group.images().len()), + ranges: pending.ranges().to_vec(), + }; + Ok(SubmittedCudaExternalBatch { + group: external_group, + pending, + }) + } +} + +impl BatchDecoder for CudaBatchDecoder { + type Output = CudaBatchDecodeResult; + type Error = CudaBatchError; + + fn decode_prepared(&mut self, prepared: &PreparedBatch) -> Result { + CudaBatchDecoder::decode_prepared(self, prepared) + } + + fn options(&self) -> BatchDecodeOptions { + self.options + } +} diff --git a/crates/j2k-cuda/src/batch/external.rs b/crates/j2k-cuda/src/batch/external.rs new file mode 100644 index 00000000..3bd9de52 --- /dev/null +++ b/crates/j2k-cuda/src/batch/external.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Caller-owned CUDA destination submission and completion. + +use super::{BatchGroupInfo, CudaBatchError, Error, J2kDecodeWarning, Rect}; + +/// Metadata for a homogeneous group decoded directly into caller-owned CUDA +/// storage. +#[cfg(feature = "cuda-runtime")] +#[derive(Debug)] +pub struct CudaExternalBatchGroup { + pub(super) info: BatchGroupInfo, + pub(super) source_indices: Vec, + pub(super) decoded_rects: Vec, + pub(super) warnings: Vec>, + pub(super) ranges: Vec, +} + +/// Asynchronously submitted CUDA external-destination batch. +/// +/// The destination ranges are metadata-only until the caller stream has been +/// ordered after codec completion or [`Self::wait`] succeeds. Dropping this +/// value waits before releasing codec-internal resources. +#[cfg(feature = "cuda-runtime")] +#[must_use = "submitted CUDA decode must be retained or waited"] +pub struct SubmittedCudaExternalBatch { + pub(super) group: CudaExternalBatchGroup, + pub(super) pending: SubmittedCudaCodecBatch, +} + +#[cfg(feature = "cuda-runtime")] +pub(super) enum SubmittedCudaCodecBatch { + Grayscale(crate::decoder::grayscale_batch::SubmittedGrayscaleExternalBatch), + Color(crate::decoder::SubmittedNativeColorExternalBatch), +} + +#[cfg(feature = "cuda-runtime")] +impl SubmittedCudaCodecBatch { + pub(super) fn ranges(&self) -> &[j2k_cuda_runtime::CudaDeviceBufferRange] { + match self { + Self::Grayscale(pending) => pending.ranges(), + Self::Color(pending) => pending.ranges(), + } + } + + fn is_complete(&self) -> Result { + match self { + Self::Grayscale(pending) => pending.is_complete(), + Self::Color(pending) => pending.is_complete(), + } + } + + fn finish( + self, + ) -> Result< + ( + Vec, + crate::CudaHtj2kProfileReport, + ), + Error, + > { + match self { + Self::Grayscale(pending) => pending.finish(), + Self::Color(pending) => pending.finish(), + } + } +} + +/// Result of a nonblocking attempt to retire an external CUDA batch. +#[cfg(feature = "cuda-runtime")] +#[derive(Debug)] +#[must_use = "pending CUDA work must remain retained until it completes"] +#[expect( + clippy::large_enum_variant, + reason = "boxing would allocate on every incomplete retirement poll in the throughput path" +)] +pub enum CudaExternalBatchTryFinish { + /// GPU work is still in flight; retain this completion owner. + Pending(SubmittedCudaExternalBatch), + /// GPU work completed and codec status validation succeeded. + Complete(CudaExternalBatchGroup), +} + +#[cfg(feature = "cuda-runtime")] +impl core::fmt::Debug for SubmittedCudaExternalBatch { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("SubmittedCudaExternalBatch") + .field("group", &self.group) + .field("pending", &true) + .finish() + } +} + +#[cfg(feature = "cuda-runtime")] +impl SubmittedCudaExternalBatch { + /// Metadata and destination ranges for the submitted group. + #[must_use] + pub fn group(&self) -> &CudaExternalBatchGroup { + &self.group + } + + /// Query final-store completion without waiting on the host. + pub fn is_complete(&self) -> Result { + let source_indices = self.group().source_indices.clone(); + self.pending + .is_complete() + .map_err(|source| CudaBatchError::GroupExecution { + source_indices, + source: Box::new(source), + }) + } + + /// Retire completed work without waiting, or return the still-pending + /// completion owner unchanged. + pub fn try_finish(self) -> Result { + if self.is_complete()? { + self.wait().map(CudaExternalBatchTryFinish::Complete) + } else { + Ok(CudaExternalBatchTryFinish::Pending(self)) + } + } + + /// Wait for final-store completion and validate entropy-kernel status. + pub fn wait(self) -> Result { + let Self { mut group, pending } = self; + let source_indices = group.source_indices.clone(); + let (ranges, _report) = + pending + .finish() + .map_err(|source| CudaBatchError::GroupExecution { + source_indices, + source: Box::new(source), + })?; + group.ranges = ranges; + Ok(group) + } +} + +#[cfg(feature = "cuda-runtime")] +impl CudaExternalBatchGroup { + /// Shared decoded dimensions, type, color, transform, route, and layout. + #[must_use] + pub const fn info(&self) -> &BatchGroupInfo { + &self.info + } + + /// Original input indices in destination batch order. + #[must_use] + pub fn source_indices(&self) -> &[usize] { + &self.source_indices + } + + /// Actual decoded rectangle for each image. + #[must_use] + pub fn decoded_rects(&self) -> &[Rect] { + &self.decoded_rects + } + + /// Non-fatal codec warnings for each image. + #[must_use] + pub fn warnings(&self) -> &[Vec] { + &self.warnings + } + + /// Validated byte ranges written inside the caller allocation. + #[must_use] + pub fn ranges(&self) -> &[j2k_cuda_runtime::CudaDeviceBufferRange] { + &self.ranges + } +} diff --git a/crates/j2k-cuda/src/batch/input.rs b/crates/j2k-cuda/src/batch/input.rs new file mode 100644 index 00000000..41edba47 --- /dev/null +++ b/crates/j2k-cuda/src/batch/input.rs @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Prepared-input validation and CUDA adapter mapping. + +#[cfg(feature = "cuda-runtime")] +use super::{ + cuda_range_storage, Arc, BackendKind, CudaResidentBatchBuffer, CudaSurfaceStats, + PreparedBatchGroup, Surface, SurfaceResidency, +}; +use super::{ + BatchColor, BatchDecodeOptions, BatchGroupInfo, BatchLayout, Error, J2kDecodeWarning, + PixelFormat, +}; + +pub(super) fn group_pixel_format(info: &BatchGroupInfo) -> Result { + info.native_pixel_format() + .ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA batch output color/sample type is unsupported", + }) +} + +pub(super) fn validate_layout(info: &BatchGroupInfo) -> Result<(), Error> { + if !matches!(info.layout, BatchLayout::Nchw | BatchLayout::Nhwc) { + return Err(Error::UnsupportedCudaRequest { + reason: "CUDA batch output layout is unsupported", + }); + } + Ok(()) +} + +#[cfg(feature = "cuda-runtime")] +pub(super) fn native_decode_settings(settings: j2k::DecodeSettings) -> j2k_native::DecodeSettings { + j2k_native::DecodeSettings { + resolve_palette_indices: true, + strict: settings.is_strict(), + target_resolution: None, + } +} + +#[cfg(feature = "cuda-runtime")] +pub(super) fn native_referenced_htj2k_plan( + plan: &j2k::PreparedHtj2kPlan, +) -> Result<&j2k_native::J2kReferencedHtj2kPlan, Error> { + plan.adapter_view() + .downcast_ref::() + .ok_or(Error::UnsupportedCudaRequest { + reason: "prepared HTJ2K plan is not compatible with the CUDA adapter", + }) +} + +#[cfg(feature = "cuda-runtime")] +pub(super) fn native_referenced_classic_plan( + plan: &j2k::PreparedClassicPlan, +) -> Result<&j2k_native::J2kReferencedClassicPlan, Error> { + plan.adapter_view() + .downcast_ref::() + .ok_or(Error::UnsupportedCudaRequest { + reason: "prepared classic plan is not compatible with the CUDA adapter", + }) +} + +#[cfg(feature = "cuda-runtime")] +pub(super) fn native_color_inputs( + group: &PreparedBatchGroup, +) -> Result>, Error> { + group + .images() + .iter() + .zip(group.source_indices().iter().copied()) + .map(|(image, source_index)| { + let referenced_plan = match image.htj2k_plan() { + Some(prepared_plan) + if group.info().color == BatchColor::Rgb && prepared_plan.is_color() => + { + Some(native_referenced_htj2k_plan(prepared_plan)?) + } + Some(prepared_plan) + if group.info().color == BatchColor::Rgba && prepared_plan.is_rgba() => + { + Some(native_referenced_htj2k_plan(prepared_plan)?) + } + Some(_) => { + return Err(Error::UnsupportedCudaRequest { + reason: "exact CUDA color batch received incompatible prepared geometry", + }) + } + None => None, + }; + let referenced_classic_plan = match image.classic_plan() { + Some(prepared_plan) + if group.info().color == BatchColor::Rgb && prepared_plan.is_color() => + { + Some(native_referenced_classic_plan(prepared_plan)?) + } + Some(prepared_plan) + if group.info().color == BatchColor::Rgba && prepared_plan.is_rgba() => + { + Some(native_referenced_classic_plan(prepared_plan)?) + } + Some(_) => { + return Err(Error::UnsupportedCudaRequest { + reason: "exact CUDA color batch received incompatible classic geometry", + }); + } + None => None, + }; + if referenced_plan.is_none() && referenced_classic_plan.is_none() { + return Err(Error::UnsupportedCudaRequest { + reason: "exact CUDA color batch requires a supported prepared device plan", + }); + } + Ok(crate::decoder::NativeColorBatchInput { + source_index, + bytes: image.bytes().as_ref(), + device_plan: image.plan(), + referenced_plan, + referenced_classic_plan, + settings: native_decode_settings(group.options().settings), + }) + }) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +pub(super) fn native_color_group_storage( + info: &BatchGroupInfo, + fmt: PixelFormat, + output: crate::decoder::NativeColorOwnedBatch, +) -> (Vec, CudaResidentBatchBuffer) { + let crate::decoder::NativeColorOwnedBatch { + buffer, + ranges, + execution, + } = output; + let shared = Arc::new(buffer); + let surfaces = if info.layout == BatchLayout::Nhwc { + ranges + .iter() + .map(|range| Surface { + backend: BackendKind::Cuda, + residency: SurfaceResidency::CudaResidentDecode, + dimensions: info.dimensions, + fmt, + pitch_bytes: info.dimensions.0 as usize * fmt.bytes_per_pixel(), + stats: CudaSurfaceStats { + total: execution.kernel_dispatches(), + copy: execution.copy_kernel_dispatches(), + decode: execution.decode_kernel_dispatches(), + }, + storage: cuda_range_storage(shared.clone(), range.offset, range.len), + }) + .collect() + } else { + Vec::new() + }; + ( + surfaces, + CudaResidentBatchBuffer { + buffer: shared, + ranges, + }, + ) +} + +pub(super) fn decode_warnings( + options: BatchDecodeOptions, + count: usize, +) -> Vec> { + (0..count) + .map(|_| { + if options.settings.lenient_tolerance_enabled() { + vec![J2kDecodeWarning::LenientDecodeMode] + } else { + Vec::new() + } + }) + .collect() +} diff --git a/crates/j2k-cuda/src/batch/resident_submission.rs b/crates/j2k-cuda/src/batch/resident_submission.rs new file mode 100644 index 00000000..61889263 --- /dev/null +++ b/crates/j2k-cuda/src/batch/resident_submission.rs @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::{ + EncodedImage, IndexedBatchError, J2kDecodeWarning, PreparedBatch, PreparedBatchGroup, Rect, +}; +use j2k_core::{BatchInfrastructureError, PixelFormat}; + +use super::{ + decode_warnings, group_pixel_format, native_color_group_storage, native_color_inputs, + native_decode_settings, native_referenced_classic_plan, native_referenced_htj2k_plan, + validate_layout, CudaBatchDecodeResult, CudaBatchDecoder, CudaBatchError, CudaBatchGroup, + CudaBatchGroupError, CudaResidentBatchBuffer, Error, Surface, +}; + +struct ResidentGroupMetadata { + info: j2k::BatchGroupInfo, + source_indices: Vec, + decoded_rects: Vec, + warnings: Vec>, +} + +impl ResidentGroupMetadata { + fn from_prepared(group: &PreparedBatchGroup) -> Self { + Self { + info: group.info().clone(), + source_indices: group.source_indices().to_vec(), + decoded_rects: group + .images() + .iter() + .map(|image| image.plan().output_rect()) + .collect(), + warnings: decode_warnings(group.options(), group.images().len()), + } + } + + fn finish( + self, + surfaces: Vec, + dense_output: CudaResidentBatchBuffer, + ) -> CudaBatchGroup { + CudaBatchGroup { + info: self.info, + source_indices: self.source_indices, + decoded_rects: self.decoded_rects, + warnings: self.warnings, + surfaces, + dense_output, + } + } +} + +enum SubmittedResidentCodecGroup { + Grayscale(crate::decoder::grayscale_batch::SubmittedGrayscaleResidentBatch), + Color(crate::decoder::SubmittedNativeColorResidentBatch), +} + +impl SubmittedResidentCodecGroup { + fn is_complete(&self) -> Result { + match self { + Self::Grayscale(pending) => pending.is_complete(), + Self::Color(pending) => pending.is_complete(), + } + } + + fn finish( + self, + info: &j2k::BatchGroupInfo, + ) -> Result<(Vec, CudaResidentBatchBuffer), Error> { + match self { + Self::Grayscale(pending) => { + let (output, _report) = pending.finish()?; + Ok(( + output.surfaces, + CudaResidentBatchBuffer { + buffer: output.buffer, + ranges: output.ranges, + }, + )) + } + Self::Color(pending) => { + let (output, _report) = pending.finish()?; + let fmt = group_pixel_format(info)?; + Ok(native_color_group_storage(info, fmt, output)) + } + } + } +} + +struct SubmittedResidentGroup { + metadata: ResidentGroupMetadata, + pending: SubmittedResidentCodecGroup, +} + +/// Asynchronously submitted codec-owned CUDA resident batch. +/// +/// Every homogeneous group owns its final CUDA allocation before submission. +/// [`Self::wait`] exposes those allocations only after final-store completion +/// and entropy-status validation. Dropping this guard safely retires all work. +#[must_use = "submitted CUDA resident decode must be retained or waited"] +pub struct SubmittedCudaResidentBatch { + pending: Vec, + errors: Vec, + group_errors: Vec, +} + +impl core::fmt::Debug for SubmittedCudaResidentBatch { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("SubmittedCudaResidentBatch") + .field("pending_groups", &self.pending.len()) + .field("errors", &self.errors) + .field("group_errors", &self.group_errors) + .finish() + } +} + +impl SubmittedCudaResidentBatch { + /// Number of successfully submitted homogeneous groups still retained. + #[must_use] + pub fn pending_group_count(&self) -> usize { + self.pending.len() + } + + /// Query whether every successfully submitted group has completed. + pub fn is_complete(&self) -> Result { + for group in &self.pending { + let source_indices = group.metadata.source_indices.clone(); + if !group + .pending + .is_complete() + .map_err(|source| CudaBatchError::GroupExecution { + source_indices, + source: Box::new(source), + })? + { + return Ok(false); + } + } + Ok(true) + } + + /// Wait once for all group final stores and return codec-owned outputs. + pub fn wait(mut self) -> Result { + let mut groups = Vec::new(); + groups.try_reserve_exact(self.pending.len()).map_err(|_| { + BatchInfrastructureError::HostAllocationFailed { + what: "completed CUDA resident batch groups", + bytes: self + .pending + .len() + .saturating_mul(core::mem::size_of::()), + } + })?; + let mut fatal = None; + for submitted in self.pending.drain(..) { + let SubmittedResidentGroup { metadata, pending } = submitted; + let source_indices = metadata.source_indices.clone(); + match pending.finish(&metadata.info) { + Ok((surfaces, dense_output)) => { + groups.push(metadata.finish(surfaces, dense_output)); + } + Err(source) if source.session_is_unusable() => { + if fatal.is_none() { + fatal = Some(CudaBatchError::GroupExecution { + source_indices, + source: Box::new(source), + }); + } + } + Err(source) => self + .group_errors + .push(CudaBatchGroupError::from_parts(source_indices, source)), + } + } + if let Some(error) = fatal { + return Err(error); + } + Ok(CudaBatchDecodeResult { + groups, + errors: core::mem::take(&mut self.errors), + group_errors: core::mem::take(&mut self.group_errors), + }) + } +} + +impl CudaBatchDecoder { + /// Prepare and asynchronously submit a codec-owned CUDA resident batch. + pub fn submit_batch( + &mut self, + inputs: Vec, + ) -> Result { + let prepared = self.prepare(inputs)?; + self.submit_prepared(&prepared) + } + + /// Asynchronously submit reusable prepared inputs to codec-owned CUDA output. + /// + /// Recoverable execution failures remain group-local and do not suppress + /// later groups. No decoded host staging or final device copy is used. + pub fn submit_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result { + let mut pending = Vec::new(); + pending + .try_reserve_exact(prepared.groups().len()) + .map_err(|_| BatchInfrastructureError::HostAllocationFailed { + what: "submitted CUDA resident groups", + bytes: prepared + .groups() + .len() + .saturating_mul(core::mem::size_of::()), + })?; + let mut group_errors = Vec::new(); + group_errors + .try_reserve_exact(prepared.groups().len()) + .map_err(|_| BatchInfrastructureError::HostAllocationFailed { + what: "submitted CUDA resident group failures", + bytes: prepared + .groups() + .len() + .saturating_mul(core::mem::size_of::()), + })?; + for group in prepared.groups() { + match self.submit_resident_group(group) { + Ok(submitted) => pending.push(submitted), + Err(source) if source.session_is_unusable() => { + return Err(CudaBatchError::group(group, source)); + } + Err(source) => group_errors.push(CudaBatchGroupError::new(group, source)), + } + } + Ok(SubmittedCudaResidentBatch { + pending, + errors: prepared.errors().to_vec(), + group_errors, + }) + } + + fn submit_resident_group( + &mut self, + group: &PreparedBatchGroup, + ) -> Result { + let fmt = group_pixel_format(group.info())?; + validate_layout(group.info())?; + let pending = if matches!( + fmt, + PixelFormat::Rgb8 + | PixelFormat::Rgb16 + | PixelFormat::RgbI16 + | PixelFormat::Rgba8 + | PixelFormat::Rgba16 + | PixelFormat::RgbaI16 + ) { + let inputs = native_color_inputs(group)?; + SubmittedResidentCodecGroup::Color( + crate::decoder::submit_native_color_resident_prepared_batch( + &inputs, + &mut self.session, + fmt, + group.info().layout, + )?, + ) + } else if matches!( + fmt, + PixelFormat::Gray8 | PixelFormat::Gray16 | PixelFormat::GrayI16 + ) { + let inputs = resident_grayscale_inputs(group)?; + SubmittedResidentCodecGroup::Grayscale( + crate::decoder::grayscale_batch::submit_grayscale_cuda_resident_prepared_batch( + &inputs, + native_decode_settings(group.options().settings), + &mut self.session, + fmt, + )?, + ) + } else { + return Err(Error::UnsupportedCudaRequest { + reason: "resident CUDA batch submission requires exact Gray, RGB, or RGBA output", + }); + }; + Ok(SubmittedResidentGroup { + metadata: ResidentGroupMetadata::from_prepared(group), + pending, + }) + } +} + +fn resident_grayscale_inputs( + group: &PreparedBatchGroup, +) -> Result>, Error> { + group + .images() + .iter() + .zip(group.source_indices().iter().copied()) + .map(|(image, source_index)| { + let referenced_plan = image + .htj2k_plan() + .map(native_referenced_htj2k_plan) + .transpose()?; + let referenced_classic_plan = image + .classic_plan() + .map(native_referenced_classic_plan) + .transpose()?; + Ok(crate::decoder::grayscale_batch::GrayscaleBatchInput { + source_index, + bytes: image.bytes().as_ref(), + device_plan: Some(image.plan()), + referenced_plan, + referenced_classic_plan, + }) + }) + .collect() +} diff --git a/crates/j2k-cuda/src/batch/types.rs b/crates/j2k-cuda/src/batch/types.rs new file mode 100644 index 00000000..1e0aef47 --- /dev/null +++ b/crates/j2k-cuda/src/batch/types.rs @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! CUDA batch result and resident-output contracts. + +#[cfg(feature = "cuda-runtime")] +use super::Arc; +use super::{ + BatchGroupInfo, BatchInfrastructureError, Error, IndexedBatchError, J2kDecodeWarning, + PreparedBatchGroup, Rect, Surface, +}; + +/// Failure while preparing or executing an owned CUDA batch. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum CudaBatchError { + /// Shared codec preparation or host-side batch infrastructure failed. + #[error(transparent)] + Infrastructure(#[from] BatchInfrastructureError), + /// CUDA execution failed for one homogeneous group. + /// + /// No output surfaces from the affected group are exposed. + #[error("CUDA batch group containing source indices {source_indices:?} failed: {source}")] + GroupExecution { + /// Every original input index whose dense group output was discarded. + source_indices: Vec, + /// Strict CUDA adapter or runtime failure. + #[source] + source: Box, + }, +} + +impl CudaBatchError { + pub(super) fn group(group: &PreparedBatchGroup, source: Error) -> Self { + Self::GroupExecution { + source_indices: group.source_indices().to_vec(), + source: Box::new(source), + } + } + + /// Whether submitted CUDA work may still reference an external + /// destination because completion could not be established. + #[cfg(feature = "cuda-runtime")] + #[doc(hidden)] + pub fn completion_is_uncertain(&self) -> bool { + match self { + Self::GroupExecution { source, .. } => source.completion_is_uncertain(), + Self::Infrastructure(_) => false, + } + } + + /// Whether this failure prevents the current persistent batch operation + /// from safely continuing with later groups. + #[doc(hidden)] + #[must_use] + pub fn session_is_unusable(&self) -> bool { + match self { + Self::Infrastructure(_) => true, + Self::GroupExecution { source, .. } => source.session_is_unusable(), + } + } +} + +#[cfg(test)] +mod classification_tests { + use j2k_core::BatchInfrastructureError; + + use super::CudaBatchError; + use crate::Error; + + fn group_error(source: Error) -> CudaBatchError { + CudaBatchError::GroupExecution { + source_indices: vec![3], + source: Box::new(source), + } + } + + #[test] + fn cuda_batch_session_classification_is_owned_by_codec_errors() { + assert!( + CudaBatchError::Infrastructure(BatchInfrastructureError::EmptyBatchPlan) + .session_is_unusable() + ); + assert!(group_error(Error::CudaUnavailable).session_is_unusable()); + assert!(!group_error(Error::UnsupportedCudaRequest { + reason: "test contract rejection", + }) + .session_is_unusable()); + } +} + +/// Failure while executing one homogeneous CUDA group. +/// +/// No partially written dense output from the affected group is exposed. +/// Other prepared groups may still succeed when the retained CUDA session +/// remains usable. +#[derive(Debug, thiserror::Error)] +#[error("CUDA batch group containing source indices {source_indices:?} failed: {source}")] +pub struct CudaBatchGroupError { + source_indices: Vec, + #[source] + source: Box, +} + +impl CudaBatchGroupError { + #[cfg(feature = "cuda-runtime")] + pub(super) fn new(group: &PreparedBatchGroup, source: Error) -> Self { + Self { + source_indices: group.source_indices().to_vec(), + source: Box::new(source), + } + } + + #[cfg(feature = "cuda-runtime")] + pub(super) fn from_parts(source_indices: Vec, source: Error) -> Self { + Self { + source_indices, + source: Box::new(source), + } + } + + /// Original input indices whose dense group output was discarded. + #[must_use] + pub fn source_indices(&self) -> &[usize] { + &self.source_indices + } + + /// Strict CUDA adapter or runtime failure for this group. + #[must_use] + pub fn source(&self) -> &Error { + &self.source + } + + /// Consume the group failure into affected indices and its source. + #[must_use] + pub fn into_parts(self) -> (Vec, Error) { + (self.source_indices, *self.source) + } +} + +/// One successful homogeneous CUDA-resident output group. +#[derive(Debug)] +pub struct CudaBatchGroup { + pub(super) info: BatchGroupInfo, + pub(super) source_indices: Vec, + pub(super) decoded_rects: Vec, + pub(super) warnings: Vec>, + pub(super) surfaces: Vec, + #[cfg(feature = "cuda-runtime")] + pub(super) dense_output: CudaResidentBatchBuffer, +} + +/// One codec-owned dense CUDA allocation containing a homogeneous batch. +/// +/// This owner is the canonical resident representation for every exact-native +/// grayscale or color group. Grayscale and NHWC RGB/RGBA groups also expose +/// ordinary [`Surface`] views over the same allocation for compatibility. +#[cfg(feature = "cuda-runtime")] +#[derive(Debug)] +pub struct CudaResidentBatchBuffer { + pub(super) buffer: Arc, + pub(super) ranges: Vec, +} + +#[cfg(feature = "cuda-runtime")] +impl CudaResidentBatchBuffer { + /// Codec-owned CUDA allocation containing every image range. + #[must_use] + pub fn buffer(&self) -> &j2k_cuda_runtime::CudaDeviceBuffer { + &self.buffer + } + + /// Tightly concatenated per-image byte ranges in dense batch order. + #[must_use] + pub fn ranges(&self) -> &[j2k_cuda_runtime::CudaDeviceBufferRange] { + &self.ranges + } +} + +impl CudaBatchGroup { + /// Shared decoded dimensions, type, color, transform, route, and layout. + #[must_use] + pub const fn info(&self) -> &BatchGroupInfo { + &self.info + } + + /// Original input indices in dense batch order. + #[must_use] + pub fn source_indices(&self) -> &[usize] { + &self.source_indices + } + + /// Actual decoded rectangle for each image. + #[must_use] + pub fn decoded_rects(&self) -> &[Rect] { + &self.decoded_rects + } + + /// Non-fatal codec warnings for each image. + #[must_use] + pub fn warnings(&self) -> &[Vec] { + &self.warnings + } + + /// CUDA-resident image views in dense batch order. + /// + /// Grayscale groups expose one view per image. NHWC RGB/RGBA groups also + /// expose compatible interleaved views over their dense group allocation. + /// No decoded host staging is used. + #[must_use] + pub fn surfaces(&self) -> &[Surface] { + &self.surfaces + } + + /// Dense codec-owned allocation for exact-native grayscale or color output. + /// + /// NCHW color groups must be consumed through this owner because a + /// [`Surface`] describes interleaved pixels. Grayscale and NHWC RGB/RGBA + /// groups return both this owner and compatible surface views. + #[cfg(feature = "cuda-runtime")] + #[must_use] + pub const fn dense_output(&self) -> &CudaResidentBatchBuffer { + &self.dense_output + } + + /// Consume the group into metadata and CUDA-resident views. + #[must_use] + #[expect( + clippy::type_complexity, + reason = "the tuple mirrors the group's five explicitly documented owners" + )] + #[cfg(not(feature = "cuda-runtime"))] + pub fn into_parts( + self, + ) -> ( + BatchGroupInfo, + Vec, + Vec, + Vec>, + Vec, + ) { + ( + self.info, + self.source_indices, + self.decoded_rects, + self.warnings, + self.surfaces, + ) + } + + /// Consume the group into metadata, compatible surfaces, and its required + /// dense exact-native allocation. + #[cfg(feature = "cuda-runtime")] + #[must_use] + #[expect( + clippy::type_complexity, + reason = "the tuple mirrors the group's explicitly documented owners" + )] + pub fn into_parts( + self, + ) -> ( + BatchGroupInfo, + Vec, + Vec, + Vec>, + Vec, + CudaResidentBatchBuffer, + ) { + ( + self.info, + self.source_indices, + self.decoded_rects, + self.warnings, + self.surfaces, + self.dense_output, + ) + } +} + +/// CUDA batch successes plus indexed codec preflight failures. +#[derive(Debug)] +pub struct CudaBatchDecodeResult { + pub(super) groups: Vec, + pub(super) errors: Vec, + pub(super) group_errors: Vec, +} + +impl CudaBatchDecodeResult { + /// Successfully decoded homogeneous device groups. + #[must_use] + pub fn groups(&self) -> &[CudaBatchGroup] { + &self.groups + } + + /// Per-input parsing and representability failures from shared preflight. + #[must_use] + pub fn errors(&self) -> &[IndexedBatchError] { + &self.errors + } + + /// Homogeneous groups that failed during recoverable CUDA execution. + #[must_use] + pub fn group_errors(&self) -> &[CudaBatchGroupError] { + &self.group_errors + } + + /// Consume this result into successful groups, indexed preflight errors, + /// and homogeneous execution failures. + #[must_use] + pub fn into_parts( + self, + ) -> ( + Vec, + Vec, + Vec, + ) { + (self.groups, self.errors, self.group_errors) + } +} diff --git a/crates/j2k-cuda/src/codec.rs b/crates/j2k-cuda/src/codec.rs index ad4a2861..72afeb30 100644 --- a/crates/j2k-cuda/src/codec.rs +++ b/crates/j2k-cuda/src/codec.rs @@ -43,7 +43,13 @@ impl Codec { fn supports_cuda_batch_format(fmt: PixelFormat) -> bool { matches!( fmt, - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 | PixelFormat::Rgba16 + PixelFormat::Gray8 + | PixelFormat::Gray16 + | PixelFormat::GrayI16 + | PixelFormat::Rgb8 + | PixelFormat::Rgba8 + | PixelFormat::Rgb16 + | PixelFormat::Rgba16 ) } diff --git a/crates/j2k-cuda/src/decoder.rs b/crates/j2k-cuda/src/decoder.rs index 4253c34f..070f5b45 100644 --- a/crates/j2k-cuda/src/decoder.rs +++ b/crates/j2k-cuda/src/decoder.rs @@ -45,7 +45,7 @@ const CUDA_HTJ2K_KERNELS_NOT_READY: &str = "strict CUDA HTJ2K resident codestream decode kernels are not available in this build"; #[cfg(feature = "cuda-runtime")] const CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED: &str = - "strict CUDA HTJ2K resident decode currently accepts Gray8, Gray16, Rgb8, Rgba8, Rgb16, and Rgba16 output"; + "strict CUDA HTJ2K resident decode currently accepts Gray8, Gray16, GrayI16, Rgb8, Rgba8, Rgb16, and Rgba16 output"; #[cfg(feature = "cuda-runtime")] const CUDA_HTJ2K_PLAN_INVARIANT_FAILED: &str = "strict CUDA HTJ2K resident decode plan has invalid internal ranges"; @@ -64,10 +64,20 @@ mod color_batch; #[path = "decoder/profile.rs"] mod decode_profile; #[cfg(feature = "cuda-runtime")] +pub(crate) mod grayscale_batch; +#[cfg(feature = "cuda-runtime")] +mod pending_completion; +#[cfg(feature = "cuda-runtime")] mod plan; #[cfg(feature = "cuda-runtime")] mod resident; +#[cfg(feature = "cuda-runtime")] +pub(crate) use self::color_batch::native_batch::{ + submit_native_color_resident_prepared_batch, submit_native_color_resident_prepared_batch_into, + NativeColorBatchInput, NativeColorOwnedBatch, SubmittedNativeColorExternalBatch, + SubmittedNativeColorResidentBatch, +}; #[cfg(all(test, feature = "cuda-runtime"))] pub(crate) use self::color_batch::{ testing_cuda_htj2k_batch_decode_calls, testing_reset_cuda_htj2k_batch_decode_calls, @@ -142,6 +152,30 @@ struct CudaQueuedIdwtBatch { #[cfg(feature = "cuda-runtime")] impl CudaQueuedIdwtBatch { + fn merge(mut self, mut next: Self) -> Result { + if !self.context.is_same_context(&next.context) { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + }); + } + self.queued + .try_reserve_exact(next.queued.len()) + .map_err(|_| { + crate::allocation::host_allocation_error::( + self.queued.len().saturating_add(next.queued.len()), + "j2k CUDA independent IDWT completion guards", + ) + })?; + self.queued.append(&mut next.queued); + self.kernel_dispatches = self + .kernel_dispatches + .saturating_add(next.kernel_dispatches); + self.decode_dispatches = self + .decode_dispatches + .saturating_add(next.decode_dispatches); + Ok(self) + } + fn resources_pending(&self) -> bool { self.kernel_dispatches != 0 && !self.queued.is_empty() } @@ -207,9 +241,10 @@ struct CudaDecodedComponent { #[cfg(feature = "cuda-runtime")] struct CudaHtj2kColorDecodePlans { + output_index: usize, dimensions: (u32, u32), mct_dimensions: (u32, u32), - bit_depths: [u8; 3], + bit_depths: [u8; 4], mct: bool, transform: CudaHtj2kTransform, payload: Vec, @@ -217,6 +252,13 @@ struct CudaHtj2kColorDecodePlans { report: CudaHtj2kProfileReport, } +#[cfg(feature = "cuda-runtime")] +impl CudaHtj2kColorDecodePlans { + const fn rgb_bit_depths(&self) -> [u8; 3] { + [self.bit_depths[0], self.bit_depths[1], self.bit_depths[2]] + } +} + #[cfg(all(test, feature = "cuda-runtime"))] mod tests { use super::{ diff --git a/crates/j2k-cuda/src/decoder/api.rs b/crates/j2k-cuda/src/decoder/api.rs index 55bc73b1..074cb74d 100644 --- a/crates/j2k-cuda/src/decoder/api.rs +++ b/crates/j2k-cuda/src/decoder/api.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +#[cfg(feature = "cuda-runtime")] +use super::grayscale_batch::decode_grayscale_cuda_resident_batch_into_with_profile; #[cfg(feature = "cuda-runtime")] use super::resident::{ decode_batch_to_cuda_resident_surface_with_profile_control, @@ -196,6 +198,31 @@ impl<'a> J2kDecoder<'a> { decode_batch_to_cuda_resident_surface_with_profile_control(inputs, session, fmt, true) } + /// Strictly decode a full grayscale batch directly into a validated + /// caller-owned CUDA destination. + #[cfg(feature = "cuda-runtime")] + #[doc(hidden)] + pub fn decode_batch_into_external_device_with_session( + inputs: &[&[u8]], + fmt: PixelFormat, + destination: &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>, + session: &mut CudaSession, + ) -> Result< + ( + Vec, + CudaHtj2kProfileReport, + ), + Error, + > { + decode_grayscale_cuda_resident_batch_into_with_profile( + inputs, + session, + fmt, + destination, + false, + ) + } + /// Strictly decode a full-resolution HTJ2K region into a CUDA-backed /// surface using an existing backend session. pub(crate) fn decode_region_to_device_with_session( diff --git a/crates/j2k-cuda/src/decoder/color_batch.rs b/crates/j2k-cuda/src/decoder/color_batch.rs index 6b842694..1b261f79 100644 --- a/crates/j2k-cuda/src/decoder/color_batch.rs +++ b/crates/j2k-cuda/src/decoder/color_batch.rs @@ -3,18 +3,35 @@ #[cfg(all(test, feature = "cuda-runtime"))] use core::cell::Cell; +#[cfg(feature = "cuda-runtime")] +mod batch_execution; #[cfg(feature = "cuda-runtime")] mod finish; #[cfg(feature = "cuda-runtime")] mod host_owners; #[cfg(feature = "cuda-runtime")] +pub(crate) mod native_batch; +#[cfg(feature = "cuda-runtime")] +mod single; +#[cfg(feature = "cuda-runtime")] mod store; #[cfg(feature = "cuda-runtime")] -use self::finish::finish_color_cuda_resident_surface_with_component_work; +pub(in crate::decoder) use self::batch_execution::{ + decode_color_cuda_resident_batch_surfaces_with_profile, finalize_color_batch_decode_report, +}; +#[cfg(feature = "cuda-runtime")] +use self::finish::{ + finish_color_cuda_resident_surface_with_component_work, FinishColorCudaResidentSurfaceRequest, +}; #[cfg(feature = "cuda-runtime")] use self::host_owners::{append_color_payload_to_shared, take_component_work}; #[cfg(feature = "cuda-runtime")] +pub(in crate::decoder) use self::single::{ + decode_color_cuda_resident_region_scaled_surface, decode_color_cuda_resident_region_surface, + decode_color_cuda_resident_scaled_surface, decode_color_cuda_resident_surface_with_profile, +}; +#[cfg(feature = "cuda-runtime")] use self::store::{ can_fuse_mct_store_for_stores, dispatch_color_store, prepare_rgb8_mct_batch_store, rgb8_mct_batch_store_target, run_color_mct, validate_color_stores, ColorStoreInputs, @@ -54,669 +71,3 @@ pub(crate) fn testing_reset_cuda_htj2k_batch_decode_calls() { pub(crate) fn testing_cuda_htj2k_batch_decode_calls() -> usize { CUDA_HTJ2K_BATCH_DECODE_CALLS.with(Cell::get) } - -#[cfg(feature = "cuda-runtime")] -pub(super) fn decode_color_cuda_resident_surface_with_profile( - decoder: &mut J2kDecoder<'_>, - session: &mut CudaSession, - fmt: PixelFormat, - wall_started: Option, - collect_stage_timings: bool, -) -> Result<(Surface, CudaHtj2kProfileReport), Error> { - let color = decoder.build_cuda_htj2k_color_plans_with_profile(fmt)?; - decode_color_cuda_resident_surface_with_plans_profile( - session, - fmt, - color, - wall_started, - collect_stage_timings, - ) -} - -#[cfg(feature = "cuda-runtime")] -pub(super) fn decode_color_cuda_resident_scaled_surface( - decoder: &mut J2kDecoder<'_>, - session: &mut CudaSession, - fmt: PixelFormat, - output_dimensions: (u32, u32), -) -> Result { - let collect_stage_timings = profile::profile_stages_enabled(); - let wall_started = profile::profile_now(collect_stage_timings); - let color = decoder.build_cuda_htj2k_color_scaled_plans_with_profile(fmt, output_dimensions)?; - decode_color_cuda_resident_surface_with_plans_profile( - session, - fmt, - color, - wall_started, - collect_stage_timings, - ) - .map(|(surface, _report)| surface) -} - -#[cfg(feature = "cuda-runtime")] -pub(super) fn decode_color_cuda_resident_region_surface( - decoder: &mut J2kDecoder<'_>, - session: &mut CudaSession, - fmt: PixelFormat, - roi: Rect, -) -> Result { - let collect_stage_timings = profile::profile_stages_enabled(); - let wall_started = profile::profile_now(collect_stage_timings); - let color = decoder.build_cuda_htj2k_color_region_plans_with_profile(fmt, roi)?; - decode_color_cuda_resident_surface_with_plans_profile( - session, - fmt, - color, - wall_started, - collect_stage_timings, - ) - .map(|(surface, _report)| surface) -} - -#[cfg(feature = "cuda-runtime")] -pub(super) fn decode_color_cuda_resident_region_scaled_surface( - decoder: &mut J2kDecoder<'_>, - session: &mut CudaSession, - fmt: PixelFormat, - scaled_roi: Rect, - scaled_dimensions: (u32, u32), -) -> Result { - let collect_stage_timings = profile::profile_stages_enabled(); - let wall_started = profile::profile_now(collect_stage_timings); - let color = decoder.build_cuda_htj2k_color_region_scaled_plans_with_profile( - fmt, - scaled_roi, - scaled_dimensions, - )?; - decode_color_cuda_resident_surface_with_plans_profile( - session, - fmt, - color, - wall_started, - collect_stage_timings, - ) - .map(|(surface, _report)| surface) -} - -#[cfg(feature = "cuda-runtime")] -pub(super) fn decode_color_cuda_resident_surface_with_plans_profile( - session: &mut CudaSession, - fmt: PixelFormat, - mut color: CudaHtj2kColorDecodePlans, - wall_started: Option, - collect_stage_timings: bool, -) -> Result<(Surface, CudaHtj2kProfileReport), Error> { - if color.components.len() != 3 { - return Err(Error::UnsupportedCudaRequest { - reason: CUDA_HTJ2K_KERNELS_NOT_READY, - }); - } - let context = session.cuda_context()?; - let pool = session.decode_buffer_pool()?; - let table_upload_start = profile::profile_now(collect_stage_timings); - let table_resources = if color - .components - .iter() - .all(|plan| plan.subbands().is_empty()) - { - None - } else { - Some(session.htj2k_decode_table_resources()?) - }; - let table_upload_us = profile::elapsed_us(table_upload_start); - color.report.h2d_us = color.report.h2d_us.saturating_add(table_upload_us); - color.report.detail.table_upload_us = color - .report - .detail - .table_upload_us - .saturating_add(table_upload_us); - let payload_upload_start = profile::profile_now(collect_stage_timings); - let decode_resources = match table_resources.as_ref() { - Some(tables) => context.upload_htj2k_decode_resources_with_tables(&color.payload, tables), - None => context.upload_j2k_decode_payload(&color.payload), - } - .map_err(cuda_error)?; - let payload_upload_us = profile::elapsed_us(payload_upload_start); - profile::add_payload_resource_upload_us(&mut color.report, payload_upload_us); - let mut host_budget = HostPhaseBudget::new("j2k CUDA color decode execution graph"); - color.account_host_owners(&mut host_budget)?; - let mut component_work = host_budget.try_vec_with_capacity(3)?; - for plan in &color.components { - component_work.push(decode_cuda_component_subbands_with_resources( - &context, - plan, - &pool, - collect_stage_timings, - &mut host_budget, - )?); - } - run_component_cleanup_dequant_batches( - &context, - &decode_resources, - &mut component_work, - &pool, - collect_stage_timings, - host_budget.live_bytes(), - )?; - finish_color_cuda_resident_surface_with_component_work(FinishColorCudaResidentSurfaceRequest { - context: &context, - pool: &pool, - fmt, - color, - component_work, - wall_started, - collect_stage_timings, - run_idwt: true, - emit_report: true, - }) -} - -#[cfg(feature = "cuda-runtime")] -#[expect( - clippy::too_many_lines, - reason = "resident color batch orchestration keeps CUDA submission and profile ordering atomic" -)] -pub(super) fn decode_color_cuda_resident_batch_surfaces_with_profile( - inputs: &[&[u8]], - session: &mut CudaSession, - fmt: PixelFormat, - collect_stage_timings: bool, -) -> Result<(Vec, CudaHtj2kProfileReport), Error> { - let batch_wall_started = profile::profile_now(collect_stage_timings); - let mut initial_budget = HostPhaseBudget::new("j2k CUDA color batch plan owners"); - let mut colors = initial_budget.try_vec_with_capacity(inputs.len())?; - let mut shared_payload = Vec::new(); - let mut native_context = NativeDecoderContext::default(); - for input in inputs { - let mut color = - build_cuda_htj2k_color_plans_from_bytes_with_profile(input, fmt, &mut native_context)?; - if color.components.len() != 3 { - return Err(Error::UnsupportedCudaRequest { - reason: CUDA_HTJ2K_KERNELS_NOT_READY, - }); - } - let mut append_budget = host_owners::color_batch_budget( - &colors, - &shared_payload, - Some(&color), - "j2k CUDA color batch plan owners", - )?; - append_color_payload_to_shared(&mut color, &mut shared_payload, &mut append_budget)?; - colors.push(color); - host_owners::color_batch_budget( - &colors, - &shared_payload, - None, - "j2k CUDA color batch plan owners", - )?; - } - - let context = session.cuda_context()?; - let pool = session.decode_batch_buffer_pool()?; - let table_upload_start = profile::profile_now(collect_stage_timings); - let table_resources = if colors.iter().all(|color| { - color - .components - .iter() - .all(|plan| plan.subbands().is_empty()) - }) { - None - } else { - Some(session.htj2k_decode_table_resources()?) - }; - let table_upload_us = profile::elapsed_us(table_upload_start); - let payload_upload_start = profile::profile_now(collect_stage_timings); - let decode_resources = match table_resources.as_ref() { - Some(tables) => context.upload_htj2k_decode_resources_with_tables_and_pool( - &shared_payload, - tables, - &pool, - ), - None => context.upload_j2k_decode_payload_with_pool(&shared_payload, &pool), - } - .map_err(cuda_error)?; - let payload_upload_us = profile::elapsed_us(payload_upload_start); - drop(shared_payload); - - let component_count = colors - .iter() - .map(|color| color.components.len()) - .sum::(); - let mut host_budget = HostPhaseBudget::new("j2k CUDA color batch execution graph"); - host_owners::account_colors(&mut host_budget, &colors)?; - let mut all_component_work = host_budget.try_vec_with_capacity(component_count)?; - for color in &colors { - for plan in &color.components { - all_component_work.push(decode_cuda_component_subbands_with_resources( - &context, - plan, - &pool, - collect_stage_timings, - &mut host_budget, - )?); - } - } - run_component_cleanup_dequant_batches( - &context, - &decode_resources, - &mut all_component_work, - &pool, - collect_stage_timings, - host_budget.live_bytes(), - )?; - let mut batch_components = host_budget.try_vec_with_capacity(component_count)?; - for color in &colors { - for component in &color.components { - batch_components.push(component); - } - } - let idwt_batched = can_batch_color_idwt(&batch_components); - let pending_idwt_batch = if idwt_batched { - run_color_component_idwt_batches( - &context, - &batch_components, - &mut all_component_work, - &pool, - collect_stage_timings, - host_budget.live_bytes(), - )? - } else { - None - }; - drop(batch_components); - - let completion_result = (|| { - let can_use_batch_store = - idwt_batched && can_batch_rgb8_mct_color_store(fmt, &colors, &all_component_work)?; - let (surfaces, reports) = if can_use_batch_store { - finish_color_cuda_resident_batch_surfaces_with_rgb8_mct_store( - &context, - fmt, - colors, - all_component_work, - collect_stage_timings, - )? - } else { - let mut output_budget = HostPhaseBudget::new("j2k CUDA color batch output graph"); - host_owners::account_colors(&mut output_budget, &colors)?; - host_owners::account_component_work(&mut output_budget, &all_component_work)?; - let mut surfaces = output_budget.try_vec_with_capacity(colors.len())?; - let mut reports = output_budget.try_vec_with_capacity(colors.len())?; - let mut work_iter = all_component_work.into_iter(); - for color in colors { - let component_count = color.components.len(); - let component_work = - take_component_work(&mut work_iter, component_count, &mut output_budget)?; - let (surface, report) = finish_color_cuda_resident_surface_with_component_work( - FinishColorCudaResidentSurfaceRequest { - context: &context, - pool: &pool, - fmt, - color, - component_work, - wall_started: None, - collect_stage_timings, - run_idwt: !idwt_batched, - emit_report: false, - }, - )?; - surfaces.push(surface); - reports.push(report); - } - (surfaces, reports) - }; - // Runtime MCT/store launches synchronize before returning, so a - // recorded dispatch is also a completion point for preceding IDWT. - let completion_established = reports.iter().any(|report| { - report.detail.mct_dispatch_count != 0 || report.detail.store_dispatch_count != 0 - }); - Ok(((surfaces, reports), completion_established)) - })(); - let (surfaces, reports) = CudaQueuedIdwtBatch::resolve_optional_after_completed_work( - pending_idwt_batch, - completion_result, - )?; - - let aggregate = finalize_color_batch_decode_report( - &reports, - table_upload_us, - payload_upload_us, - batch_wall_started, - ); - aggregate.emit("decode_batch"); - - Ok((surfaces, aggregate)) -} - -#[cfg(feature = "cuda-runtime")] -pub(super) fn finalize_color_batch_decode_report( - reports: &[CudaHtj2kProfileReport], - table_upload_us: u128, - payload_upload_us: u128, - batch_wall_started: Option, -) -> CudaHtj2kProfileReport { - let mut aggregate = aggregate_decode_reports(reports); - aggregate.h2d_us = aggregate - .h2d_us - .saturating_add(table_upload_us) - .saturating_add(payload_upload_us); - aggregate.detail.table_upload_us = aggregate - .detail - .table_upload_us - .saturating_add(table_upload_us); - aggregate.detail.payload_upload_us = aggregate - .detail - .payload_upload_us - .saturating_add(payload_upload_us); - aggregate.detail.wall_total_us = profile::elapsed_us(batch_wall_started); - profile::finalize_decode_total_us(&mut aggregate); - aggregate -} - -#[cfg(feature = "cuda-runtime")] -pub(super) fn can_batch_rgb8_mct_color_store( - fmt: PixelFormat, - colors: &[CudaHtj2kColorDecodePlans], - all_component_work: &[CudaComponentDecodeWork], -) -> Result { - if !matches!(fmt, PixelFormat::Rgb8 | PixelFormat::Rgba8) { - return Ok(false); - } - - let mut offset = 0usize; - for color in colors { - let component_count = color.components.len(); - if component_count != 3 || offset.saturating_add(component_count) > all_component_work.len() - { - return Err(Error::UnsupportedCudaRequest { - reason: CUDA_HTJ2K_KERNELS_NOT_READY, - }); - } - if !color.mct { - return Ok(false); - } - let component_work = &all_component_work[offset..offset + component_count]; - let stores = [ - &component_work[0].store, - &component_work[1].store, - &component_work[2].store, - ]; - validate_color_stores(stores, color.dimensions)?; - if !can_fuse_mct_store_for_stores(stores) { - return Ok(false); - } - offset = offset.saturating_add(component_count); - } - - if offset != all_component_work.len() { - return Err(Error::UnsupportedCudaRequest { - reason: CUDA_HTJ2K_KERNELS_NOT_READY, - }); - } - Ok(!colors.is_empty()) -} - -#[cfg(feature = "cuda-runtime")] -pub(super) fn finish_color_cuda_resident_batch_surfaces_with_rgb8_mct_store( - context: &j2k_cuda_runtime::CudaContext, - fmt: PixelFormat, - colors: Vec, - all_component_work: Vec, - collect_stage_timings: bool, -) -> Result<(Vec, Vec), Error> { - let mut host_budget = HostPhaseBudget::new("j2k CUDA prepared color batch store graph"); - host_owners::account_colors(&mut host_budget, &colors)?; - host_owners::account_component_work(&mut host_budget, &all_component_work)?; - let mut prepared = host_budget.try_vec_with_capacity(colors.len())?; - let mut work_iter = all_component_work.into_iter(); - for color in colors { - let component_count = color.components.len(); - let component_work = - take_component_work(&mut work_iter, component_count, &mut host_budget)?; - prepared.push(prepare_rgb8_mct_batch_store(fmt, color, component_work)?); - } - if work_iter.next().is_some() { - return Err(Error::UnsupportedCudaRequest { - reason: CUDA_HTJ2K_KERNELS_NOT_READY, - }); - } - - let targets = - host_budget.try_collect_results_exact(prepared.iter().map(rgb8_mct_batch_store_target))?; - let (store_output, store_us) = context - .time_default_stream_named_us_if( - collect_stage_timings, - "j2k.htj2k.decode.store.color.batch", - || { - context.j2k_store_rgb8_mct_batch_contiguous_device_with_live_host_bytes( - &targets, - host_budget.live_bytes(), - ) - }, - ) - .map_err(cuda_error)?; - drop(targets); - let (surface_buffer, surface_ranges, store_stats) = store_output.into_parts(); - if surface_ranges.len() != prepared.len() { - return Err(Error::UnsupportedCudaRequest { - reason: CUDA_HTJ2K_KERNELS_NOT_READY, - }); - } - let shared_surface_buffer = Arc::new(surface_buffer); - - let mut output_budget = HostPhaseBudget::new("j2k CUDA stored color batch output graph"); - output_budget.account_vec(&prepared)?; - for item in &prepared { - item.color.account_host_owners(&mut output_budget)?; - } - output_budget.account_vec(&surface_ranges)?; - let mut surfaces = output_budget.try_vec_with_capacity(prepared.len())?; - let mut reports = output_budget.try_vec_with_capacity(prepared.len())?; - let store_dispatches = store_stats.kernel_dispatches(); - let store_decode_dispatches = store_stats.decode_kernel_dispatches(); - for (index, (mut prepared, surface_range)) in - prepared.into_iter().zip(surface_ranges).enumerate() - { - let report_store_dispatches = if index == 0 { store_dispatches } else { 0 }; - let report_store_decode_dispatches = if index == 0 { - store_decode_dispatches - } else { - 0 - }; - let report_store_us = if index == 0 { store_us } else { 0 }; - let dispatches = prepared.dispatches.saturating_add(report_store_dispatches); - let decode_dispatches = prepared - .decode_dispatches - .saturating_add(report_store_decode_dispatches); - prepared.color.report.dispatch_count = dispatches; - prepared.color.report.store_us = prepared - .color - .report - .store_us - .saturating_add(report_store_us); - prepared.color.report.detail.store_dispatch_count = prepared - .color - .report - .detail - .store_dispatch_count - .saturating_add(report_store_dispatches); - profile::finalize_decode_total_us(&mut prepared.color.report); - - let dimensions = prepared.color.dimensions; - surfaces.push(Surface { - backend: BackendKind::Cuda, - residency: SurfaceResidency::CudaResidentDecode, - dimensions, - fmt, - pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), - stats: CudaSurfaceStats { - total: dispatches, - copy: 0, - decode: decode_dispatches, - }, - storage: cuda_range_storage( - shared_surface_buffer.clone(), - surface_range.offset, - surface_range.len, - ), - }); - reports.push(prepared.color.report); - } - - Ok((surfaces, reports)) -} - -#[cfg(feature = "cuda-runtime")] -struct FinishColorCudaResidentSurfaceRequest<'a> { - context: &'a j2k_cuda_runtime::CudaContext, - pool: &'a CudaBufferPool, - fmt: PixelFormat, - color: CudaHtj2kColorDecodePlans, - component_work: Vec, - wall_started: Option, - collect_stage_timings: bool, - run_idwt: bool, - emit_report: bool, -} - -#[cfg(feature = "cuda-runtime")] -struct PreparedColorComponents { - components: [CudaDecodedComponent; 3], - dispatches: usize, - decode_dispatches: usize, -} - -#[cfg(feature = "cuda-runtime")] -struct FinalizeColorSurfaceRequest { - fmt: PixelFormat, - color: CudaHtj2kColorDecodePlans, - surface_buffer: CudaDeviceBuffer, - dispatches: usize, - decode_dispatches: usize, - store_stats: CudaExecutionStats, - store_us: u128, - wall_started: Option, - emit_report: bool, -} - -#[cfg(feature = "cuda-runtime")] -fn run_pending_color_idwt( - context: &j2k_cuda_runtime::CudaContext, - color: &CudaHtj2kColorDecodePlans, - component_work: &mut [CudaComponentDecodeWork], - pool: &CudaBufferPool, - collect_stage_timings: bool, - host_budget: &mut HostPhaseBudget, -) -> Result, Error> { - let mut batch_components = host_budget.try_vec_with_capacity(color.components.len())?; - for component in &color.components { - batch_components.push(component); - } - if can_batch_color_idwt(&batch_components) { - run_color_component_idwt_batches( - context, - &batch_components, - component_work, - pool, - collect_stage_timings, - host_budget.live_bytes(), - ) - } else { - for (plan, work) in color.components.iter().zip(component_work.iter_mut()) { - run_cuda_component_idwt_steps( - context, - plan.idwt_steps(), - work, - pool, - collect_stage_timings, - )?; - } - Ok(None) - } -} - -#[cfg(feature = "cuda-runtime")] -fn finish_color_components( - component_work: Vec, - color: &mut CudaHtj2kColorDecodePlans, -) -> Result { - let [work0, work1, work2]: [CudaComponentDecodeWork; 3] = - component_work - .try_into() - .map_err(|_| Error::UnsupportedCudaRequest { - reason: CUDA_HTJ2K_KERNELS_NOT_READY, - })?; - let components = [ - finish_cuda_component_decode(work0)?, - finish_cuda_component_decode(work1)?, - finish_cuda_component_decode(work2)?, - ]; - let stores = [ - &components[0].store, - &components[1].store, - &components[2].store, - ]; - validate_color_stores(stores, color.dimensions)?; - - let dispatches = components - .iter() - .map(|component| component.dispatches) - .sum::(); - let decode_dispatches = components - .iter() - .map(|component| component.decode_dispatches) - .sum::(); - for component in &components { - component.timings.add_to_report(&mut color.report); - } - Ok(PreparedColorComponents { - components, - dispatches, - decode_dispatches, - }) -} - -#[cfg(feature = "cuda-runtime")] -fn finalize_color_surface( - request: FinalizeColorSurfaceRequest, -) -> (Surface, CudaHtj2kProfileReport) { - let FinalizeColorSurfaceRequest { - fmt, - mut color, - surface_buffer, - mut dispatches, - mut decode_dispatches, - store_stats, - store_us, - wall_started, - emit_report, - } = request; - dispatches = dispatches.saturating_add(store_stats.kernel_dispatches()); - decode_dispatches = decode_dispatches.saturating_add(store_stats.decode_kernel_dispatches()); - color.report.dispatch_count = dispatches; - color.report.store_us = color.report.store_us.saturating_add(store_us); - color.report.detail.store_dispatch_count = color - .report - .detail - .store_dispatch_count - .saturating_add(store_stats.kernel_dispatches()); - color.report.detail.wall_total_us = profile::elapsed_us(wall_started); - profile::finalize_decode_total_us(&mut color.report); - if emit_report { - color.report.emit("decode"); - } - let surface = Surface { - backend: BackendKind::Cuda, - residency: SurfaceResidency::CudaResidentDecode, - dimensions: color.dimensions, - fmt, - pitch_bytes: color.dimensions.0 as usize * fmt.bytes_per_pixel(), - stats: CudaSurfaceStats { - total: dispatches, - copy: 0, - decode: decode_dispatches, - }, - storage: Storage::Cuda(surface_buffer), - }; - (surface, color.report) -} diff --git a/crates/j2k-cuda/src/decoder/color_batch/batch_execution.rs b/crates/j2k-cuda/src/decoder/color_batch/batch_execution.rs new file mode 100644 index 00000000..f9cec929 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/batch_execution.rs @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + aggregate_decode_reports, append_color_payload_to_shared, + build_cuda_htj2k_color_plans_from_bytes_with_profile, can_batch_color_idwt, + can_fuse_mct_store_for_stores, cuda_error, cuda_range_storage, + decode_cuda_component_subbands_with_resources, + finish_color_cuda_resident_surface_with_component_work, host_owners, + prepare_rgb8_mct_batch_store, profile, rgb8_mct_batch_store_target, + run_color_component_idwt_batches, run_component_cleanup_dequant_batches, take_component_work, + validate_color_stores, Arc, BackendKind, CudaComponentDecodeWork, CudaHtj2kColorDecodePlans, + CudaHtj2kProfileReport, CudaQueuedIdwtBatch, CudaSession, CudaSurfaceStats, Error, + FinishColorCudaResidentSurfaceRequest, HostPhaseBudget, NativeDecoderContext, PixelFormat, + Surface, SurfaceResidency, CUDA_HTJ2K_KERNELS_NOT_READY, +}; + +#[expect( + clippy::too_many_lines, + reason = "resident color batch orchestration keeps CUDA submission and profile ordering atomic" +)] +pub(in crate::decoder) fn decode_color_cuda_resident_batch_surfaces_with_profile( + inputs: &[&[u8]], + session: &mut CudaSession, + fmt: PixelFormat, + collect_stage_timings: bool, +) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + let batch_wall_started = profile::profile_now(collect_stage_timings); + let mut initial_budget = HostPhaseBudget::new("j2k CUDA color batch plan owners"); + let mut colors = initial_budget.try_vec_with_capacity(inputs.len())?; + let mut shared_payload = Vec::new(); + let mut native_context = NativeDecoderContext::default(); + for input in inputs { + let mut color = + build_cuda_htj2k_color_plans_from_bytes_with_profile(input, fmt, &mut native_context)?; + if color.components.len() != 3 { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + let mut append_budget = host_owners::color_batch_budget( + &colors, + &shared_payload, + Some(&color), + "j2k CUDA color batch plan owners", + )?; + append_color_payload_to_shared(&mut color, &mut shared_payload, &mut append_budget)?; + colors.push(color); + host_owners::color_batch_budget( + &colors, + &shared_payload, + None, + "j2k CUDA color batch plan owners", + )?; + } + + let context = session.cuda_context()?; + let pool = session.decode_batch_buffer_pool()?; + let table_upload_start = profile::profile_now(collect_stage_timings); + let table_resources = if colors.iter().all(|color| { + color + .components + .iter() + .all(|plan| plan.subbands().is_empty()) + }) { + None + } else { + Some(session.htj2k_decode_table_resources()?) + }; + let table_upload_us = profile::elapsed_us(table_upload_start); + let payload_upload_start = profile::profile_now(collect_stage_timings); + let decode_resources = match table_resources.as_ref() { + Some(tables) => context.upload_htj2k_decode_resources_with_tables_and_pool( + &shared_payload, + tables, + &pool, + ), + None => context.upload_j2k_decode_payload_with_pool(&shared_payload, &pool), + } + .map_err(cuda_error)?; + let payload_upload_us = profile::elapsed_us(payload_upload_start); + drop(shared_payload); + + let component_count = colors + .iter() + .map(|color| color.components.len()) + .sum::(); + let mut host_budget = HostPhaseBudget::new("j2k CUDA color batch execution graph"); + host_owners::account_colors(&mut host_budget, &colors)?; + let mut all_component_work = host_budget.try_vec_with_capacity(component_count)?; + for color in &colors { + for plan in &color.components { + all_component_work.push(decode_cuda_component_subbands_with_resources( + &context, + plan, + &pool, + collect_stage_timings, + &mut host_budget, + )?); + } + } + run_component_cleanup_dequant_batches( + &context, + &decode_resources, + &mut all_component_work, + &pool, + collect_stage_timings, + host_budget.live_bytes(), + )?; + let mut batch_components = host_budget.try_vec_with_capacity(component_count)?; + for color in &colors { + for component in &color.components { + batch_components.push(component); + } + } + let idwt_batched = can_batch_color_idwt(&batch_components); + let pending_idwt_batch = if idwt_batched { + run_color_component_idwt_batches( + &context, + &batch_components, + &mut all_component_work, + &pool, + collect_stage_timings, + host_budget.live_bytes(), + )? + } else { + None + }; + drop(batch_components); + + let completion_result = (|| { + let can_use_batch_store = + idwt_batched && can_batch_rgb8_mct_color_store(fmt, &colors, &all_component_work)?; + let (surfaces, reports) = if can_use_batch_store { + finish_color_cuda_resident_batch_surfaces_with_rgb8_mct_store( + &context, + fmt, + colors, + all_component_work, + collect_stage_timings, + )? + } else { + let mut output_budget = HostPhaseBudget::new("j2k CUDA color batch output graph"); + host_owners::account_colors(&mut output_budget, &colors)?; + host_owners::account_component_work(&mut output_budget, &all_component_work)?; + let mut surfaces = output_budget.try_vec_with_capacity(colors.len())?; + let mut reports = output_budget.try_vec_with_capacity(colors.len())?; + let mut work_iter = all_component_work.into_iter(); + for color in colors { + let component_count = color.components.len(); + let component_work = + take_component_work(&mut work_iter, component_count, &mut output_budget)?; + let (surface, report) = finish_color_cuda_resident_surface_with_component_work( + FinishColorCudaResidentSurfaceRequest { + context: &context, + pool: &pool, + fmt, + color, + component_work, + wall_started: None, + collect_stage_timings, + run_idwt: !idwt_batched, + emit_report: false, + }, + )?; + surfaces.push(surface); + reports.push(report); + } + (surfaces, reports) + }; + // Runtime MCT/store launches synchronize before returning, so a + // recorded dispatch is also a completion point for preceding IDWT. + let completion_established = reports.iter().any(|report| { + report.detail.mct_dispatch_count != 0 || report.detail.store_dispatch_count != 0 + }); + Ok(((surfaces, reports), completion_established)) + })(); + let (surfaces, reports) = CudaQueuedIdwtBatch::resolve_optional_after_completed_work( + pending_idwt_batch, + completion_result, + )?; + + let aggregate = finalize_color_batch_decode_report( + &reports, + table_upload_us, + payload_upload_us, + batch_wall_started, + ); + aggregate.emit("decode_batch"); + + Ok((surfaces, aggregate)) +} + +pub(in crate::decoder) fn finalize_color_batch_decode_report( + reports: &[CudaHtj2kProfileReport], + table_upload_us: u128, + payload_upload_us: u128, + batch_wall_started: Option, +) -> CudaHtj2kProfileReport { + let mut aggregate = aggregate_decode_reports(reports); + aggregate.h2d_us = aggregate + .h2d_us + .saturating_add(table_upload_us) + .saturating_add(payload_upload_us); + aggregate.detail.table_upload_us = aggregate + .detail + .table_upload_us + .saturating_add(table_upload_us); + aggregate.detail.payload_upload_us = aggregate + .detail + .payload_upload_us + .saturating_add(payload_upload_us); + aggregate.detail.wall_total_us = profile::elapsed_us(batch_wall_started); + profile::finalize_decode_total_us(&mut aggregate); + aggregate +} + +#[cfg(feature = "cuda-runtime")] +fn can_batch_rgb8_mct_color_store( + fmt: PixelFormat, + colors: &[CudaHtj2kColorDecodePlans], + all_component_work: &[CudaComponentDecodeWork], +) -> Result { + if !matches!(fmt, PixelFormat::Rgb8 | PixelFormat::Rgba8) { + return Ok(false); + } + + let mut offset = 0usize; + for color in colors { + let component_count = color.components.len(); + if component_count != 3 || offset.saturating_add(component_count) > all_component_work.len() + { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + if !color.mct { + return Ok(false); + } + let component_work = &all_component_work[offset..offset + component_count]; + let stores = [ + &component_work[0].store, + &component_work[1].store, + &component_work[2].store, + ]; + validate_color_stores(stores, color.dimensions)?; + if !can_fuse_mct_store_for_stores(stores) { + return Ok(false); + } + offset = offset.saturating_add(component_count); + } + + if offset != all_component_work.len() { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + Ok(!colors.is_empty()) +} + +#[cfg(feature = "cuda-runtime")] +fn finish_color_cuda_resident_batch_surfaces_with_rgb8_mct_store( + context: &j2k_cuda_runtime::CudaContext, + fmt: PixelFormat, + colors: Vec, + all_component_work: Vec, + collect_stage_timings: bool, +) -> Result<(Vec, Vec), Error> { + let mut host_budget = HostPhaseBudget::new("j2k CUDA prepared color batch store graph"); + host_owners::account_colors(&mut host_budget, &colors)?; + host_owners::account_component_work(&mut host_budget, &all_component_work)?; + let mut prepared = host_budget.try_vec_with_capacity(colors.len())?; + let mut work_iter = all_component_work.into_iter(); + for color in colors { + let component_count = color.components.len(); + let component_work = + take_component_work(&mut work_iter, component_count, &mut host_budget)?; + prepared.push(prepare_rgb8_mct_batch_store(fmt, color, component_work)?); + } + if work_iter.next().is_some() { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + + let targets = + host_budget.try_collect_results_exact(prepared.iter().map(rgb8_mct_batch_store_target))?; + let (store_output, store_us) = context + .time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.store.color.batch", + || { + context.j2k_store_rgb8_mct_batch_contiguous_device_with_live_host_bytes( + &targets, + host_budget.live_bytes(), + ) + }, + ) + .map_err(cuda_error)?; + drop(targets); + let (surface_buffer, surface_ranges, store_stats) = store_output.into_parts(); + if surface_ranges.len() != prepared.len() { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + let shared_surface_buffer = Arc::new(surface_buffer); + + let mut output_budget = HostPhaseBudget::new("j2k CUDA stored color batch output graph"); + output_budget.account_vec(&prepared)?; + for item in &prepared { + item.color.account_host_owners(&mut output_budget)?; + } + output_budget.account_vec(&surface_ranges)?; + let mut surfaces = output_budget.try_vec_with_capacity(prepared.len())?; + let mut reports = output_budget.try_vec_with_capacity(prepared.len())?; + let store_dispatches = store_stats.kernel_dispatches(); + let store_decode_dispatches = store_stats.decode_kernel_dispatches(); + for (index, (mut prepared, surface_range)) in + prepared.into_iter().zip(surface_ranges).enumerate() + { + let report_store_dispatches = if index == 0 { store_dispatches } else { 0 }; + let report_store_decode_dispatches = if index == 0 { + store_decode_dispatches + } else { + 0 + }; + let report_store_us = if index == 0 { store_us } else { 0 }; + let dispatches = prepared.dispatches.saturating_add(report_store_dispatches); + let decode_dispatches = prepared + .decode_dispatches + .saturating_add(report_store_decode_dispatches); + prepared.color.report.dispatch_count = dispatches; + prepared.color.report.store_us = prepared + .color + .report + .store_us + .saturating_add(report_store_us); + prepared.color.report.detail.store_dispatch_count = prepared + .color + .report + .detail + .store_dispatch_count + .saturating_add(report_store_dispatches); + profile::finalize_decode_total_us(&mut prepared.color.report); + + let dimensions = prepared.color.dimensions; + surfaces.push(Surface { + backend: BackendKind::Cuda, + residency: SurfaceResidency::CudaResidentDecode, + dimensions, + fmt, + pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), + stats: CudaSurfaceStats { + total: dispatches, + copy: 0, + decode: decode_dispatches, + }, + storage: cuda_range_storage( + shared_surface_buffer.clone(), + surface_range.offset, + surface_range.len, + ), + }); + reports.push(prepared.color.report); + } + + Ok((surfaces, reports)) +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/finish.rs b/crates/j2k-cuda/src/decoder/color_batch/finish.rs index 376438d9..79fd734c 100644 --- a/crates/j2k-cuda/src/decoder/color_batch/finish.rs +++ b/crates/j2k-cuda/src/decoder/color_batch/finish.rs @@ -1,10 +1,16 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +mod component; +mod surface; + +use self::{ + component::{finish_color_components, run_pending_color_idwt}, + surface::{finalize_color_surface, FinalizeColorSurfaceRequest}, +}; use super::{ - cuda_error, dispatch_color_store, finalize_color_surface, finish_color_components, host_owners, - pooled_cuda_buffer, run_color_mct, run_pending_color_idwt, ColorStoreInputs, - CudaHtj2kProfileReport, CudaQueuedIdwtBatch, Error, FinalizeColorSurfaceRequest, - FinishColorCudaResidentSurfaceRequest, Surface, + cuda_error, dispatch_color_store, host_owners, pooled_cuda_buffer, profile, run_color_mct, + ColorStoreInputs, CudaBufferPool, CudaComponentDecodeWork, CudaHtj2kColorDecodePlans, + CudaHtj2kProfileReport, CudaQueuedIdwtBatch, Error, PixelFormat, Surface, }; pub(super) fn finish_color_cuda_resident_surface_with_component_work( @@ -49,7 +55,7 @@ pub(super) fn finish_color_cuda_resident_surface_with_component_work( &prepared.components[1].store, &prepared.components[2].store, ], - bit_depths: color.bit_depths, + bit_depths: color.rgb_bit_depths(), }; let mct = run_color_mct( inputs, @@ -97,3 +103,16 @@ pub(super) fn finish_color_cuda_resident_surface_with_component_work( completion_result, ) } + +#[cfg(feature = "cuda-runtime")] +pub(super) struct FinishColorCudaResidentSurfaceRequest<'a> { + pub(super) context: &'a j2k_cuda_runtime::CudaContext, + pub(super) pool: &'a CudaBufferPool, + pub(super) fmt: PixelFormat, + pub(super) color: CudaHtj2kColorDecodePlans, + pub(super) component_work: Vec, + pub(super) wall_started: Option, + pub(super) collect_stage_timings: bool, + pub(super) run_idwt: bool, + pub(super) emit_report: bool, +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/finish/component.rs b/crates/j2k-cuda/src/decoder/color_batch/finish/component.rs new file mode 100644 index 00000000..3805ab8c --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/finish/component.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::{ + can_batch_color_idwt, finish_cuda_component_decode, run_color_component_idwt_batches, + run_cuda_component_idwt_steps, validate_color_stores, CudaBufferPool, CudaComponentDecodeWork, + CudaDecodedComponent, CudaHtj2kColorDecodePlans, CudaQueuedIdwtBatch, Error, HostPhaseBudget, + CUDA_HTJ2K_KERNELS_NOT_READY, +}; + +pub(super) struct PreparedColorComponents { + pub(super) components: [CudaDecodedComponent; 3], + pub(super) dispatches: usize, + pub(super) decode_dispatches: usize, +} + +pub(super) fn run_pending_color_idwt( + context: &j2k_cuda_runtime::CudaContext, + color: &CudaHtj2kColorDecodePlans, + component_work: &mut [CudaComponentDecodeWork], + pool: &CudaBufferPool, + collect_stage_timings: bool, + host_budget: &mut HostPhaseBudget, +) -> Result, Error> { + let mut batch_components = host_budget.try_vec_with_capacity(color.components.len())?; + for component in &color.components { + batch_components.push(component); + } + if can_batch_color_idwt(&batch_components) { + run_color_component_idwt_batches( + context, + &batch_components, + component_work, + pool, + collect_stage_timings, + host_budget.live_bytes(), + ) + } else { + for (plan, work) in color.components.iter().zip(component_work.iter_mut()) { + run_cuda_component_idwt_steps( + context, + plan.idwt_steps(), + work, + pool, + collect_stage_timings, + )?; + } + Ok(None) + } +} + +pub(super) fn finish_color_components( + component_work: Vec, + color: &mut CudaHtj2kColorDecodePlans, +) -> Result { + let [work0, work1, work2]: [CudaComponentDecodeWork; 3] = + component_work + .try_into() + .map_err(|_| Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + let components = [ + finish_cuda_component_decode(work0)?, + finish_cuda_component_decode(work1)?, + finish_cuda_component_decode(work2)?, + ]; + let stores = [ + &components[0].store, + &components[1].store, + &components[2].store, + ]; + validate_color_stores(stores, color.dimensions)?; + + let dispatches = components + .iter() + .map(|component| component.dispatches) + .sum::(); + let decode_dispatches = components + .iter() + .map(|component| component.decode_dispatches) + .sum::(); + for component in &components { + component.timings.add_to_report(&mut color.report); + } + Ok(PreparedColorComponents { + components, + dispatches, + decode_dispatches, + }) +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/finish/surface.rs b/crates/j2k-cuda/src/decoder/color_batch/finish/surface.rs new file mode 100644 index 00000000..a10a9b5c --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/finish/surface.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::{ + profile, BackendKind, CudaDeviceBuffer, CudaExecutionStats, CudaHtj2kColorDecodePlans, + CudaHtj2kProfileReport, CudaSurfaceStats, PixelFormat, Storage, Surface, SurfaceResidency, +}; + +pub(super) struct FinalizeColorSurfaceRequest { + pub(super) fmt: PixelFormat, + pub(super) color: CudaHtj2kColorDecodePlans, + pub(super) surface_buffer: CudaDeviceBuffer, + pub(super) dispatches: usize, + pub(super) decode_dispatches: usize, + pub(super) store_stats: CudaExecutionStats, + pub(super) store_us: u128, + pub(super) wall_started: Option, + pub(super) emit_report: bool, +} + +pub(super) fn finalize_color_surface( + request: FinalizeColorSurfaceRequest, +) -> (Surface, CudaHtj2kProfileReport) { + let FinalizeColorSurfaceRequest { + fmt, + mut color, + surface_buffer, + mut dispatches, + mut decode_dispatches, + store_stats, + store_us, + wall_started, + emit_report, + } = request; + dispatches = dispatches.saturating_add(store_stats.kernel_dispatches()); + decode_dispatches = decode_dispatches.saturating_add(store_stats.decode_kernel_dispatches()); + color.report.dispatch_count = dispatches; + color.report.store_us = color.report.store_us.saturating_add(store_us); + color.report.detail.store_dispatch_count = color + .report + .detail + .store_dispatch_count + .saturating_add(store_stats.kernel_dispatches()); + color.report.detail.wall_total_us = profile::elapsed_us(wall_started); + profile::finalize_decode_total_us(&mut color.report); + if emit_report { + color.report.emit("decode"); + } + let surface = Surface { + backend: BackendKind::Cuda, + residency: SurfaceResidency::CudaResidentDecode, + dimensions: color.dimensions, + fmt, + pitch_bytes: color.dimensions.0 as usize * fmt.bytes_per_pixel(), + stats: CudaSurfaceStats { + total: dispatches, + copy: 0, + decode: decode_dispatches, + }, + storage: Storage::Cuda(surface_buffer), + }; + (surface, color.report) +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/native_batch.rs b/crates/j2k-cuda/src/decoder/color_batch/native_batch.rs new file mode 100644 index 00000000..40a8cdc9 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/native_batch.rs @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +mod completion; +mod execution; +mod prepare; +mod store; + +use j2k::{BatchLayout, DeviceDecodePlan}; +use j2k_core::{DeviceSubmitSession, PixelFormat}; +use j2k_cuda_runtime::{ + CudaDeviceBuffer, CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut, + CudaQueuedJ2kStoreBatch, +}; + +use super::{ + append_color_payload_to_shared, can_batch_color_idwt, cuda_error, + finalize_color_batch_decode_report, host_owners, profile, run_color_component_idwt_batches, + CudaHtj2kColorDecodePlans, CudaHtj2kProfileReport, CudaQueuedIdwtBatch, CudaSession, Error, + NativeDecoderContext, +}; +use crate::allocation::HostPhaseBudget; +use crate::decoder::pending_completion::PendingDecodeCompletion; +use crate::decoder::plan::{ + build_cuda_classic_color_plans_from_referenced_with_profile, + build_cuda_color_plan_from_bytes_for_device_plan_with_profile, + build_cuda_htj2k_color_plans_from_referenced_with_profile, +}; +use crate::decoder::resident::{ + decode_cuda_component_subbands_with_resources, enqueue_chunked_htj2k_cleanup_dequant, + enqueue_component_classic_batches, ChunkedHtj2kCleanup, +}; +use crate::decoder::CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED; + +use self::completion::{ + NativeColorBatchOutput, NativeColorPendingCompletion, StoredNativeColorBatch, +}; +pub(crate) use self::completion::{ + SubmittedNativeColorExternalBatch, SubmittedNativeColorResidentBatch, +}; +use self::execution::decode_native_color_batch; +use self::prepare::{prepare_native_color_batch, PreparedNativeColorBatch}; +use self::store::finish_and_store_native_color; + +#[derive(Clone, Copy)] +pub(crate) struct NativeColorBatchInput<'a> { + pub(crate) source_index: usize, + pub(crate) bytes: &'a [u8], + pub(crate) device_plan: DeviceDecodePlan, + pub(crate) referenced_plan: Option<&'a j2k_native::J2kReferencedHtj2kPlan>, + pub(crate) referenced_classic_plan: Option<&'a j2k_native::J2kReferencedClassicPlan>, + pub(crate) settings: j2k_native::DecodeSettings, +} + +pub(crate) struct NativeColorOwnedBatch { + pub(crate) buffer: CudaDeviceBuffer, + pub(crate) ranges: Vec, + pub(crate) execution: j2k_cuda_runtime::CudaExecutionStats, +} + +pub(crate) fn submit_native_color_resident_prepared_batch_into( + inputs: &[NativeColorBatchInput<'_>], + session: &mut CudaSession, + fmt: PixelFormat, + layout: BatchLayout, + destination: &mut CudaExternalDeviceBufferViewMut<'_>, +) -> Result { + let (output, report, completion) = + decode_native_color_batch(inputs, session, fmt, layout, Some(destination), true)?; + let NativeColorBatchOutput::External(ranges) = output else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }); + }; + let completion = completion.ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA exact RGB submission did not retain a completion owner", + })?; + Ok(SubmittedNativeColorExternalBatch { + ranges, + report, + completion: Some(completion), + }) +} + +pub(crate) fn submit_native_color_resident_prepared_batch( + inputs: &[NativeColorBatchInput<'_>], + session: &mut CudaSession, + fmt: PixelFormat, + layout: BatchLayout, +) -> Result { + let (output, report, completion) = + decode_native_color_batch(inputs, session, fmt, layout, None, true)?; + let NativeColorBatchOutput::Owned(output) = output else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }); + }; + let completion = completion.ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA resident RGB submission did not retain a completion owner", + })?; + Ok(SubmittedNativeColorResidentBatch { + output: Some(output), + report, + completion: Some(completion), + }) +} + +#[cfg(test)] +mod tests { + use std::{num::NonZeroUsize, sync::Arc}; + + use j2k::{ + prepare_batch, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, + DeviceDecodePlan, DeviceDecodeRequest, EncodedImage, + }; + use j2k_core::{Downscale, HtGpuJobChunkLimits, PixelFormat, Rect}; + use j2k_cuda_runtime::{htj2k_cleanup_multi_descriptor_bytes, CudaContext}; + use j2k_native::{encode_htj2k, DecodeSettings, EncodeOptions, Image}; + + use super::{ + prepare_native_color_batch, submit_native_color_resident_prepared_batch, + NativeColorBatchInput, + }; + use crate::CudaSession; + + #[test] + fn retained_rgb_plan_prepares_full_roi_and_reduced_without_reparse() { + let pixels = (0_u16..16 * 16 * 3) + .map(|value| u8::try_from(value & 0x7f).unwrap()) + .collect::>(); + let encoded = encode_htj2k( + &pixels, + 16, + 16, + 3, + 7, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }, + ) + .expect("RGB HTJ2K fixture"); + for request in [ + DeviceDecodeRequest::Full, + DeviceDecodeRequest::Region { + roi: Rect { + x: 2, + y: 3, + w: 8, + h: 7, + }, + }, + DeviceDecodeRequest::Scaled { + scale: Downscale::Half, + }, + DeviceDecodeRequest::RegionScaled { + roi: Rect { + x: 1, + y: 2, + w: 6, + h: 5, + }, + scale: Downscale::Half, + }, + ] { + let plan = DeviceDecodePlan::for_image((16, 16), request).unwrap(); + let mut settings = DecodeSettings::strict(); + settings.target_resolution = (plan.scale() != Downscale::None).then_some(( + plan.source_dims().0.div_ceil(plan.scale().denominator()), + plan.source_dims().1.div_ceil(plan.scale().denominator()), + )); + let image = Image::new(&encoded, &settings).expect("parse RGB fixture"); + let output = plan.output_rect(); + let mut context = j2k_native::DecoderContext::default(); + let referenced = image + .build_referenced_htj2k_plan_region_with_context( + &mut context, + (output.x, output.y, output.w, output.h), + ) + .expect("referenced RGB plan"); + let prepared = prepare_native_color_batch( + &[NativeColorBatchInput { + source_index: 0, + bytes: &encoded, + device_plan: plan, + referenced_plan: Some(&referenced), + referenced_classic_plan: None, + settings: DecodeSettings::strict(), + }], + PixelFormat::Rgb8, + ) + .expect("prepare retained CUDA RGB plan"); + assert_eq!(prepared.colors[0].dimensions, plan.output_dims()); + assert_eq!(prepared.colors[0].report.parse_us, 0); + assert!(!prepared.shared_payload.is_empty()); + } + let _ = BatchLayout::Nhwc; + } + + #[test] + fn tiny_caps_force_multiple_exact_rgb_chunks_without_changing_output_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-rgb-s12-53-single-raw") + .expect("independent signed RGB12 single-tile fixture"); + let encoded = Arc::<[u8]>::from(fixture.encoded); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch(vec![EncodedImage::full(Arc::clone(&encoded))], options) + .expect("prepare independent signed RGB12 fixture"); + let image = &prepared.groups()[0].images()[0]; + let referenced_plan = image + .htj2k_plan() + .expect("retained HTJ2K plan") + .adapter_view() + .downcast_ref::() + .expect("native referenced HTJ2K plan adapter"); + + let mut cpu = CpuBatchDecoder::new(options); + let oracle = cpu + .decode_prepared(&prepared) + .expect("CPU signed RGB12 oracle"); + let CpuBatchSamples::I16(expected) = oracle.groups()[0].samples() else { + panic!("signed RGB12 oracle must use I16 storage") + }; + let expected = expected + .iter() + .flat_map(|sample| sample.to_ne_bytes()) + .collect::>(); + + let context = CudaContext::system_default().expect("CUDA context"); + let before = context + .diagnostics() + .expect("diagnostics before chunked decode"); + let mut session = CudaSession::with_context(context.clone()); + session.set_htj2k_decode_chunk_limits_for_test(HtGpuJobChunkLimits::new( + NonZeroUsize::MIN, + fixture.encoded.len(), + htj2k_cleanup_multi_descriptor_bytes(), + )); + let pending = submit_native_color_resident_prepared_batch( + &[NativeColorBatchInput { + source_index: 0, + bytes: &encoded, + device_plan: image.plan(), + referenced_plan: Some(referenced_plan), + referenced_classic_plan: None, + settings: DecodeSettings::strict(), + }], + &mut session, + PixelFormat::RgbI16, + BatchLayout::Nhwc, + ) + .expect("submit tiny-cap signed RGB12 chunks"); + let (output, _report) = pending + .finish() + .expect("finish tiny-cap signed RGB12 chunks"); + assert!(session.last_htj2k_decode_chunk_count_for_test() > 1); + let after = context + .diagnostics() + .expect("diagnostics after chunked decode"); + assert_eq!( + after.device_to_host_operations - before.device_to_host_operations, + 1, + "one homogeneous output group must validate all chunk statuses with one transfer" + ); + let mut actual = vec![0_u8; expected.len()]; + output + .buffer + .copy_range_to_host(output.ranges[0].offset, &mut actual) + .expect("download tiny-cap signed RGB12 output"); + assert_eq!(actual, expected); + } +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/native_batch/completion.rs b/crates/j2k-cuda/src/decoder/color_batch/native_batch/completion.rs new file mode 100644 index 00000000..ac9b9074 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/native_batch/completion.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + ChunkedHtj2kCleanup, CudaDeviceBufferRange, CudaHtj2kProfileReport, CudaQueuedJ2kStoreBatch, + Error, NativeColorOwnedBatch, PendingDecodeCompletion, +}; + +pub(super) enum NativeColorBatchOutput { + Owned(NativeColorOwnedBatch), + External(Vec), +} + +pub(super) struct StoredNativeColorBatch { + pub(super) output: NativeColorBatchOutput, + pub(super) queued: Option, +} + +pub(super) type NativeColorPendingCompletion = PendingDecodeCompletion; + +pub(crate) struct SubmittedNativeColorExternalBatch { + pub(super) ranges: Vec, + pub(super) report: CudaHtj2kProfileReport, + pub(super) completion: Option, +} + +pub(crate) struct SubmittedNativeColorResidentBatch { + pub(super) output: Option, + pub(super) report: CudaHtj2kProfileReport, + pub(super) completion: Option, +} + +impl SubmittedNativeColorResidentBatch { + pub(crate) fn is_complete(&self) -> Result { + self.completion + .as_ref() + .map_or(Ok(true), NativeColorPendingCompletion::is_complete) + } + + pub(crate) fn finish( + mut self, + ) -> Result<(NativeColorOwnedBatch, CudaHtj2kProfileReport), Error> { + if let Some(mut completion) = self.completion.take() { + if let Err(error) = completion.complete() { + if error.completion_is_uncertain() { + if let Some(output) = self.output.take() { + core::mem::forget(output); + } + } + return Err(error); + } + } + let output = self.output.take().ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA resident RGB submission lost its output owner", + })?; + Ok((output, self.report.clone())) + } +} + +impl Drop for SubmittedNativeColorResidentBatch { + fn drop(&mut self) { + let result = self + .completion + .take() + .map_or(Ok(()), |mut completion| completion.complete()); + if result.is_err_and(|error| error.completion_is_uncertain()) { + if let Some(output) = self.output.take() { + core::mem::forget(output); + } + } + } +} + +impl SubmittedNativeColorExternalBatch { + pub(crate) fn ranges(&self) -> &[CudaDeviceBufferRange] { + &self.ranges + } + + pub(crate) fn is_complete(&self) -> Result { + self.completion + .as_ref() + .map_or(Ok(true), NativeColorPendingCompletion::is_complete) + } + + pub(crate) fn finish( + mut self, + ) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + if let Some(mut completion) = self.completion.take() { + completion.complete()?; + } + Ok((self.ranges, self.report)) + } +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/native_batch/execution.rs b/crates/j2k-cuda/src/decoder/color_batch/native_batch/execution.rs new file mode 100644 index 00000000..263c6c2b --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/native_batch/execution.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[cfg(test)] +use super::ChunkedHtj2kCleanup; +use super::{ + can_batch_color_idwt, cuda_error, decode_cuda_component_subbands_with_resources, + enqueue_chunked_htj2k_cleanup_dequant, enqueue_component_classic_batches, + finalize_color_batch_decode_report, finish_and_store_native_color, host_owners, + prepare_native_color_batch, profile, run_color_component_idwt_batches, BatchLayout, + CudaExternalDeviceBufferViewMut, CudaHtj2kProfileReport, CudaQueuedIdwtBatch, CudaSession, + DeviceSubmitSession, Error, HostPhaseBudget, NativeColorBatchInput, NativeColorBatchOutput, + NativeColorPendingCompletion, PixelFormat, PreparedNativeColorBatch, +}; +use crate::decoder::pending_completion::{finish_decode_statuses, retire_decode_after_error}; + +#[expect( + clippy::too_many_lines, + reason = "exact RGB orchestration keeps shared arenas and asynchronous owner ordering atomic" +)] +pub(super) fn decode_native_color_batch( + inputs: &[NativeColorBatchInput<'_>], + session: &mut CudaSession, + fmt: PixelFormat, + layout: BatchLayout, + external: Option<&mut CudaExternalDeviceBufferViewMut<'_>>, + enqueue_external: bool, +) -> Result< + ( + NativeColorBatchOutput, + CudaHtj2kProfileReport, + Option, + ), + Error, +> { + if !matches!(layout, BatchLayout::Nhwc | BatchLayout::Nchw) { + return Err(Error::UnsupportedCudaRequest { + reason: "exact CUDA RGB batch layout must be NHWC or NCHW", + }); + } + let batch_wall_started = profile::profile_now(false); + let PreparedNativeColorBatch { + colors, + shared_payload, + source_indices, + } = prepare_native_color_batch(inputs, fmt)?; + let context = session.cuda_context()?; + let pool = session.decode_batch_buffer_pool()?; + let table_upload_start = profile::profile_now(false); + let table_resources = if colors.iter().all(|color| { + color + .components + .iter() + .all(|plan| plan.subbands().is_empty()) + }) { + None + } else { + Some(session.htj2k_decode_table_resources()?) + }; + let table_upload_us = profile::elapsed_us(table_upload_start); + let component_count = colors + .iter() + .try_fold(0usize, |count, color| { + count.checked_add(color.components.len()) + }) + .ok_or(Error::HostAllocationFailed { + bytes: usize::MAX, + what: "j2k CUDA exact color component work", + })?; + let mut host_budget = HostPhaseBudget::new("j2k CUDA exact RGB execution graph"); + host_owners::account_colors(&mut host_budget, &colors)?; + let mut component_work = host_budget.try_vec_with_capacity(component_count)?; + let mut component_source_indices = host_budget.try_vec_with_capacity(component_count)?; + for (color, source_index) in colors.iter().zip(source_indices.iter().copied()) { + for plan in &color.components { + component_work.push(decode_cuda_component_subbands_with_resources( + &context, + plan, + &pool, + false, + &mut host_budget, + )?); + component_source_indices.push(source_index); + } + } + let payload_upload_start = profile::profile_now(false); + let (pending_classic, classic_resources) = if component_work + .iter() + .any(|work| !work.pending_classic_bands.is_empty()) + { + let classic_resources = context + .upload_j2k_decode_payload_with_pool(&shared_payload, &pool) + .map_err(cuda_error)?; + let classic_tables = session.classic_decode_table_resources()?; + let pending = enqueue_component_classic_batches( + &context, + &classic_resources, + &classic_tables, + &mut component_work, + &component_source_indices, + &pool, + host_budget.live_bytes(), + )?; + (pending, Some(classic_resources)) + } else { + (None, None) + }; + let pending_cleanup = Some(enqueue_chunked_htj2k_cleanup_dequant( + &context, + table_resources.as_ref(), + &shared_payload, + &mut component_work, + &component_source_indices, + &pool, + session.htj2k_decode_chunk_limits(), + host_budget.live_bytes(), + )?); + let payload_upload_us = profile::elapsed_us(payload_upload_start); + #[cfg(test)] + session.record_htj2k_decode_chunk_count_for_test( + pending_cleanup + .as_ref() + .map_or(0, ChunkedHtj2kCleanup::chunk_count), + ); + drop(component_source_indices); + drop(shared_payload); + let mut component_plans = host_budget.try_vec_with_capacity(component_count)?; + for color in &colors { + component_plans.extend(color.components.iter()); + } + let idwt_batched = can_batch_color_idwt(&component_plans); + let pending_idwt = if idwt_batched { + run_color_component_idwt_batches( + &context, + &component_plans, + &mut component_work, + &pool, + false, + host_budget.live_bytes(), + )? + } else { + let mut pending: Option = None; + for (plan, work) in component_plans.iter().zip(component_work.iter_mut()) { + let plans = [*plan]; + let next = run_color_component_idwt_batches( + &context, + &plans, + std::slice::from_mut(work), + &pool, + false, + host_budget.live_bytes(), + )?; + pending = match (pending, next) { + (Some(current), Some(next)) => Some(current.merge(next)?), + (Some(current), None) => Some(current), + (None, next) => next, + }; + } + pending + }; + drop(component_plans); + + let completion_result = finish_and_store_native_color( + &context, + colors, + component_work, + fmt, + layout, + external, + enqueue_external, + ); + if enqueue_external { + let (stored, reports, decoded) = match completion_result { + Ok(output) => output, + Err(error) => { + return Err(retire_decode_after_error( + error, + pending_idwt, + pending_cleanup, + pending_classic, + )); + } + }; + let store = stored.queued.ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA exact RGB external store did not return a completion guard", + })?; + let pending = NativeColorPendingCompletion::new( + Some(store), + pending_idwt, + pending_cleanup, + pending_classic, + decoded, + classic_resources, + ); + let aggregate = finalize_color_batch_decode_report( + &reports, + table_upload_us, + payload_upload_us, + batch_wall_started, + ); + session.record_submit(); + return Ok((stored.output, aggregate, Some(pending))); + } + + let completion_result = completion_result.and_then(|(stored, reports, decoded)| { + if stored.queued.is_some() { + return Err(Error::UnsupportedCudaRequest { + reason: "synchronous exact CUDA RGB store unexpectedly returned pending work", + }); + } + Ok(((stored.output, reports, decoded), true)) + }); + let (output, reports, _decoded) = CudaQueuedIdwtBatch::resolve_optional_after_completed_work( + pending_idwt, + completion_result, + )?; + finish_decode_statuses(pending_cleanup, pending_classic)?; + drop(classic_resources); + let aggregate = finalize_color_batch_decode_report( + &reports, + table_upload_us, + payload_upload_us, + batch_wall_started, + ); + session.record_submit(); + Ok((output, aggregate, None)) +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/native_batch/prepare.rs b/crates/j2k-cuda/src/decoder/color_batch/native_batch/prepare.rs new file mode 100644 index 00000000..c1be1bef --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/native_batch/prepare.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + append_color_payload_to_shared, build_cuda_classic_color_plans_from_referenced_with_profile, + build_cuda_color_plan_from_bytes_for_device_plan_with_profile, + build_cuda_htj2k_color_plans_from_referenced_with_profile, host_owners, + CudaHtj2kColorDecodePlans, Error, HostPhaseBudget, NativeColorBatchInput, NativeDecoderContext, + PixelFormat, CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, +}; + +pub(super) struct PreparedNativeColorBatch { + pub(super) colors: Vec, + pub(super) shared_payload: Vec, + pub(super) source_indices: Vec, +} + +#[expect( + clippy::too_many_lines, + reason = "one preparation boundary keeps tile plans, shared payload ownership, and source identities aligned" +)] +pub(super) fn prepare_native_color_batch( + inputs: &[NativeColorBatchInput<'_>], + fmt: PixelFormat, +) -> Result { + if !matches!( + fmt, + PixelFormat::Rgb8 + | PixelFormat::Rgb16 + | PixelFormat::RgbI16 + | PixelFormat::Rgba8 + | PixelFormat::Rgba16 + | PixelFormat::RgbaI16 + ) { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }); + } + let mut initial_budget = HostPhaseBudget::new("j2k CUDA exact RGB batch plan owners"); + let mut colors = initial_budget.try_vec_with_capacity(inputs.len())?; + let mut source_indices = initial_budget.try_vec_with_capacity(inputs.len())?; + let mut shared_payload = Vec::new(); + let mut native_context = NativeDecoderContext::default(); + for (output_index, input) in inputs.iter().enumerate() { + let (mut input_colors, payload_is_shared) = + match (input.referenced_plan, input.referenced_classic_plan) { + (Some(referenced_plan), None) => { + let mut budget = host_owners::color_batch_budget( + &colors, + &shared_payload, + None, + "j2k CUDA referenced exact RGB batch plan owners", + )?; + ( + build_cuda_htj2k_color_plans_from_referenced_with_profile( + input.bytes, + referenced_plan, + fmt, + input.device_plan, + &mut shared_payload, + &mut budget, + )?, + true, + ) + } + (None, Some(referenced_plan)) => { + let mut budget = host_owners::color_batch_budget( + &colors, + &shared_payload, + None, + "j2k CUDA referenced classic color batch plan owners", + )?; + ( + build_cuda_classic_color_plans_from_referenced_with_profile( + input.bytes, + referenced_plan, + fmt, + input.device_plan, + &mut shared_payload, + &mut budget, + )?, + true, + ) + } + (None, None) => { + let mut color = build_cuda_color_plan_from_bytes_for_device_plan_with_profile( + input.bytes, + fmt, + input.device_plan, + input.settings, + &mut native_context, + )?; + let mut budget = host_owners::color_batch_budget( + &colors, + &shared_payload, + Some(&color), + "j2k CUDA exact classic RGB batch plan owners", + )?; + append_color_payload_to_shared(&mut color, &mut shared_payload, &mut budget)?; + let mut one = budget.try_vec_with_capacity(1)?; + one.push(color); + (one, true) + } + (Some(_), Some(_)) => { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA color input contains conflicting codec plans", + }); + } + }; + if !payload_is_shared { + return Err(Error::UnsupportedCudaRequest { + reason: "CUDA exact color plan payload ownership is inconsistent", + }); + } + let mut append_budget = host_owners::color_batch_budget( + &colors, + &shared_payload, + None, + "j2k CUDA exact color tile owner append", + )?; + append_budget.account_vec(&source_indices)?; + append_budget.try_vec_reserve(&mut colors, input_colors.len())?; + append_budget.try_vec_reserve(&mut source_indices, input_colors.len())?; + for color in &mut input_colors { + color.output_index = output_index; + } + source_indices.extend(std::iter::repeat_n(input.source_index, input_colors.len())); + colors.append(&mut input_colors); + } + Ok(PreparedNativeColorBatch { + colors, + shared_payload, + source_indices, + }) +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/native_batch/store.rs b/crates/j2k-cuda/src/decoder/color_batch/native_batch/store.rs new file mode 100644 index 00000000..3b43e9f9 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/native_batch/store.rs @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +mod jobs; +mod submission; + +use j2k::BatchLayout; +use j2k_core::PixelFormat; +use j2k_cuda_runtime::{ + CudaExternalDeviceBufferViewMut, CudaJ2kStoreRgbNativeTarget, CudaJ2kStoreRgbaNativeTarget, +}; + +use jobs::{native_level_shift, native_rgba_store_job, native_store_job, transform_selector}; +use submission::store_targets; + +use super::StoredNativeColorBatch; +use crate::decoder::color_batch::{ + finish_cuda_component_decode, pooled_cuda_buffer, validate_color_stores, + CudaComponentDecodeWork, CudaDecodedComponent, CudaHtj2kColorDecodePlans, + CudaHtj2kProfileReport, +}; +use crate::decoder::{Error, CUDA_HTJ2K_KERNELS_NOT_READY}; + +enum NativeColorStoreTargets<'a> { + Rgb(Vec>), + Rgba(Vec>), +} + +pub(super) fn finish_and_store_native_color( + context: &j2k_cuda_runtime::CudaContext, + colors: Vec, + component_work: Vec, + fmt: PixelFormat, + layout: BatchLayout, + external: Option<&mut CudaExternalDeviceBufferViewMut<'_>>, + enqueue_external: bool, +) -> Result< + ( + StoredNativeColorBatch, + Vec, + Vec, + ), + Error, +> { + let expected_components = colors.iter().try_fold(0usize, |count, color| { + count.checked_add(color.components.len()) + }); + let decoded = finish_components( + component_work, + expected_components.ok_or(Error::HostAllocationFailed { + bytes: usize::MAX, + what: "j2k CUDA exact color decoded component count", + })?, + )?; + let (targets, reports) = build_store_targets(colors, &decoded, fmt, layout)?; + let stored = store_targets(context, fmt, external, enqueue_external, &targets)?; + Ok((stored, reports, decoded)) +} + +fn finish_components( + component_work: Vec, + expected_components: usize, +) -> Result, Error> { + let mut decoded = Vec::new(); + decoded + .try_reserve_exact(component_work.len()) + .map_err(|_| Error::HostAllocationFailed { + bytes: component_work + .len() + .saturating_mul(std::mem::size_of::()), + what: "j2k CUDA exact RGB decoded components", + })?; + for work in component_work { + decoded.push(finish_cuda_component_decode(work)?); + } + if decoded.len() != expected_components { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + Ok(decoded) +} + +#[expect( + clippy::too_many_lines, + reason = "one target builder keeps RGB/RGBA plane ownership and exact final-store metadata aligned" +)] +fn build_store_targets( + colors: Vec, + decoded: &[CudaDecodedComponent], + fmt: PixelFormat, + layout: BatchLayout, +) -> Result<(NativeColorStoreTargets<'_>, Vec), Error> { + let mut reports = Vec::new(); + reports + .try_reserve_exact(colors.len()) + .map_err(|_| Error::HostAllocationFailed { + bytes: colors + .len() + .saturating_mul(std::mem::size_of::()), + what: "j2k CUDA exact RGB reports", + })?; + let rgba = matches!( + fmt, + PixelFormat::Rgba8 | PixelFormat::Rgba16 | PixelFormat::RgbaI16 + ); + let mut three_channel_targets = Vec::new(); + let mut four_channel_targets = Vec::new(); + if rgba { + four_channel_targets + .try_reserve_exact(colors.len()) + .map_err(|_| Error::HostAllocationFailed { + bytes: colors + .len() + .saturating_mul(std::mem::size_of::>()), + what: "j2k CUDA exact RGBA store targets", + })?; + } else { + three_channel_targets + .try_reserve_exact(colors.len()) + .map_err(|_| Error::HostAllocationFailed { + bytes: colors + .len() + .saturating_mul(std::mem::size_of::>()), + what: "j2k CUDA exact RGB store targets", + })?; + } + let mut component_offset = 0usize; + for (image_index, mut color) in colors.into_iter().enumerate() { + let component_count = color.components.len(); + let component_end = + component_offset + .checked_add(component_count) + .ok_or(Error::HostAllocationFailed { + bytes: usize::MAX, + what: "j2k CUDA exact color component range", + })?; + let components = + decoded + .get(component_offset..component_end) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + component_offset = component_end; + let transform = transform_selector(color.mct, color.transform); + if rgba { + let stores = [ + &components[0].store, + &components[1].store, + &components[2].store, + &components[3].store, + ]; + validate_color_stores(stores, color.dimensions)?; + let addends = if color.mct && fmt != PixelFormat::RgbaI16 { + let mut addends = color.bit_depths.map(native_level_shift); + addends[3] = stores[3].addend; + addends + } else { + stores.map(|store| store.addend) + }; + four_channel_targets.push(CudaJ2kStoreRgbaNativeTarget { + output_index: color.output_index, + plane0: pooled_cuda_buffer(&components[0].buffer)?, + plane1: pooled_cuda_buffer(&components[1].buffer)?, + plane2: pooled_cuda_buffer(&components[2].buffer)?, + plane3: pooled_cuda_buffer(&components[3].buffer)?, + job: native_rgba_store_job(stores, color.bit_depths, addends, layout, transform), + }); + } else { + let stores = [ + &components[0].store, + &components[1].store, + &components[2].store, + ]; + validate_color_stores(stores, color.dimensions)?; + let bit_depths = color.rgb_bit_depths(); + let addends = if color.mct && fmt != PixelFormat::RgbI16 { + bit_depths.map(native_level_shift) + } else { + stores.map(|store| store.addend) + }; + three_channel_targets.push(CudaJ2kStoreRgbNativeTarget { + output_index: color.output_index, + plane0: pooled_cuda_buffer(&components[0].buffer)?, + plane1: pooled_cuda_buffer(&components[1].buffer)?, + plane2: pooled_cuda_buffer(&components[2].buffer)?, + job: native_store_job(stores, bit_depths, addends, layout, transform), + }); + } + color.report.dispatch_count = components + .iter() + .map(|component| component.dispatches) + .sum::() + + usize::from(image_index == 0); + for component in components { + component.timings.add_to_report(&mut color.report); + } + reports.push(color.report); + } + let targets = if rgba { + NativeColorStoreTargets::Rgba(four_channel_targets) + } else { + NativeColorStoreTargets::Rgb(three_channel_targets) + }; + Ok((targets, reports)) +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/native_batch/store/jobs.rs b/crates/j2k-cuda/src/decoder/color_batch/native_batch/store/jobs.rs new file mode 100644 index 00000000..4ae50cdb --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/native_batch/store/jobs.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::BatchLayout; +use j2k_cuda_runtime::{CudaJ2kStoreRgbNativeJob, CudaJ2kStoreRgbaNativeJob}; + +use crate::decoder::CudaHtj2kTransform; + +pub(super) fn transform_selector(mct: bool, transform: CudaHtj2kTransform) -> u32 { + if !mct { + return 0; + } + match transform { + CudaHtj2kTransform::Reversible53 => 1, + CudaHtj2kTransform::Irreversible97 => 2, + } +} + +pub(super) fn native_store_job( + stores: [&crate::CudaHtj2kStoreStep; 3], + bit_depths: [u8; 3], + addends: [f32; 3], + layout: BatchLayout, + transform: u32, +) -> CudaJ2kStoreRgbNativeJob { + let input_width = + |store: &crate::CudaHtj2kStoreStep| store.input_rect.x1.saturating_sub(store.input_rect.x0); + CudaJ2kStoreRgbNativeJob { + input_width0: input_width(stores[0]), + input_width1: input_width(stores[1]), + input_width2: input_width(stores[2]), + source_x0: stores[0].source_x, + source_y0: stores[0].source_y, + source_x1: stores[1].source_x, + source_y1: stores[1].source_y, + source_x2: stores[2].source_x, + source_y2: stores[2].source_y, + copy_width: stores[0].copy_width, + copy_height: stores[0].copy_height, + output_width: stores[0].output_width, + output_height: stores[0].output_height, + output_x: stores[0].output_x, + output_y: stores[0].output_y, + addend0: addends[0], + addend1: addends[1], + addend2: addends[2], + bit_depth0: u32::from(bit_depths[0]), + bit_depth1: u32::from(bit_depths[1]), + bit_depth2: u32::from(bit_depths[2]), + layout: match layout { + BatchLayout::Nhwc => 0, + BatchLayout::Nchw => 1, + _ => u32::MAX, + }, + transform, + reserved: 0, + } +} + +pub(super) fn native_rgba_store_job( + stores: [&crate::CudaHtj2kStoreStep; 4], + bit_depths: [u8; 4], + addends: [f32; 4], + layout: BatchLayout, + transform: u32, +) -> CudaJ2kStoreRgbaNativeJob { + let input_width = + |store: &crate::CudaHtj2kStoreStep| store.input_rect.x1.saturating_sub(store.input_rect.x0); + CudaJ2kStoreRgbaNativeJob { + input_width0: input_width(stores[0]), + input_width1: input_width(stores[1]), + input_width2: input_width(stores[2]), + input_width3: input_width(stores[3]), + source_x0: stores[0].source_x, + source_y0: stores[0].source_y, + source_x1: stores[1].source_x, + source_y1: stores[1].source_y, + source_x2: stores[2].source_x, + source_y2: stores[2].source_y, + source_x3: stores[3].source_x, + source_y3: stores[3].source_y, + copy_width: stores[0].copy_width, + copy_height: stores[0].copy_height, + output_width: stores[0].output_width, + output_height: stores[0].output_height, + output_x: stores[0].output_x, + output_y: stores[0].output_y, + addend0: addends[0], + addend1: addends[1], + addend2: addends[2], + addend3: addends[3], + bit_depth0: u32::from(bit_depths[0]), + bit_depth1: u32::from(bit_depths[1]), + bit_depth2: u32::from(bit_depths[2]), + bit_depth3: u32::from(bit_depths[3]), + layout: match layout { + BatchLayout::Nhwc => 0, + BatchLayout::Nchw => 1, + _ => u32::MAX, + }, + transform, + reserved: 0, + } +} + +pub(super) fn native_level_shift(bit_depth: u8) -> f32 { + f32::from(1_u16 << bit_depth.saturating_sub(1).min(15)) +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/native_batch/store/submission.rs b/crates/j2k-cuda/src/decoder/color_batch/native_batch/store/submission.rs new file mode 100644 index 00000000..8a87274e --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/native_batch/store/submission.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_core::PixelFormat; +use j2k_cuda_runtime::CudaExternalDeviceBufferViewMut; + +use super::NativeColorStoreTargets; +use crate::decoder::color_batch::cuda_error; +use crate::decoder::color_batch::native_batch::{ + NativeColorBatchOutput, NativeColorOwnedBatch, StoredNativeColorBatch, +}; +use crate::decoder::{Error, CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED}; + +#[expect( + clippy::too_many_lines, + reason = "the exhaustive dtype/channel/ownership dispatch matrix is one auditable final-store boundary" +)] +pub(super) fn store_targets( + context: &j2k_cuda_runtime::CudaContext, + fmt: PixelFormat, + external: Option<&mut CudaExternalDeviceBufferViewMut<'_>>, + enqueue_external: bool, + targets: &NativeColorStoreTargets<'_>, +) -> Result { + match (fmt, external, targets) { + (PixelFormat::Rgb8, Some(destination), NativeColorStoreTargets::Rgb(targets)) + if enqueue_external => + { + // SAFETY: the returned high-level owner retains decoded planes, + // uploaded store metadata, and the caller-owned destination. + let (ranges, queued) = unsafe { + context + .j2k_store_rgb8_native_batch_into_external_device_enqueue(targets, destination) + } + .map_err(cuda_error)?; + Ok(StoredNativeColorBatch { + output: NativeColorBatchOutput::External(ranges), + queued: Some(queued), + }) + } + (PixelFormat::Rgb16, Some(destination), NativeColorStoreTargets::Rgb(targets)) + if enqueue_external => + { + // SAFETY: same retained ownership as the U8 branch. + let (ranges, queued) = unsafe { + context + .j2k_store_rgb16_native_batch_into_external_device_enqueue(targets, destination) + } + .map_err(cuda_error)?; + Ok(StoredNativeColorBatch { + output: NativeColorBatchOutput::External(ranges), + queued: Some(queued), + }) + } + (PixelFormat::RgbI16, Some(destination), NativeColorStoreTargets::Rgb(targets)) + if enqueue_external => + { + // SAFETY: same retained ownership as the unsigned branches. + let (ranges, queued) = unsafe { + context.j2k_store_rgbi16_native_batch_into_external_device_enqueue( + targets, + destination, + ) + } + .map_err(cuda_error)?; + Ok(StoredNativeColorBatch { + output: NativeColorBatchOutput::External(ranges), + queued: Some(queued), + }) + } + (PixelFormat::Rgba8, Some(destination), NativeColorStoreTargets::Rgba(targets)) + if enqueue_external => + { + // SAFETY: the pending owner retains decoded planes and metadata. + let (ranges, queued) = unsafe { + context + .j2k_store_rgba8_native_batch_into_external_device_enqueue(targets, destination) + } + .map_err(cuda_error)?; + Ok(StoredNativeColorBatch { + output: NativeColorBatchOutput::External(ranges), + queued: Some(queued), + }) + } + (PixelFormat::Rgba16, Some(destination), NativeColorStoreTargets::Rgba(targets)) + if enqueue_external => + { + // SAFETY: the pending owner retains decoded planes and metadata. + let (ranges, queued) = unsafe { + context.j2k_store_rgba16_native_batch_into_external_device_enqueue( + targets, + destination, + ) + } + .map_err(cuda_error)?; + Ok(StoredNativeColorBatch { + output: NativeColorBatchOutput::External(ranges), + queued: Some(queued), + }) + } + (PixelFormat::RgbaI16, Some(destination), NativeColorStoreTargets::Rgba(targets)) + if enqueue_external => + { + // SAFETY: the pending owner retains decoded planes and metadata. + let (ranges, queued) = unsafe { + context.j2k_store_rgbai16_native_batch_into_external_device_enqueue( + targets, + destination, + ) + } + .map_err(cuda_error)?; + Ok(StoredNativeColorBatch { + output: NativeColorBatchOutput::External(ranges), + queued: Some(queued), + }) + } + (PixelFormat::Rgb8, None, NativeColorStoreTargets::Rgb(targets)) if enqueue_external => { + context + .j2k_store_rgb8_native_batch_contiguous_device_enqueue(targets) + .map_err(cuda_error) + .map(queued_owned_store) + } + (PixelFormat::Rgb16, None, NativeColorStoreTargets::Rgb(targets)) if enqueue_external => { + context + .j2k_store_rgb16_native_batch_contiguous_device_enqueue(targets) + .map_err(cuda_error) + .map(queued_owned_store) + } + (PixelFormat::RgbI16, None, NativeColorStoreTargets::Rgb(targets)) if enqueue_external => { + context + .j2k_store_rgbi16_native_batch_contiguous_device_enqueue(targets) + .map_err(cuda_error) + .map(queued_owned_store) + } + (PixelFormat::Rgba8, None, NativeColorStoreTargets::Rgba(targets)) if enqueue_external => { + context + .j2k_store_rgba8_native_batch_contiguous_device_enqueue(targets) + .map_err(cuda_error) + .map(queued_owned_store) + } + (PixelFormat::Rgba16, None, NativeColorStoreTargets::Rgba(targets)) if enqueue_external => { + context + .j2k_store_rgba16_native_batch_contiguous_device_enqueue(targets) + .map_err(cuda_error) + .map(queued_owned_store) + } + (PixelFormat::RgbaI16, None, NativeColorStoreTargets::Rgba(targets)) + if enqueue_external => + { + context + .j2k_store_rgbai16_native_batch_contiguous_device_enqueue(targets) + .map_err(cuda_error) + .map(queued_owned_store) + } + (PixelFormat::Rgb8, None, NativeColorStoreTargets::Rgb(targets)) => context + .j2k_store_rgb8_native_batch_contiguous_device(targets) + .map_err(cuda_error) + .map(owned_store), + (PixelFormat::Rgb16, None, NativeColorStoreTargets::Rgb(targets)) => context + .j2k_store_rgb16_native_batch_contiguous_device(targets) + .map_err(cuda_error) + .map(owned_store), + (PixelFormat::RgbI16, None, NativeColorStoreTargets::Rgb(targets)) => context + .j2k_store_rgbi16_native_batch_contiguous_device(targets) + .map_err(cuda_error) + .map(owned_store), + (PixelFormat::Rgba8, None, NativeColorStoreTargets::Rgba(targets)) => context + .j2k_store_rgba8_native_batch_contiguous_device(targets) + .map_err(cuda_error) + .map(owned_store), + (PixelFormat::Rgba16, None, NativeColorStoreTargets::Rgba(targets)) => context + .j2k_store_rgba16_native_batch_contiguous_device(targets) + .map_err(cuda_error) + .map(owned_store), + (PixelFormat::RgbaI16, None, NativeColorStoreTargets::Rgba(targets)) => context + .j2k_store_rgbai16_native_batch_contiguous_device(targets) + .map_err(cuda_error) + .map(owned_store), + _ => Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }), + } +} + +fn queued_owned_store( + (buffer, ranges, queued): ( + j2k_cuda_runtime::CudaDeviceBuffer, + Vec, + j2k_cuda_runtime::CudaQueuedJ2kStoreBatch, + ), +) -> StoredNativeColorBatch { + let execution = queued.execution(); + StoredNativeColorBatch { + output: NativeColorBatchOutput::Owned(NativeColorOwnedBatch { + buffer, + ranges, + execution, + }), + queued: Some(queued), + } +} + +fn owned_store( + output: j2k_cuda_runtime::CudaKernelContiguousBatchOutput, +) -> StoredNativeColorBatch { + let (buffer, ranges, execution) = output.into_parts(); + StoredNativeColorBatch { + output: NativeColorBatchOutput::Owned(NativeColorOwnedBatch { + buffer, + ranges, + execution, + }), + queued: None, + } +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/single.rs b/crates/j2k-cuda/src/decoder/color_batch/single.rs new file mode 100644 index 00000000..8cd76612 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/color_batch/single.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + cuda_error, decode_cuda_component_subbands_with_resources, + finish_color_cuda_resident_surface_with_component_work, profile, + run_component_cleanup_dequant_batches, CudaHtj2kColorDecodePlans, CudaHtj2kProfileReport, + CudaSession, Error, FinishColorCudaResidentSurfaceRequest, HostPhaseBudget, J2kDecoder, + PixelFormat, Rect, Surface, CUDA_HTJ2K_KERNELS_NOT_READY, +}; + +pub(in crate::decoder) fn decode_color_cuda_resident_surface_with_profile( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + wall_started: Option, + collect_stage_timings: bool, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + let color = decoder.build_cuda_htj2k_color_plans_with_profile(fmt)?; + decode_color_cuda_resident_surface_with_plans_profile( + session, + fmt, + color, + wall_started, + collect_stage_timings, + ) +} + +#[cfg(feature = "cuda-runtime")] +pub(in crate::decoder) fn decode_color_cuda_resident_scaled_surface( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + output_dimensions: (u32, u32), +) -> Result { + let collect_stage_timings = profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + let color = decoder.build_cuda_htj2k_color_scaled_plans_with_profile(fmt, output_dimensions)?; + decode_color_cuda_resident_surface_with_plans_profile( + session, + fmt, + color, + wall_started, + collect_stage_timings, + ) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +pub(in crate::decoder) fn decode_color_cuda_resident_region_surface( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + roi: Rect, +) -> Result { + let collect_stage_timings = profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + let color = decoder.build_cuda_htj2k_color_region_plans_with_profile(fmt, roi)?; + decode_color_cuda_resident_surface_with_plans_profile( + session, + fmt, + color, + wall_started, + collect_stage_timings, + ) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +pub(in crate::decoder) fn decode_color_cuda_resident_region_scaled_surface( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + scaled_roi: Rect, + scaled_dimensions: (u32, u32), +) -> Result { + let collect_stage_timings = profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + let color = decoder.build_cuda_htj2k_color_region_scaled_plans_with_profile( + fmt, + scaled_roi, + scaled_dimensions, + )?; + decode_color_cuda_resident_surface_with_plans_profile( + session, + fmt, + color, + wall_started, + collect_stage_timings, + ) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_color_cuda_resident_surface_with_plans_profile( + session: &mut CudaSession, + fmt: PixelFormat, + mut color: CudaHtj2kColorDecodePlans, + wall_started: Option, + collect_stage_timings: bool, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + if color.components.len() != 3 { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + let context = session.cuda_context()?; + let pool = session.decode_buffer_pool()?; + let table_upload_start = profile::profile_now(collect_stage_timings); + let table_resources = if color + .components + .iter() + .all(|plan| plan.subbands().is_empty()) + { + None + } else { + Some(session.htj2k_decode_table_resources()?) + }; + let table_upload_us = profile::elapsed_us(table_upload_start); + color.report.h2d_us = color.report.h2d_us.saturating_add(table_upload_us); + color.report.detail.table_upload_us = color + .report + .detail + .table_upload_us + .saturating_add(table_upload_us); + let payload_upload_start = profile::profile_now(collect_stage_timings); + let decode_resources = match table_resources.as_ref() { + Some(tables) => context.upload_htj2k_decode_resources_with_tables(&color.payload, tables), + None => context.upload_j2k_decode_payload(&color.payload), + } + .map_err(cuda_error)?; + let payload_upload_us = profile::elapsed_us(payload_upload_start); + profile::add_payload_resource_upload_us(&mut color.report, payload_upload_us); + let mut host_budget = HostPhaseBudget::new("j2k CUDA color decode execution graph"); + color.account_host_owners(&mut host_budget)?; + let mut component_work = host_budget.try_vec_with_capacity(3)?; + for plan in &color.components { + component_work.push(decode_cuda_component_subbands_with_resources( + &context, + plan, + &pool, + collect_stage_timings, + &mut host_budget, + )?); + } + run_component_cleanup_dequant_batches( + &context, + &decode_resources, + &mut component_work, + &pool, + collect_stage_timings, + host_budget.live_bytes(), + )?; + finish_color_cuda_resident_surface_with_component_work(FinishColorCudaResidentSurfaceRequest { + context: &context, + pool: &pool, + fmt, + color, + component_work, + wall_started, + collect_stage_timings, + run_idwt: true, + emit_report: true, + }) +} diff --git a/crates/j2k-cuda/src/decoder/color_batch/store/batch.rs b/crates/j2k-cuda/src/decoder/color_batch/store/batch.rs index c4c267a4..c223b957 100644 --- a/crates/j2k-cuda/src/decoder/color_batch/store/batch.rs +++ b/crates/j2k-cuda/src/decoder/color_batch/store/batch.rs @@ -65,7 +65,7 @@ pub(in crate::decoder::color_batch) fn prepare_rgb8_mct_batch_store( ]; let store_plan = ColorStorePlan::new( stores, - color.bit_depths, + color.rgb_bit_depths(), addends, ColorStoreRoute::for_mct(true, color.transform), ); diff --git a/crates/j2k-cuda/src/decoder/color_batch/store/validation.rs b/crates/j2k-cuda/src/decoder/color_batch/store/validation.rs index f1fb4c88..77117907 100644 --- a/crates/j2k-cuda/src/decoder/color_batch/store/validation.rs +++ b/crates/j2k-cuda/src/decoder/color_batch/store/validation.rs @@ -7,11 +7,13 @@ pub(super) fn bit_depth_addend(bit_depth: u8) -> f32 { f32::from(1_u16 << shift) } -pub(in crate::decoder::color_batch) fn validate_color_stores( - stores: [&CudaHtj2kStoreStep; 3], +pub(in crate::decoder::color_batch) fn validate_color_stores( + stores: [&CudaHtj2kStoreStep; N], dimensions: (u32, u32), ) -> Result<(), Error> { - let first = stores[0]; + let first = stores.first().ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; for store in stores { let input_width = store.input_rect.x1.saturating_sub(store.input_rect.x0); let input_height = store.input_rect.y1.saturating_sub(store.input_rect.y0); @@ -29,16 +31,32 @@ pub(in crate::decoder::color_batch) fn validate_color_stores( .ok_or(Error::UnsupportedCudaRequest { reason: CUDA_HTJ2K_KERNELS_NOT_READY, })?; - if store.output_x != 0 - || store.output_y != 0 - || store.copy_width != dimensions.0 - || store.copy_height != dimensions.1 - || store.output_width != dimensions.0 + let output_end_x = + store + .output_x + .checked_add(store.copy_width) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + let output_end_y = + store + .output_y + .checked_add(store.copy_height) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + if store.output_width != dimensions.0 || store.output_height != dimensions.1 + || output_end_x > dimensions.0 + || output_end_y > dimensions.1 || source_end_x > input_width || source_end_y > input_height || store.source_x != first.source_x || store.source_y != first.source_y + || store.copy_width != first.copy_width + || store.copy_height != first.copy_height + || store.output_x != first.output_x + || store.output_y != first.output_y { return Err(Error::UnsupportedCudaRequest { reason: CUDA_HTJ2K_KERNELS_NOT_READY, diff --git a/crates/j2k-cuda/src/decoder/grayscale_batch.rs b/crates/j2k-cuda/src/decoder/grayscale_batch.rs new file mode 100644 index 00000000..c152be71 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/grayscale_batch.rs @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k_core::DeviceSubmitSession; +use j2k_cuda_runtime::{ + CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut, CudaJ2kStoreGray16Job, + CudaJ2kStoreGray16Target, CudaJ2kStoreGray8Job, CudaJ2kStoreGray8Target, + CudaJ2kStoreGrayI16Target, CudaQueuedJ2kStoreBatch, +}; + +use super::color_batch::finalize_color_batch_decode_report; +use super::pending_completion::{PendingCleanup, PendingDecodeCompletion}; +use super::plan::{ + build_cuda_classic_grayscale_plans_from_referenced_with_profile, + build_cuda_htj2k_grayscale_plan_from_bytes_for_device_plan_with_profile, + build_cuda_htj2k_grayscale_plan_from_bytes_with_profile, + build_cuda_htj2k_grayscale_plans_from_referenced_with_profile, +}; +use super::resident::{ + can_batch_color_idwt, decode_cuda_component_subbands_with_resources, + enqueue_component_classic_batches, enqueue_component_cleanup_dequant_batches, + finish_cuda_component_decode, pooled_cuda_buffer, run_color_component_idwt_batches, + run_component_cleanup_dequant_batches, run_cuda_component_idwt_steps, +}; +use super::{ + cuda_error, cuda_range_storage, profile, BackendKind, CudaDecodedComponent, + CudaHtj2kDecodePlan, CudaHtj2kProfileReport, CudaQueuedIdwtBatch, CudaSession, + CudaSurfaceStats, DecodeSettings, DeviceDecodePlan, Error, NativeDecoderContext, PixelFormat, + Surface, SurfaceResidency, CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, +}; +use crate::allocation::HostPhaseBudget; + +mod completion; +mod execution; +mod preparation; +mod store; + +use self::completion::{ + grayscale_htj2k_job_identities, GrayscaleBatchOutput, GrayscaleHtj2kCleanup, + GrayscalePendingCompletion, StoredGrayscaleBatch, +}; +pub(crate) use self::completion::{ + GrayscaleOwnedBatch, SubmittedGrayscaleExternalBatch, SubmittedGrayscaleResidentBatch, +}; +use self::execution::decode_grayscale_cuda_batch_with_profile; +#[cfg(test)] +use self::preparation::prepare_grayscale_batch; +pub(crate) use self::preparation::GrayscaleBatchInput; + +pub(super) fn decode_grayscale_cuda_resident_batch_surfaces_with_profile( + inputs: &[&[u8]], + session: &mut CudaSession, + fmt: PixelFormat, + collect_stage_timings: bool, +) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + let inputs = inputs + .iter() + .map(|input| GrayscaleBatchInput::full(input)) + .collect::>(); + decode_grayscale_cuda_resident_prepared_batch_surfaces_with_profile( + &inputs, + DecodeSettings::default(), + session, + fmt, + collect_stage_timings, + ) +} + +pub(crate) fn decode_grayscale_cuda_resident_prepared_batch_surfaces_with_profile( + inputs: &[GrayscaleBatchInput<'_>], + settings: DecodeSettings, + session: &mut CudaSession, + fmt: PixelFormat, + collect_stage_timings: bool, +) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + let (output, report, pending) = decode_grayscale_cuda_batch_with_profile( + inputs, + settings, + session, + fmt, + collect_stage_timings, + None, + false, + )?; + if pending.is_some() { + return Err(Error::UnsupportedCudaRequest { + reason: "synchronous CUDA grayscale decode unexpectedly retained pending work", + }); + } + let GrayscaleBatchOutput::Owned(output) = output else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }); + }; + Ok((output.surfaces, report)) +} + +pub(super) fn decode_grayscale_cuda_resident_batch_into_with_profile( + inputs: &[&[u8]], + session: &mut CudaSession, + fmt: PixelFormat, + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + collect_stage_timings: bool, +) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + let inputs = inputs + .iter() + .map(|input| GrayscaleBatchInput::full(input)) + .collect::>(); + decode_grayscale_cuda_resident_prepared_batch_into_with_profile( + &inputs, + DecodeSettings::default(), + session, + fmt, + destination, + collect_stage_timings, + ) +} + +pub(crate) fn decode_grayscale_cuda_resident_prepared_batch_into_with_profile( + inputs: &[GrayscaleBatchInput<'_>], + settings: DecodeSettings, + session: &mut CudaSession, + fmt: PixelFormat, + destination: &mut CudaExternalDeviceBufferViewMut<'_>, + collect_stage_timings: bool, +) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + let (output, report, pending) = decode_grayscale_cuda_batch_with_profile( + inputs, + settings, + session, + fmt, + collect_stage_timings, + Some(destination), + false, + )?; + if pending.is_some() { + return Err(Error::UnsupportedCudaRequest { + reason: "synchronous external CUDA grayscale decode unexpectedly retained pending work", + }); + } + let GrayscaleBatchOutput::External(ranges) = output else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }); + }; + Ok((ranges, report)) +} + +pub(crate) fn submit_grayscale_cuda_resident_prepared_batch_into( + inputs: &[GrayscaleBatchInput<'_>], + settings: DecodeSettings, + session: &mut CudaSession, + fmt: PixelFormat, + destination: &mut CudaExternalDeviceBufferViewMut<'_>, +) -> Result { + let (output, report, completion) = decode_grayscale_cuda_batch_with_profile( + inputs, + settings, + session, + fmt, + false, + Some(destination), + true, + )?; + let GrayscaleBatchOutput::External(ranges) = output else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }); + }; + let completion = completion.ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA external batch submission did not retain a completion owner", + })?; + Ok(SubmittedGrayscaleExternalBatch { + ranges, + report, + completion: Some(completion), + }) +} + +pub(crate) fn submit_grayscale_cuda_resident_prepared_batch( + inputs: &[GrayscaleBatchInput<'_>], + settings: DecodeSettings, + session: &mut CudaSession, + fmt: PixelFormat, +) -> Result { + let (output, report, completion) = decode_grayscale_cuda_batch_with_profile( + inputs, settings, session, fmt, false, None, true, + )?; + let GrayscaleBatchOutput::Owned(output) = output else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }); + }; + let completion = completion.ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA resident grayscale submission did not retain a completion owner", + })?; + Ok(SubmittedGrayscaleResidentBatch { + output: Some(output), + report, + completion: Some(completion), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use j2k::{ + prepare_batch, BatchDecodeOptions, DeviceDecodePlan, DeviceDecodeRequest, EncodedImage, + PreparationDepth, + }; + use j2k_core::{Downscale, PixelFormat, Rect}; + use j2k_native::{encode_htj2k, DecodeSettings, EncodeOptions}; + + use super::completion::{map_grayscale_status_error, GrayscaleJobIdentity}; + use super::{prepare_grayscale_batch, GrayscaleBatchInput}; + + #[test] + fn grayscale_kernel_failure_maps_to_responsible_source_index() { + let identities = [ + GrayscaleJobIdentity { + source_index: 3, + original_job_index: 0, + }, + GrayscaleJobIdentity { + source_index: 9, + original_job_index: 1, + }, + ]; + let mapped = map_grayscale_status_error( + j2k_cuda_runtime::CudaError::KernelJobStatus { + kernel: "injected", + job_index: 1, + code: 7, + detail: 11, + }, + &identities, + ); + assert!(matches!( + mapped, + crate::Error::CudaTier1JobFailed { + source_index: 9, + original_job_index: 1, + .. + } + )); + } + + #[test] + fn grayscale_batch_rebases_two_plans_into_one_shared_payload() { + let pixels = (0_u16..64) + .map(|value| u8::try_from(value).expect("fixture byte")) + .collect::>(); + let encoded = encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + .expect("HTJ2K grayscale fixture"); + let prepared = prepare_grayscale_batch( + &[ + GrayscaleBatchInput::full(encoded.as_slice()), + GrayscaleBatchInput::full(encoded.as_slice()), + ], + PixelFormat::Gray8, + DecodeSettings::strict(), + ) + .expect("shared grayscale batch plan"); + + assert_eq!(prepared.plans.len(), 2); + assert!(!prepared.shared_payload.is_empty()); + assert!(prepared.plans.iter().all(|plan| plan.payload().is_empty())); + let first_max = prepared.plans[0] + .code_blocks() + .iter() + .map(|block| block.payload_offset + u64::from(block.payload_len)) + .max() + .expect("first block payload"); + let second_min = prepared.plans[1] + .code_blocks() + .iter() + .map(|block| block.payload_offset) + .min() + .expect("second block payload"); + assert!(second_min >= first_max); + } + + #[test] + fn prepared_htj2k_batch_uses_retained_offsets_without_reparsing() { + let pixels = (0_u8..64).collect::>(); + let encoded = encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + .expect("HTJ2K retained-plan fixture"); + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(encoded))], + BatchDecodeOptions::default(), + ) + .expect("shared retained-plan preparation"); + let [group] = prepared.groups() else { + panic!("expected one prepared group") + }; + let [image] = group.images() else { + panic!("expected one prepared image") + }; + assert_eq!(image.preparation_depth(), PreparationDepth::Htj2kOffsetPlan); + let referenced_plan = image + .htj2k_plan() + .expect("retained HTJ2K plan") + .adapter_view() + .downcast_ref::() + .expect("native referenced HTJ2K plan adapter"); + let settings = DecodeSettings { + resolve_palette_indices: true, + strict: group.options().settings.is_strict(), + target_resolution: None, + }; + + let cuda = prepare_grayscale_batch( + &[GrayscaleBatchInput { + source_index: 0, + bytes: image.bytes(), + device_plan: Some(image.plan()), + referenced_plan: Some(referenced_plan), + referenced_classic_plan: None, + }], + PixelFormat::Gray8, + settings, + ) + .expect("CUDA retained-plan preparation"); + + assert_eq!(cuda.reports[0].parse_us, 0); + assert_eq!(cuda.reports[0].plan_us, 0); + assert!(cuda.plans[0].payload().is_empty()); + assert!(!cuda.shared_payload.is_empty()); + } + + #[test] + fn grayscale_batch_prepares_roi_and_reduced_requests_in_one_payload_arena() { + let pixels = (0_u16..16 * 16) + .map(|value| u8::try_from(value).expect("fixture byte")) + .collect::>(); + let encoded = encode_htj2k( + &pixels, + 16, + 16, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }, + ) + .expect("HTJ2K grayscale geometry fixture"); + let roi_plan = DeviceDecodePlan::for_image( + (16, 16), + DeviceDecodeRequest::Region { + roi: Rect { + x: 3, + y: 5, + w: 7, + h: 6, + }, + }, + ) + .expect("ROI plan"); + let reduced_plan = DeviceDecodePlan::for_image( + (16, 16), + DeviceDecodeRequest::Scaled { + scale: Downscale::Half, + }, + ) + .expect("reduced plan"); + + let prepared = prepare_grayscale_batch( + &[ + GrayscaleBatchInput { + source_index: 0, + bytes: &encoded, + device_plan: Some(roi_plan), + referenced_plan: None, + referenced_classic_plan: None, + }, + GrayscaleBatchInput { + source_index: 1, + bytes: &encoded, + device_plan: Some(reduced_plan), + referenced_plan: None, + referenced_classic_plan: None, + }, + ], + PixelFormat::Gray8, + DecodeSettings::strict(), + ) + .expect("prepare geometry batch"); + + assert_eq!(prepared.plans[0].dimensions(), roi_plan.output_dims()); + assert_eq!(prepared.plans[1].dimensions(), reduced_plan.output_dims()); + assert!(prepared.plans.iter().all(|plan| plan.payload().is_empty())); + assert!(!prepared.shared_payload.is_empty()); + } +} diff --git a/crates/j2k-cuda/src/decoder/grayscale_batch/completion.rs b/crates/j2k-cuda/src/decoder/grayscale_batch/completion.rs new file mode 100644 index 00000000..11713d9b --- /dev/null +++ b/crates/j2k-cuda/src/decoder/grayscale_batch/completion.rs @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Grayscale status identity mapping and pending output ownership. + +use super::{ + cuda_error, Arc, CudaDeviceBufferRange, CudaHtj2kProfileReport, CudaQueuedJ2kStoreBatch, Error, + HostPhaseBudget, PendingCleanup, PendingDecodeCompletion, Surface, +}; + +pub(crate) struct GrayscaleOwnedBatch { + pub(crate) surfaces: Vec, + pub(crate) buffer: Arc, + pub(crate) ranges: Vec, +} + +pub(super) enum GrayscaleBatchOutput { + Owned(GrayscaleOwnedBatch), + External(Vec), +} + +pub(super) struct StoredGrayscaleBatch { + pub(super) output: GrayscaleBatchOutput, + pub(super) queued: Option, +} + +pub(super) type GrayscalePendingCompletion = PendingDecodeCompletion; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct GrayscaleJobIdentity { + pub(super) source_index: usize, + pub(super) original_job_index: usize, +} + +pub(super) struct GrayscaleHtj2kCleanup { + queued: Option, + identities: Vec, +} + +impl GrayscaleHtj2kCleanup { + pub(super) fn new( + queued: j2k_cuda_runtime::CudaQueuedHtj2kCleanup, + identities: Vec, + ) -> Self { + Self { + queued: Some(queued), + identities, + } + } + + fn has_status_readback(&self) -> bool { + self.queued + .as_ref() + .is_some_and(|queued| queued.status_count() != 0) + } + + fn finish(mut self) -> Result<(), Error> { + let Some(queued) = self.queued.take() else { + return Ok(()); + }; + queued + .finish() + .map(|_| ()) + .map_err(|error| map_grayscale_status_error(error, &self.identities)) + } +} + +impl PendingCleanup for GrayscaleHtj2kCleanup { + fn has_status_readback(&self) -> bool { + Self::has_status_readback(self) + } + + fn finish(self) -> Result<(), Error> { + Self::finish(self) + } +} + +impl Drop for GrayscaleHtj2kCleanup { + fn drop(&mut self) { + if let Some(queued) = self.queued.take() { + let _ = queued.finish(); + } + } +} + +/// Pending external grayscale batch whose internal codec resources remain live +/// until the asynchronous final store completes. +pub(crate) struct SubmittedGrayscaleExternalBatch { + pub(super) ranges: Vec, + pub(super) report: CudaHtj2kProfileReport, + pub(super) completion: Option, +} + +pub(crate) struct SubmittedGrayscaleResidentBatch { + pub(super) output: Option, + pub(super) report: CudaHtj2kProfileReport, + pub(super) completion: Option, +} + +impl SubmittedGrayscaleResidentBatch { + pub(crate) fn is_complete(&self) -> Result { + self.completion + .as_ref() + .map_or(Ok(true), GrayscalePendingCompletion::is_complete) + } + + pub(crate) fn finish(mut self) -> Result<(GrayscaleOwnedBatch, CudaHtj2kProfileReport), Error> { + if let Some(mut completion) = self.completion.take() { + if let Err(error) = completion.complete() { + if error.completion_is_uncertain() { + if let Some(output) = self.output.take() { + core::mem::forget(output); + } + } + return Err(error); + } + } + let output = self.output.take().ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA resident grayscale submission lost its output owner", + })?; + Ok((output, self.report.clone())) + } +} + +impl Drop for SubmittedGrayscaleResidentBatch { + fn drop(&mut self) { + let result = self + .completion + .take() + .map_or(Ok(()), |mut completion| completion.complete()); + if result.is_err_and(|error| error.completion_is_uncertain()) { + if let Some(output) = self.output.take() { + core::mem::forget(output); + } + } + } +} + +impl SubmittedGrayscaleExternalBatch { + pub(crate) fn ranges(&self) -> &[CudaDeviceBufferRange] { + &self.ranges + } + + pub(crate) fn is_complete(&self) -> Result { + self.completion + .as_ref() + .map_or(Ok(true), GrayscalePendingCompletion::is_complete) + } + + pub(crate) fn finish( + mut self, + ) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + if let Some(mut completion) = self.completion.take() { + completion.complete()?; + } + Ok((self.ranges, self.report)) + } +} + +pub(super) fn grayscale_htj2k_job_identities( + component_work: &[super::super::CudaComponentDecodeWork], + source_indices: &[usize], + live_host_bytes: usize, +) -> Result, Error> { + if component_work.len() != source_indices.len() { + return Err(Error::UnsupportedCudaRequest { + reason: "CUDA grayscale source identity count does not match component work", + }); + } + let job_count = component_work + .iter() + .flat_map(|work| &work.pending_dequant_bands) + .try_fold(0usize, |count, pending| { + count.checked_add(pending.jobs.len()) + }) + .ok_or(Error::HostAllocationFailed { + bytes: usize::MAX, + what: "CUDA grayscale HTJ2K status identities", + })?; + let mut budget = HostPhaseBudget::with_live_bytes( + "CUDA grayscale HTJ2K status identities", + live_host_bytes, + )?; + let mut identities = budget.try_vec_with_capacity(job_count)?; + for (work, source_index) in component_work.iter().zip(source_indices.iter().copied()) { + for pending in &work.pending_dequant_bands { + for _ in &pending.jobs { + identities.push(GrayscaleJobIdentity { + source_index, + original_job_index: identities.len(), + }); + } + } + } + Ok(identities) +} + +pub(super) fn map_grayscale_status_error( + error: j2k_cuda_runtime::CudaError, + identities: &[GrayscaleJobIdentity], +) -> Error { + let Some(job_index) = error.kernel_job_index() else { + return cuda_error(error); + }; + let Some(identity) = identities.get(job_index) else { + return cuda_error(error); + }; + Error::CudaTier1JobFailed { + source_index: identity.source_index, + original_job_index: identity.original_job_index, + source: error, + } +} diff --git a/crates/j2k-cuda/src/decoder/grayscale_batch/execution.rs b/crates/j2k-cuda/src/decoder/grayscale_batch/execution.rs new file mode 100644 index 00000000..a767cb4f --- /dev/null +++ b/crates/j2k-cuda/src/decoder/grayscale_batch/execution.rs @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Grayscale CUDA execution orchestration. + +use super::preparation::{grayscale_owner_budget, prepare_grayscale_batch, PreparedGrayscaleBatch}; +use super::store::{store_gray16_batch, store_gray8_batch, store_grayi16_batch}; +use super::{ + can_batch_color_idwt, cuda_error, decode_cuda_component_subbands_with_resources, + enqueue_component_classic_batches, enqueue_component_cleanup_dequant_batches, + finalize_color_batch_decode_report, finish_cuda_component_decode, + grayscale_htj2k_job_identities, profile, run_color_component_idwt_batches, + run_component_cleanup_dequant_batches, run_cuda_component_idwt_steps, CudaDecodedComponent, + CudaExternalDeviceBufferViewMut, CudaHtj2kProfileReport, CudaQueuedIdwtBatch, CudaSession, + DecodeSettings, DeviceSubmitSession, Error, GrayscaleBatchInput, GrayscaleBatchOutput, + GrayscaleHtj2kCleanup, GrayscalePendingCompletion, PixelFormat, + CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, +}; +use crate::decoder::pending_completion::{finish_decode_statuses, retire_decode_after_error}; + +#[expect( + clippy::too_many_lines, + reason = "grayscale batch orchestration keeps shared resources and completion ordering atomic" +)] +pub(super) fn decode_grayscale_cuda_batch_with_profile( + inputs: &[GrayscaleBatchInput<'_>], + settings: DecodeSettings, + session: &mut CudaSession, + fmt: PixelFormat, + collect_stage_timings: bool, + external: Option<&mut CudaExternalDeviceBufferViewMut<'_>>, + enqueue_external: bool, +) -> Result< + ( + GrayscaleBatchOutput, + CudaHtj2kProfileReport, + Option, + ), + Error, +> { + if !matches!( + fmt, + PixelFormat::Gray8 | PixelFormat::Gray16 | PixelFormat::GrayI16 + ) { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }); + } + let batch_wall_started = profile::profile_now(collect_stage_timings); + let PreparedGrayscaleBatch { + plans, + mut reports, + shared_payload, + output_indices, + output_dimensions, + source_indices, + } = prepare_grayscale_batch(inputs, fmt, settings)?; + let context = session.cuda_context()?; + let pool = session.decode_batch_buffer_pool()?; + let table_upload_start = profile::profile_now(collect_stage_timings); + let table_resources = if plans.iter().all(|plan| plan.subbands().is_empty()) { + None + } else { + Some(session.htj2k_decode_table_resources()?) + }; + let table_upload_us = profile::elapsed_us(table_upload_start); + let payload_upload_start = profile::profile_now(collect_stage_timings); + let decode_resources = match table_resources.as_ref() { + Some(tables) => context.upload_htj2k_decode_resources_with_tables_and_pool( + &shared_payload, + tables, + &pool, + ), + None => context.upload_j2k_decode_payload_with_pool(&shared_payload, &pool), + } + .map_err(cuda_error)?; + let payload_upload_us = profile::elapsed_us(payload_upload_start); + drop(shared_payload); + + let empty_payload = Vec::new(); + let mut host_budget = grayscale_owner_budget( + &plans, + &reports, + &empty_payload, + None, + "j2k CUDA grayscale batch execution graph", + )?; + host_budget.account_vec(&output_indices)?; + host_budget.account_vec(&output_dimensions)?; + host_budget.account_vec(&source_indices)?; + let mut component_work = host_budget.try_vec_with_capacity(plans.len())?; + let mut component_source_indices = host_budget.try_vec_with_capacity(plans.len())?; + for (plan, source_index) in plans.iter().zip(source_indices.iter().copied()) { + component_work.push(decode_cuda_component_subbands_with_resources( + &context, + plan, + &pool, + collect_stage_timings, + &mut host_budget, + )?); + component_source_indices.push(source_index); + } + let (pending_cleanup, pending_classic) = if collect_stage_timings { + run_component_cleanup_dequant_batches( + &context, + &decode_resources, + &mut component_work, + &pool, + true, + host_budget.live_bytes(), + )?; + (None, None) + } else { + let pending_classic = if component_work + .iter() + .any(|work| !work.pending_classic_bands.is_empty()) + { + let classic_tables = session.classic_decode_table_resources()?; + enqueue_component_classic_batches( + &context, + &decode_resources, + &classic_tables, + &mut component_work, + &component_source_indices, + &pool, + host_budget.live_bytes(), + )? + } else { + None + }; + let identities = grayscale_htj2k_job_identities( + &component_work, + &component_source_indices, + host_budget.live_bytes(), + )?; + let pending_cleanup = enqueue_component_cleanup_dequant_batches( + &context, + &decode_resources, + &mut component_work, + &pool, + host_budget.live_bytes(), + )? + .map(|queued| GrayscaleHtj2kCleanup::new(queued, identities)); + (pending_cleanup, pending_classic) + }; + drop(component_source_indices); + + let plan_refs = plans.iter().collect::>(); + let idwt_batched = can_batch_color_idwt(&plan_refs); + let pending_idwt = if idwt_batched { + run_color_component_idwt_batches( + &context, + &plan_refs, + &mut component_work, + &pool, + collect_stage_timings, + host_budget.live_bytes(), + )? + } else if collect_stage_timings { + for (plan, work) in plans.iter().zip(component_work.iter_mut()) { + run_cuda_component_idwt_steps(&context, plan.idwt_steps(), work, &pool, true)?; + } + None + } else { + let mut pending: Option = None; + for (plan, work) in plans.iter().zip(component_work.iter_mut()) { + let components = [plan]; + let next = run_color_component_idwt_batches( + &context, + &components, + std::slice::from_mut(work), + &pool, + false, + host_budget.live_bytes(), + )?; + pending = match (pending, next) { + (Some(current), Some(next)) => Some(current.merge(next)?), + (Some(current), None) => Some(current), + (None, next) => next, + }; + } + pending + }; + drop(plan_refs); + + let completion_result = (|| { + let mut decoded = Vec::new(); + decoded + .try_reserve_exact(component_work.len()) + .map_err(|_| Error::HostAllocationFailed { + bytes: component_work + .len() + .saturating_mul(std::mem::size_of::()), + what: "j2k CUDA grayscale decoded components", + })?; + for work in component_work { + decoded.push(finish_cuda_component_decode(work)?); + } + + let store_started = profile::profile_now(collect_stage_timings); + let stored = match fmt { + PixelFormat::Gray8 => store_gray8_batch( + &context, + &plans, + &output_indices, + &output_dimensions, + &decoded, + external, + enqueue_external, + )?, + PixelFormat::Gray16 => store_gray16_batch( + &context, + &plans, + &output_indices, + &output_dimensions, + &decoded, + external, + enqueue_external, + )?, + PixelFormat::GrayI16 => store_grayi16_batch( + &context, + &plans, + &output_indices, + &output_dimensions, + &decoded, + external, + enqueue_external, + )?, + _ => { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }) + } + }; + let store_us = profile::elapsed_us(store_started); + let store_dispatches = usize::from(!decoded.is_empty()); + for (index, (component, report)) in decoded.iter().zip(reports.iter_mut()).enumerate() { + let report_store_dispatches = usize::from(index == 0) * store_dispatches; + report.dispatch_count = component.dispatches.saturating_add(report_store_dispatches); + component.timings.add_to_report(report); + if index == 0 { + report.store_us = report.store_us.saturating_add(store_us); + report.detail.store_dispatch_count = report + .detail + .store_dispatch_count + .saturating_add(store_dispatches); + } + profile::finalize_decode_total_us(report); + } + Ok((stored, decoded)) + })(); + + if enqueue_external { + let (stored, decoded) = match completion_result { + Ok(output) => output, + Err(error) => { + return Err(retire_decode_after_error( + error, + pending_idwt, + pending_cleanup, + pending_classic, + )); + } + }; + let store = stored.queued.ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA external batch store did not return a completion guard", + })?; + let output = stored.output; + let pending = GrayscalePendingCompletion::new( + Some(store), + pending_idwt, + pending_cleanup, + pending_classic, + decoded, + Some(decode_resources), + ); + let aggregate = finalize_color_batch_decode_report( + &reports, + table_upload_us, + payload_upload_us, + batch_wall_started, + ); + session.record_submit(); + aggregate.emit("submit_batch_gray"); + return Ok((output, aggregate, Some(pending))); + } + + let completion_result = completion_result.and_then(|(stored, decoded)| { + if stored.queued.is_some() { + return Err(Error::UnsupportedCudaRequest { + reason: "synchronous CUDA grayscale store unexpectedly returned pending work", + }); + } + Ok(((stored.output, decoded), true)) + }); + let resolved = + CudaQueuedIdwtBatch::resolve_optional_after_completed_work(pending_idwt, completion_result); + let (output, _decoded_owners) = match resolved { + Ok(output) => { + finish_decode_statuses(pending_cleanup, pending_classic)?; + output + } + Err(error) => { + return Err(retire_decode_after_error( + error, + None, + pending_cleanup, + pending_classic, + )); + } + }; + + let aggregate = finalize_color_batch_decode_report( + &reports, + table_upload_us, + payload_upload_us, + batch_wall_started, + ); + session.record_submit(); + aggregate.emit("decode_batch_gray"); + Ok((output, aggregate, None)) +} diff --git a/crates/j2k-cuda/src/decoder/grayscale_batch/preparation.rs b/crates/j2k-cuda/src/decoder/grayscale_batch/preparation.rs new file mode 100644 index 00000000..0bd8fe25 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/grayscale_batch/preparation.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Grayscale batch planning and retained input contracts. + +use super::{ + build_cuda_classic_grayscale_plans_from_referenced_with_profile, + build_cuda_htj2k_grayscale_plan_from_bytes_for_device_plan_with_profile, + build_cuda_htj2k_grayscale_plan_from_bytes_with_profile, + build_cuda_htj2k_grayscale_plans_from_referenced_with_profile, CudaHtj2kDecodePlan, + CudaHtj2kProfileReport, DecodeSettings, DeviceDecodePlan, Error, HostPhaseBudget, + NativeDecoderContext, PixelFormat, +}; + +pub(super) struct PreparedGrayscaleBatch { + pub(super) plans: Vec, + pub(super) reports: Vec, + pub(super) shared_payload: Vec, + pub(super) output_indices: Vec, + pub(super) output_dimensions: Vec<(u32, u32)>, + pub(super) source_indices: Vec, +} + +/// Borrowed encoded bytes plus normalized geometry for one shared CUDA batch +/// plan. `None` retains the legacy full-frame path that discovers dimensions +/// while parsing. +#[derive(Clone, Copy)] +pub(crate) struct GrayscaleBatchInput<'a> { + pub(crate) source_index: usize, + pub(crate) bytes: &'a [u8], + pub(crate) device_plan: Option, + pub(crate) referenced_plan: Option<&'a j2k_native::J2kReferencedHtj2kPlan>, + pub(crate) referenced_classic_plan: Option<&'a j2k_native::J2kReferencedClassicPlan>, +} + +impl<'a> GrayscaleBatchInput<'a> { + pub(crate) const fn full(bytes: &'a [u8]) -> Self { + Self { + source_index: 0, + bytes, + device_plan: None, + referenced_plan: None, + referenced_classic_plan: None, + } + } +} + +#[expect( + clippy::too_many_lines, + reason = "one preparation boundary keeps tile plans, shared payload ownership, and dense output identities aligned" +)] +pub(super) fn prepare_grayscale_batch( + inputs: &[GrayscaleBatchInput<'_>], + fmt: PixelFormat, + settings: DecodeSettings, +) -> Result { + let mut initial_budget = HostPhaseBudget::new("j2k CUDA grayscale batch plan owners"); + let mut plans = initial_budget.try_vec_with_capacity(inputs.len())?; + let mut reports = initial_budget.try_vec_with_capacity(inputs.len())?; + let mut output_indices = initial_budget.try_vec_with_capacity(inputs.len())?; + let mut output_dimensions = initial_budget.try_vec_with_capacity(inputs.len())?; + let mut source_indices = initial_budget.try_vec_with_capacity(inputs.len())?; + let mut shared_payload = Vec::new(); + let mut native_context = NativeDecoderContext::default(); + + for (output_index, input) in inputs.iter().enumerate() { + let (mut input_plans, payload_is_shared) = + match (input.referenced_plan, input.referenced_classic_plan) { + (Some(referenced), None) => { + let device_plan = input.device_plan.ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA HTJ2K plan is missing normalized output geometry", + })?; + let mut append_budget = grayscale_owner_budget( + &plans, + &reports, + &shared_payload, + None, + "j2k CUDA referenced grayscale batch plan owners", + )?; + let plans = build_cuda_htj2k_grayscale_plans_from_referenced_with_profile( + input.bytes, + referenced, + fmt, + device_plan, + &mut shared_payload, + &mut append_budget, + )?; + (plans, true) + } + (None, Some(referenced)) => { + let device_plan = input.device_plan.ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic plan is missing normalized output geometry", + })?; + let mut append_budget = grayscale_owner_budget( + &plans, + &reports, + &shared_payload, + None, + "j2k CUDA referenced classic grayscale batch plan owners", + )?; + let plans = build_cuda_classic_grayscale_plans_from_referenced_with_profile( + input.bytes, + referenced, + fmt, + device_plan, + &mut shared_payload, + &mut append_budget, + )?; + (plans, true) + } + (None, None) if input.device_plan.is_some() => { + let device_plan = input.device_plan.expect("checked prepared device plan"); + let (plan, report) = + build_cuda_htj2k_grayscale_plan_from_bytes_for_device_plan_with_profile( + input.bytes, + fmt, + Some(device_plan), + settings, + &mut native_context, + )?; + let mut one = initial_budget.try_vec_with_capacity(1)?; + one.push((plan, report)); + (one, false) + } + (None, None) => { + let (plan, report) = build_cuda_htj2k_grayscale_plan_from_bytes_with_profile( + input.bytes, + fmt, + &mut native_context, + )?; + let mut one = initial_budget.try_vec_with_capacity(1)?; + one.push((plan, report)); + (one, false) + } + (Some(_), Some(_)) => { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA grayscale input contains conflicting codec plans", + }); + } + }; + if input_plans.is_empty() { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA grayscale input produced no executable tile plans", + }); + } + if !payload_is_shared { + for (plan, _) in &mut input_plans { + let mut append_budget = grayscale_owner_budget( + &plans, + &reports, + &shared_payload, + Some(plan), + "j2k CUDA grayscale batch plan owners", + )?; + plan.append_payload_to_shared_with_budget(&mut shared_payload, &mut append_budget)?; + } + } + let dimensions = input + .device_plan + .map_or(input_plans[0].0.dimensions(), DeviceDecodePlan::output_dims); + let mut append_budget = grayscale_owner_budget( + &plans, + &reports, + &shared_payload, + None, + "j2k CUDA grayscale tile owner append", + )?; + append_budget.account_vec(&output_indices)?; + append_budget.account_vec(&output_dimensions)?; + append_budget.account_vec(&source_indices)?; + append_budget.try_vec_reserve(&mut plans, input_plans.len())?; + append_budget.try_vec_reserve(&mut reports, input_plans.len())?; + append_budget.try_vec_reserve(&mut output_indices, input_plans.len())?; + append_budget.try_vec_reserve(&mut source_indices, input_plans.len())?; + for (plan, report) in input_plans { + plans.push(plan); + reports.push(report); + output_indices.push(output_index); + source_indices.push(input.source_index); + } + output_dimensions.push(dimensions); + } + + Ok(PreparedGrayscaleBatch { + plans, + reports, + shared_payload, + output_indices, + output_dimensions, + source_indices, + }) +} + +pub(super) fn grayscale_owner_budget( + plans: &Vec, + reports: &Vec, + payload: &Vec, + pending: Option<&CudaHtj2kDecodePlan>, + what: &'static str, +) -> Result { + let mut budget = HostPhaseBudget::new(what); + budget.account_vec(plans)?; + budget.account_vec(reports)?; + budget.account_vec(payload)?; + for plan in plans { + plan.account_host_owners(&mut budget)?; + } + if let Some(plan) = pending { + plan.account_host_owners(&mut budget)?; + } + Ok(budget) +} diff --git a/crates/j2k-cuda/src/decoder/grayscale_batch/store.rs b/crates/j2k-cuda/src/decoder/grayscale_batch/store.rs new file mode 100644 index 00000000..55e9b2dd --- /dev/null +++ b/crates/j2k-cuda/src/decoder/grayscale_batch/store.rs @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Typed grayscale final-store materialization. + +use super::{ + cuda_error, cuda_range_storage, pooled_cuda_buffer, Arc, BackendKind, CudaDecodedComponent, + CudaDeviceBufferRange, CudaExternalDeviceBufferViewMut, CudaHtj2kDecodePlan, + CudaJ2kStoreGray16Job, CudaJ2kStoreGray16Target, CudaJ2kStoreGray8Job, CudaJ2kStoreGray8Target, + CudaJ2kStoreGrayI16Target, CudaSurfaceStats, Error, GrayscaleBatchOutput, GrayscaleOwnedBatch, + PixelFormat, StoredGrayscaleBatch, Surface, SurfaceResidency, +}; + +fn gray_store_job(component: &CudaDecodedComponent) -> (u32, super::super::CudaHtj2kStoreStep) { + let store = component.store; + let input_width = store.input_rect.x1.saturating_sub(store.input_rect.x0); + (input_width, store) +} + +pub(super) fn store_gray8_batch( + context: &j2k_cuda_runtime::CudaContext, + plans: &[CudaHtj2kDecodePlan], + output_indices: &[usize], + output_dimensions: &[(u32, u32)], + decoded: &[CudaDecodedComponent], + external: Option<&mut CudaExternalDeviceBufferViewMut<'_>>, + enqueue_external: bool, +) -> Result { + let targets = plans + .iter() + .zip(decoded) + .zip(output_indices.iter().copied()) + .map(|((plan, component), output_index)| { + let (input_width, store) = gray_store_job(component); + Ok(CudaJ2kStoreGray8Target { + output_index, + input: pooled_cuda_buffer(&component.buffer)?, + job: CudaJ2kStoreGray8Job { + input_width, + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_height: store.output_height, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + bit_depth: u32::from(plan.bit_depth()), + }, + }) + }) + .collect::, Error>>()?; + materialize_gray8_output( + context, + output_dimensions, + &targets, + external, + enqueue_external, + ) +} + +fn materialize_gray8_output( + context: &j2k_cuda_runtime::CudaContext, + output_dimensions: &[(u32, u32)], + targets: &[CudaJ2kStoreGray8Target<'_>], + external: Option<&mut CudaExternalDeviceBufferViewMut<'_>>, + enqueue_external: bool, +) -> Result { + if let Some(destination) = external { + if enqueue_external { + // SAFETY: the pending high-level owner retains every decoded + // coefficient buffer and the caller guarantees the external + // allocation lifetime until that owner is retired. + let (ranges, queued) = unsafe { + context.j2k_store_gray8_batch_into_external_device_enqueue(targets, destination) + } + .map_err(cuda_error)?; + return Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::External(ranges), + queued: Some(queued), + }); + } + // SAFETY: this synchronous path retains all decoded coefficient + // owners and the caller-owned destination through the completion + // boundary. Its public caller carries the quarantine-on-error contract. + let (ranges, _) = + unsafe { context.j2k_store_gray8_batch_into_external_device(targets, destination) } + .map_err(cuda_error)?; + return Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::External(ranges), + queued: None, + }); + } + if enqueue_external { + let (buffer, ranges, queued) = context + .j2k_store_gray8_batch_contiguous_device_enqueue(targets) + .map_err(cuda_error)?; + let stats = queued.execution(); + return Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::Owned(owned_batch_from_contiguous( + output_dimensions, + PixelFormat::Gray8, + buffer, + ranges, + stats, + )), + queued: Some(queued), + }); + } + let output = context + .j2k_store_gray8_batch_contiguous_device(targets) + .map_err(cuda_error)?; + let (buffer, ranges, stats) = output.into_parts(); + Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::Owned(owned_batch_from_contiguous( + output_dimensions, + PixelFormat::Gray8, + buffer, + ranges, + stats, + )), + queued: None, + }) +} + +pub(super) fn store_gray16_batch( + context: &j2k_cuda_runtime::CudaContext, + plans: &[CudaHtj2kDecodePlan], + output_indices: &[usize], + output_dimensions: &[(u32, u32)], + decoded: &[CudaDecodedComponent], + external: Option<&mut CudaExternalDeviceBufferViewMut<'_>>, + enqueue_external: bool, +) -> Result { + let targets = plans + .iter() + .zip(decoded) + .zip(output_indices.iter().copied()) + .map(|((plan, component), output_index)| { + let (input_width, store) = gray_store_job(component); + Ok(CudaJ2kStoreGray16Target { + output_index, + input: pooled_cuda_buffer(&component.buffer)?, + job: CudaJ2kStoreGray16Job { + input_width, + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_height: store.output_height, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + bit_depth: u32::from(plan.bit_depth()), + }, + }) + }) + .collect::, Error>>()?; + if let Some(destination) = external { + if enqueue_external { + // SAFETY: the pending high-level owner retains every decoded + // coefficient buffer and the caller guarantees the external + // allocation lifetime until that owner is retired. + let (ranges, queued) = unsafe { + context.j2k_store_gray16_batch_into_external_device_enqueue(&targets, destination) + } + .map_err(cuda_error)?; + return Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::External(ranges), + queued: Some(queued), + }); + } + // SAFETY: this synchronous path retains all decoded coefficient + // owners and the caller-owned destination through the completion + // boundary. Its public caller carries the quarantine-on-error contract. + let (ranges, _) = + unsafe { context.j2k_store_gray16_batch_into_external_device(&targets, destination) } + .map_err(cuda_error)?; + return Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::External(ranges), + queued: None, + }); + } + if enqueue_external { + let (buffer, ranges, queued) = context + .j2k_store_gray16_batch_contiguous_device_enqueue(&targets) + .map_err(cuda_error)?; + let stats = queued.execution(); + return Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::Owned(owned_batch_from_contiguous( + output_dimensions, + PixelFormat::Gray16, + buffer, + ranges, + stats, + )), + queued: Some(queued), + }); + } + let output = context + .j2k_store_gray16_batch_contiguous_device(&targets) + .map_err(cuda_error)?; + let (buffer, ranges, stats) = output.into_parts(); + Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::Owned(owned_batch_from_contiguous( + output_dimensions, + PixelFormat::Gray16, + buffer, + ranges, + stats, + )), + queued: None, + }) +} + +pub(super) fn store_grayi16_batch( + context: &j2k_cuda_runtime::CudaContext, + plans: &[CudaHtj2kDecodePlan], + output_indices: &[usize], + output_dimensions: &[(u32, u32)], + decoded: &[CudaDecodedComponent], + external: Option<&mut CudaExternalDeviceBufferViewMut<'_>>, + enqueue_external: bool, +) -> Result { + let targets = plans + .iter() + .zip(decoded) + .zip(output_indices.iter().copied()) + .map(|((plan, component), output_index)| { + let (input_width, store) = gray_store_job(component); + Ok(CudaJ2kStoreGrayI16Target { + output_index, + input: pooled_cuda_buffer(&component.buffer)?, + job: CudaJ2kStoreGray16Job { + input_width, + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_height: store.output_height, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + bit_depth: u32::from(plan.bit_depth()), + }, + }) + }) + .collect::, Error>>()?; + if let Some(destination) = external { + if enqueue_external { + // SAFETY: the pending high-level owner retains every decoded + // coefficient buffer and the caller guarantees the external + // allocation lifetime until that owner is retired. + let (ranges, queued) = unsafe { + context.j2k_store_grayi16_batch_into_external_device_enqueue(&targets, destination) + } + .map_err(cuda_error)?; + return Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::External(ranges), + queued: Some(queued), + }); + } + // SAFETY: this synchronous path retains all decoded coefficient + // owners and the caller-owned destination through the completion + // boundary. Its public caller carries the quarantine-on-error contract. + let (ranges, _) = + unsafe { context.j2k_store_grayi16_batch_into_external_device(&targets, destination) } + .map_err(cuda_error)?; + return Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::External(ranges), + queued: None, + }); + } + if enqueue_external { + let (buffer, ranges, queued) = context + .j2k_store_grayi16_batch_contiguous_device_enqueue(&targets) + .map_err(cuda_error)?; + let stats = queued.execution(); + return Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::Owned(owned_batch_from_contiguous( + output_dimensions, + PixelFormat::GrayI16, + buffer, + ranges, + stats, + )), + queued: Some(queued), + }); + } + let output = context + .j2k_store_grayi16_batch_contiguous_device(&targets) + .map_err(cuda_error)?; + let (buffer, ranges, stats) = output.into_parts(); + Ok(StoredGrayscaleBatch { + output: GrayscaleBatchOutput::Owned(owned_batch_from_contiguous( + output_dimensions, + PixelFormat::GrayI16, + buffer, + ranges, + stats, + )), + queued: None, + }) +} + +fn owned_batch_from_contiguous( + output_dimensions: &[(u32, u32)], + fmt: PixelFormat, + buffer: j2k_cuda_runtime::CudaDeviceBuffer, + ranges: Vec, + stats: j2k_cuda_runtime::CudaExecutionStats, +) -> GrayscaleOwnedBatch { + let shared = Arc::new(buffer); + let surfaces = output_dimensions + .iter() + .zip(ranges.iter().copied()) + .map(|(&dimensions, range)| Surface { + backend: BackendKind::Cuda, + residency: SurfaceResidency::CudaResidentDecode, + dimensions, + fmt, + pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), + stats: CudaSurfaceStats { + total: stats.kernel_dispatches(), + copy: stats.copy_kernel_dispatches(), + decode: stats.decode_kernel_dispatches(), + }, + storage: cuda_range_storage(shared.clone(), range.offset, range.len), + }) + .collect(); + GrayscaleOwnedBatch { + surfaces, + buffer: shared, + ranges, + } +} diff --git a/crates/j2k-cuda/src/decoder/pending_completion.rs b/crates/j2k-cuda/src/decoder/pending_completion.rs new file mode 100644 index 00000000..76131ec9 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/pending_completion.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Shared lifetime state for asynchronous resident and external batch completion. + +use j2k_cuda_runtime::{CudaHtj2kDecodeResources, CudaQueuedJ2kStoreBatch}; + +use super::resident::{ChunkedHtj2kCleanup, QueuedComponentClassicDecode}; +use super::{ + combine_cuda_cleanup_errors, cuda_error, CudaDecodedComponent, CudaQueuedIdwtBatch, Error, +}; + +pub(in crate::decoder) trait PendingCleanup: Sized { + fn has_status_readback(&self) -> bool; + fn finish(self) -> Result<(), Error>; +} + +impl PendingCleanup for ChunkedHtj2kCleanup { + fn has_status_readback(&self) -> bool { + Self::has_status_readback(self) + } + + fn finish(self) -> Result<(), Error> { + Self::finish(self) + } +} + +pub(in crate::decoder) fn retire_decode_after_error( + error: Error, + idwt: Option, + cleanup: Option, + classic: Option, +) -> Error { + let mut completion_error = Some(error); + if let Some(idwt) = idwt { + if let Err(error) = idwt.finish() { + accumulate_completion_error(&mut completion_error, error); + } + } + if let Some(cleanup) = cleanup { + if let Err(error) = cleanup.finish() { + accumulate_completion_error(&mut completion_error, error); + } + } + if let Some(classic) = classic { + if let Err(error) = classic.finish() { + accumulate_completion_error(&mut completion_error, error); + } + } + completion_error.expect("primary decode error is always present") +} + +pub(in crate::decoder) fn finish_decode_statuses( + cleanup: Option, + classic: Option, +) -> Result<(), Error> { + let mut completion_error = None; + if let Some(cleanup) = cleanup { + if let Err(error) = cleanup.finish() { + accumulate_completion_error(&mut completion_error, error); + } + } + if let Some(classic) = classic { + if let Err(error) = classic.finish() { + accumulate_completion_error(&mut completion_error, error); + } + } + completion_error.map_or(Ok(()), Err) +} + +pub(in crate::decoder) struct PendingDecodeCompletion { + store: Option, + idwt: Option, + cleanup: Option, + classic: Option, + decoded: Option>, + resources: Option, +} + +impl PendingDecodeCompletion { + pub(in crate::decoder) fn new( + store: Option, + idwt: Option, + cleanup: Option, + classic: Option, + decoded: Vec, + resources: Option, + ) -> Self { + Self { + store, + idwt, + cleanup, + classic, + decoded: Some(decoded), + resources, + } + } + + pub(in crate::decoder) fn is_complete(&self) -> Result { + self.store + .as_ref() + .map_or(Ok(true), CudaQueuedJ2kStoreBatch::is_complete) + .map_err(cuda_error) + } + + fn abandon_remaining(&mut self) { + if let Some(idwt) = self.idwt.take() { + core::mem::forget(idwt); + } + if let Some(cleanup) = self.cleanup.take() { + core::mem::forget(cleanup); + } + if let Some(classic) = self.classic.take() { + core::mem::forget(classic); + } + if let Some(decoded) = self.decoded.take() { + core::mem::forget(decoded); + } + if let Some(resources) = self.resources.take() { + core::mem::forget(resources); + } + } + + pub(in crate::decoder) fn complete(&mut self) -> Result<(), Error> { + let has_status_readback = self + .cleanup + .as_ref() + .is_some_and(PendingCleanup::has_status_readback) + || self + .classic + .as_ref() + .is_some_and(QueuedComponentClassicDecode::has_status_readback); + let mut completion_error = None; + if let Some(cleanup) = self.cleanup.take() { + if let Err(error) = cleanup.finish() { + accumulate_completion_error(&mut completion_error, error); + } + } + if let Some(classic) = self.classic.take() { + if let Err(error) = classic.finish() { + accumulate_completion_error(&mut completion_error, error); + } + } + let status_established_completion = has_status_readback + && completion_error + .as_ref() + .is_none_or(|error: &Error| !error.completion_is_uncertain()); + if let Some(store) = self.store.take() { + let store_result = if status_established_completion { + // SAFETY: the group status operation is ordered after the + // final store on the same stream and established completion. + unsafe { + store.release_after_stream_completion(); + } + Ok(()) + } else { + store.finish().map(|_| ()).map_err(cuda_error) + }; + if let Err(error) = store_result { + accumulate_completion_error(&mut completion_error, error); + self.abandon_remaining(); + return Err(completion_error.expect("store error was recorded")); + } + } + if let Some(mut idwt) = self.idwt.take() { + if let Err(error) = idwt.release_after_completion() { + accumulate_completion_error(&mut completion_error, error); + } + } + self.decoded.take(); + self.resources.take(); + completion_error.map_or(Ok(()), Err) + } +} + +fn accumulate_completion_error(current: &mut Option, error: Error) { + *current = Some(match current.take() { + Some(primary) => combine_cuda_cleanup_errors(primary, error), + None => error, + }); +} + +impl Drop for PendingDecodeCompletion { + fn drop(&mut self) { + let _ = self.complete(); + } +} + +#[cfg(test)] +mod tests { + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + + use super::{retire_decode_after_error, PendingCleanup}; + use crate::Error; + + struct FailingCleanup { + finished: Arc, + } + + impl PendingCleanup for FailingCleanup { + fn has_status_readback(&self) -> bool { + false + } + + fn finish(self) -> Result<(), Error> { + self.finished.store(true, Ordering::Relaxed); + Err(Error::UnsupportedCudaRequest { + reason: "cleanup failure", + }) + } + } + + #[test] + fn error_retirement_attempts_cleanup_and_preserves_both_errors() { + let finished = Arc::new(AtomicBool::new(false)); + let error = retire_decode_after_error( + Error::UnsupportedCudaRequest { + reason: "primary failure", + }, + None, + Some(FailingCleanup { + finished: finished.clone(), + }), + None, + ); + + assert!(finished.load(Ordering::Relaxed)); + let Error::CudaCleanupFailed { primary, cleanup } = error else { + panic!("retirement must preserve primary and cleanup errors") + }; + assert!(matches!( + *primary, + Error::UnsupportedCudaRequest { + reason: "primary failure" + } + )); + assert!(matches!( + *cleanup, + Error::UnsupportedCudaRequest { + reason: "cleanup failure" + } + )); + } +} diff --git a/crates/j2k-cuda/src/decoder/plan.rs b/crates/j2k-cuda/src/decoder/plan.rs index 676ed67f..2c4c7dee 100644 --- a/crates/j2k-cuda/src/decoder/plan.rs +++ b/crates/j2k-cuda/src/decoder/plan.rs @@ -3,502 +3,47 @@ use super::{ native_decode_error, profile, CudaHtj2kColorDecodePlans, CudaHtj2kDecodePlan, CudaHtj2kDecodeProfileDetail, CudaHtj2kProfileReport, CudaHtj2kTransform, DecodeSettings, - Error, J2kDecoder, NativeDecoderContext, NativeImage, PixelFormat, Rect, + DeviceDecodePlan, DeviceDecodeRequest, Downscale, Error, J2kDecoder, NativeDecoderContext, + NativeImage, PixelFormat, Rect, }; #[cfg(feature = "cuda-runtime")] +use crate::allocation::HostPhaseBudget; +#[cfg(feature = "cuda-runtime")] +use j2k_native::{J2kReferencedClassicPlan, J2kReferencedHtj2kPlan}; +#[cfg(feature = "cuda-runtime")] mod color_owners; #[cfg(feature = "cuda-runtime")] -use self::color_owners::flatten_cuda_color_components; - -impl J2kDecoder<'_> { - /// Build a flat CUDA HTJ2K grayscale decode plan and return stage timings. - pub(crate) fn build_cuda_htj2k_grayscale_plan_with_profile( - &mut self, - fmt: PixelFormat, - ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { - let total_start = profile::profile_now(true); - - let parse_start = profile::profile_now(true); - let image = NativeImage::new(self.bytes, &DecodeSettings::default()) - .map_err(native_decode_error)?; - let parse_us = profile::elapsed_us(parse_start); - - let plan_start = profile::profile_now(true); - let mut native_context = NativeDecoderContext::default(); - let native_plan = image - .build_direct_grayscale_plan_with_context(&mut native_context) - .map_err(native_decode_error)?; - let plan_us = profile::elapsed_us(plan_start); - - let flatten_start = profile::profile_now(true); - let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan(&native_plan, fmt, (0, 0))?; - let flatten_us = profile::elapsed_us(flatten_start); - - let report = CudaHtj2kProfileReport { - parse_us, - plan_us, - flatten_us, - total_us: profile::elapsed_us(total_start), - block_count: cuda_plan.block_count(), - classic_block_count: cuda_plan.classic_code_blocks().len(), - ht_block_count: cuda_plan.code_blocks().len(), - payload_bytes: cuda_plan.payload().len(), - dispatch_count: 0, - residency: crate::SurfaceResidency::CudaResidentDecode, - detail: CudaHtj2kDecodeProfileDetail::default(), - ..CudaHtj2kProfileReport::default() - }; - report.emit("plan"); - Ok((cuda_plan, report)) - } - - /// Build a flat CUDA HTJ2K grayscale region decode plan and return stage timings. - pub(crate) fn build_cuda_htj2k_grayscale_region_plan_with_profile( - &mut self, - fmt: PixelFormat, - roi: Rect, - ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { - let total_start = profile::profile_now(true); - - let parse_start = profile::profile_now(true); - let image = NativeImage::new(self.bytes, &DecodeSettings::default()) - .map_err(native_decode_error)?; - let parse_us = profile::elapsed_us(parse_start); - - let plan_start = profile::profile_now(true); - let mut native_context = NativeDecoderContext::default(); - let native_plan = image - .build_direct_grayscale_plan_region_with_context( - &mut native_context, - (roi.x, roi.y, roi.w, roi.h), - ) - .map_err(native_decode_error)?; - let plan_us = profile::elapsed_us(plan_start); - - let flatten_start = profile::profile_now(true); - let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( - &native_plan, - fmt, - (roi.x, roi.y), - (roi.w, roi.h), - )?; - let flatten_us = profile::elapsed_us(flatten_start); - - let report = CudaHtj2kProfileReport { - parse_us, - plan_us, - flatten_us, - total_us: profile::elapsed_us(total_start), - block_count: cuda_plan.block_count(), - classic_block_count: cuda_plan.classic_code_blocks().len(), - ht_block_count: cuda_plan.code_blocks().len(), - payload_bytes: cuda_plan.payload().len(), - dispatch_count: 0, - residency: crate::SurfaceResidency::CudaResidentDecode, - detail: CudaHtj2kDecodeProfileDetail::default(), - ..CudaHtj2kProfileReport::default() - }; - report.emit("plan"); - Ok((cuda_plan, report)) - } - - /// Build a flat reduced-resolution CUDA HTJ2K grayscale decode plan and - /// return stage timings. - pub(crate) fn build_cuda_htj2k_grayscale_scaled_plan_with_profile( - &mut self, - fmt: PixelFormat, - output_dimensions: (u32, u32), - ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { - let total_start = profile::profile_now(true); - - let parse_start = profile::profile_now(true); - let image = NativeImage::new( - self.bytes, - &DecodeSettings { - target_resolution: Some(output_dimensions), - ..DecodeSettings::default() - }, - ) - .map_err(native_decode_error)?; - let parse_us = profile::elapsed_us(parse_start); - - let plan_start = profile::profile_now(true); - let mut native_context = NativeDecoderContext::default(); - let native_plan = image - .build_direct_grayscale_plan_with_context(&mut native_context) - .map_err(native_decode_error)?; - let plan_us = profile::elapsed_us(plan_start); - - let flatten_start = profile::profile_now(true); - let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan(&native_plan, fmt, (0, 0))?; - let flatten_us = profile::elapsed_us(flatten_start); - - let report = CudaHtj2kProfileReport { - parse_us, - plan_us, - flatten_us, - total_us: profile::elapsed_us(total_start), - block_count: cuda_plan.block_count(), - classic_block_count: cuda_plan.classic_code_blocks().len(), - ht_block_count: cuda_plan.code_blocks().len(), - payload_bytes: cuda_plan.payload().len(), - dispatch_count: 0, - residency: crate::SurfaceResidency::CudaResidentDecode, - detail: CudaHtj2kDecodeProfileDetail::default(), - ..CudaHtj2kProfileReport::default() - }; - report.emit("plan"); - Ok((cuda_plan, report)) - } - - /// Build a flat reduced-resolution CUDA HTJ2K grayscale region decode - /// plan and return stage timings. - pub(crate) fn build_cuda_htj2k_grayscale_region_scaled_plan_with_profile( - &mut self, - fmt: PixelFormat, - scaled_roi: Rect, - output_dimensions: (u32, u32), - ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { - let total_start = profile::profile_now(true); - - let parse_start = profile::profile_now(true); - let image = NativeImage::new( - self.bytes, - &DecodeSettings { - target_resolution: Some(output_dimensions), - ..DecodeSettings::default() - }, - ) - .map_err(native_decode_error)?; - let parse_us = profile::elapsed_us(parse_start); - - let plan_start = profile::profile_now(true); - let mut native_context = NativeDecoderContext::default(); - let native_plan = image - .build_direct_grayscale_plan_region_with_context( - &mut native_context, - (scaled_roi.x, scaled_roi.y, scaled_roi.w, scaled_roi.h), - ) - .map_err(native_decode_error)?; - let plan_us = profile::elapsed_us(plan_start); - - let flatten_start = profile::profile_now(true); - let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( - &native_plan, - fmt, - (scaled_roi.x, scaled_roi.y), - (scaled_roi.w, scaled_roi.h), - )?; - let flatten_us = profile::elapsed_us(flatten_start); - - let report = CudaHtj2kProfileReport { - parse_us, - plan_us, - flatten_us, - total_us: profile::elapsed_us(total_start), - block_count: cuda_plan.block_count(), - classic_block_count: cuda_plan.classic_code_blocks().len(), - ht_block_count: cuda_plan.code_blocks().len(), - payload_bytes: cuda_plan.payload().len(), - dispatch_count: 0, - residency: crate::SurfaceResidency::CudaResidentDecode, - detail: CudaHtj2kDecodeProfileDetail::default(), - ..CudaHtj2kProfileReport::default() - }; - report.emit("plan"); - Ok((cuda_plan, report)) - } - - /// Build flat CUDA HTJ2K RGB component plans and return stage timings. - #[cfg(feature = "cuda-runtime")] - pub(super) fn build_cuda_htj2k_color_plans_with_profile( - &mut self, - fmt: PixelFormat, - ) -> Result { - let mut native_context = NativeDecoderContext::default(); - build_cuda_htj2k_color_plans_from_bytes_with_profile(self.bytes, fmt, &mut native_context) - } - - #[cfg(feature = "cuda-runtime")] - pub(super) fn build_cuda_htj2k_color_scaled_plans_with_profile( - &mut self, - fmt: PixelFormat, - output_dimensions: (u32, u32), - ) -> Result { - let total_start = profile::profile_now(true); - - let parse_start = profile::profile_now(true); - let image = NativeImage::new( - self.bytes, - &DecodeSettings { - target_resolution: Some(output_dimensions), - ..DecodeSettings::default() - }, - ) - .map_err(native_decode_error)?; - let parse_us = profile::elapsed_us(parse_start); - - let plan_start = profile::profile_now(true); - let mut native_context = NativeDecoderContext::default(); - let native_plan = image - .build_direct_color_plan_with_context(&mut native_context) - .map_err(native_decode_error)?; - let plan_us = profile::elapsed_us(plan_start); - - let flatten_start = profile::profile_now(true); - let (payload, components) = flatten_cuda_color_components( - &native_plan, - fmt, - None, - "j2k CUDA scaled color decode plans", - )?; - let flatten_us = profile::elapsed_us(flatten_start); - let block_count = components - .iter() - .map(CudaHtj2kDecodePlan::block_count) - .sum::(); - let classic_block_count = components - .iter() - .map(|plan| plan.classic_code_blocks().len()) - .sum::(); - let ht_block_count = components - .iter() - .map(|plan| plan.code_blocks().len()) - .sum::(); - let payload_bytes = payload.len(); - let report = CudaHtj2kProfileReport { - parse_us, - plan_us, - flatten_us, - total_us: profile::elapsed_us(total_start), - block_count, - classic_block_count, - ht_block_count, - payload_bytes, - dispatch_count: 0, - residency: crate::SurfaceResidency::CudaResidentDecode, - detail: CudaHtj2kDecodeProfileDetail::default(), - ..CudaHtj2kProfileReport::default() - }; - report.emit("plan"); - - Ok(CudaHtj2kColorDecodePlans { - dimensions: native_plan.dimensions, - mct_dimensions: native_plan.dimensions, - bit_depths: native_plan.bit_depths, - mct: native_plan.mct, - transform: CudaHtj2kTransform::from_native(native_plan.transform), - payload, - components, - report, - }) - } - - #[cfg(feature = "cuda-runtime")] - pub(super) fn build_cuda_htj2k_color_region_plans_with_profile( - &mut self, - fmt: PixelFormat, - roi: Rect, - ) -> Result { - let total_start = profile::profile_now(true); - - let parse_start = profile::profile_now(true); - let image = NativeImage::new(self.bytes, &DecodeSettings::default()) - .map_err(native_decode_error)?; - let parse_us = profile::elapsed_us(parse_start); - - let plan_start = profile::profile_now(true); - let mut native_context = NativeDecoderContext::default(); - let native_plan = image - .build_direct_color_plan_with_context(&mut native_context) - .map_err(native_decode_error)?; - let plan_us = profile::elapsed_us(plan_start); - - let flatten_start = profile::profile_now(true); - let (payload, components) = flatten_cuda_color_components( - &native_plan, - fmt, - Some(((roi.x, roi.y), (roi.w, roi.h))), - "j2k CUDA region color decode plans", - )?; - let flatten_us = profile::elapsed_us(flatten_start); - let block_count = components - .iter() - .map(CudaHtj2kDecodePlan::block_count) - .sum::(); - let classic_block_count = components - .iter() - .map(|plan| plan.classic_code_blocks().len()) - .sum::(); - let ht_block_count = components - .iter() - .map(|plan| plan.code_blocks().len()) - .sum::(); - let payload_bytes = payload.len(); - let report = CudaHtj2kProfileReport { - parse_us, - plan_us, - flatten_us, - total_us: profile::elapsed_us(total_start), - block_count, - classic_block_count, - ht_block_count, - payload_bytes, - dispatch_count: 0, - residency: crate::SurfaceResidency::CudaResidentDecode, - detail: CudaHtj2kDecodeProfileDetail::default(), - ..CudaHtj2kProfileReport::default() - }; - report.emit("plan"); - - Ok(CudaHtj2kColorDecodePlans { - dimensions: (roi.w, roi.h), - mct_dimensions: native_plan.dimensions, - bit_depths: native_plan.bit_depths, - mct: native_plan.mct, - transform: CudaHtj2kTransform::from_native(native_plan.transform), - payload, - components, - report, - }) - } - - #[cfg(feature = "cuda-runtime")] - pub(super) fn build_cuda_htj2k_color_region_scaled_plans_with_profile( - &mut self, - fmt: PixelFormat, - scaled_roi: Rect, - output_dimensions: (u32, u32), - ) -> Result { - let total_start = profile::profile_now(true); - - let parse_start = profile::profile_now(true); - let image = NativeImage::new( - self.bytes, - &DecodeSettings { - target_resolution: Some(output_dimensions), - ..DecodeSettings::default() - }, - ) - .map_err(native_decode_error)?; - let parse_us = profile::elapsed_us(parse_start); - - let plan_start = profile::profile_now(true); - let mut native_context = NativeDecoderContext::default(); - let native_plan = image - .build_direct_color_plan_with_context(&mut native_context) - .map_err(native_decode_error)?; - let plan_us = profile::elapsed_us(plan_start); - - let flatten_start = profile::profile_now(true); - let (payload, components) = flatten_cuda_color_components( - &native_plan, - fmt, - Some(((scaled_roi.x, scaled_roi.y), (scaled_roi.w, scaled_roi.h))), - "j2k CUDA scaled region color decode plans", - )?; - let flatten_us = profile::elapsed_us(flatten_start); - let block_count = components - .iter() - .map(CudaHtj2kDecodePlan::block_count) - .sum::(); - let classic_block_count = components - .iter() - .map(|plan| plan.classic_code_blocks().len()) - .sum::(); - let ht_block_count = components - .iter() - .map(|plan| plan.code_blocks().len()) - .sum::(); - let payload_bytes = payload.len(); - let report = CudaHtj2kProfileReport { - parse_us, - plan_us, - flatten_us, - total_us: profile::elapsed_us(total_start), - block_count, - classic_block_count, - ht_block_count, - payload_bytes, - dispatch_count: 0, - residency: crate::SurfaceResidency::CudaResidentDecode, - detail: CudaHtj2kDecodeProfileDetail::default(), - ..CudaHtj2kProfileReport::default() - }; - report.emit("plan"); - - Ok(CudaHtj2kColorDecodePlans { - dimensions: (scaled_roi.w, scaled_roi.h), - mct_dimensions: native_plan.dimensions, - bit_depths: native_plan.bit_depths, - mct: native_plan.mct, - transform: CudaHtj2kTransform::from_native(native_plan.transform), - payload, - components, - report, - }) - } -} +use self::color_owners::{ + flatten_cuda_color_components, flatten_referenced_classic_cuda_color_tile_components, + flatten_referenced_cuda_color_tile_components, +}; #[cfg(feature = "cuda-runtime")] -pub(super) fn build_cuda_htj2k_color_plans_from_bytes_with_profile<'a>( - input: &'a [u8], - fmt: PixelFormat, - native_context: &mut NativeDecoderContext<'a>, -) -> Result { - let total_start = profile::profile_now(true); - - let parse_start = profile::profile_now(true); - let image = NativeImage::new(input, &DecodeSettings::default()).map_err(native_decode_error)?; - let parse_us = profile::elapsed_us(parse_start); - - let plan_start = profile::profile_now(true); - let native_plan = image - .build_direct_color_plan_with_context(native_context) - .map_err(native_decode_error)?; - let plan_us = profile::elapsed_us(plan_start); +mod color; +#[cfg(feature = "cuda-runtime")] +mod color_decoder; +#[cfg(feature = "cuda-runtime")] +mod color_referenced; +mod grayscale; - let flatten_start = profile::profile_now(true); - let (payload, components) = - flatten_cuda_color_components(&native_plan, fmt, None, "j2k CUDA color decode plans")?; - let flatten_us = profile::elapsed_us(flatten_start); - let block_count = components - .iter() - .map(CudaHtj2kDecodePlan::block_count) - .sum::(); - let classic_block_count = components - .iter() - .map(|plan| plan.classic_code_blocks().len()) - .sum::(); - let ht_block_count = components - .iter() - .map(|plan| plan.code_blocks().len()) - .sum::(); - let payload_bytes = payload.len(); - let report = CudaHtj2kProfileReport { - parse_us, - plan_us, - flatten_us, - total_us: profile::elapsed_us(total_start), - block_count, - classic_block_count, - ht_block_count, - payload_bytes, - dispatch_count: 0, - residency: crate::SurfaceResidency::CudaResidentDecode, - detail: CudaHtj2kDecodeProfileDetail::default(), - ..CudaHtj2kProfileReport::default() - }; - report.emit("plan"); +#[cfg(feature = "cuda-runtime")] +pub(super) use self::color::{ + build_cuda_color_plan_from_bytes_for_device_plan_with_profile, + build_cuda_htj2k_color_plans_from_bytes_with_profile, +}; +#[cfg(feature = "cuda-runtime")] +pub(super) use self::color_referenced::{ + build_cuda_classic_color_plans_from_referenced_with_profile, + build_cuda_htj2k_color_plans_from_referenced_with_profile, +}; +pub(super) use self::grayscale::{ + build_cuda_classic_grayscale_plans_from_referenced_with_profile, + build_cuda_htj2k_grayscale_plan_from_bytes_for_device_plan_with_profile, + build_cuda_htj2k_grayscale_plan_from_bytes_with_profile, + build_cuda_htj2k_grayscale_plans_from_referenced_with_profile, +}; - Ok(CudaHtj2kColorDecodePlans { - dimensions: native_plan.dimensions, - mct_dimensions: native_plan.dimensions, - bit_depths: native_plan.bit_depths, - mct: native_plan.mct, - transform: CudaHtj2kTransform::from_native(native_plan.transform), - payload, - components, - report, - }) +#[cfg(feature = "cuda-runtime")] +const fn rgba_bit_depths_from_rgb(bit_depths: [u8; 3]) -> [u8; 4] { + [bit_depths[0], bit_depths[1], bit_depths[2], 0] } diff --git a/crates/j2k-cuda/src/decoder/plan/color.rs b/crates/j2k-cuda/src/decoder/plan/color.rs new file mode 100644 index 00000000..7c687dff --- /dev/null +++ b/crates/j2k-cuda/src/decoder/plan/color.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! One-shot classic and HTJ2K color-plan construction. + +use super::{ + flatten_cuda_color_components, native_decode_error, profile, rgba_bit_depths_from_rgb, + CudaHtj2kColorDecodePlans, CudaHtj2kDecodePlan, CudaHtj2kDecodeProfileDetail, + CudaHtj2kProfileReport, CudaHtj2kTransform, DecodeSettings, DeviceDecodePlan, Downscale, Error, + NativeDecoderContext, NativeImage, PixelFormat, Rect, +}; + +#[cfg(feature = "cuda-runtime")] +pub(in crate::decoder) fn build_cuda_color_plan_from_bytes_for_device_plan_with_profile<'a>( + input: &'a [u8], + fmt: PixelFormat, + device_plan: DeviceDecodePlan, + settings: DecodeSettings, + native_context: &mut NativeDecoderContext<'a>, +) -> Result { + let total_start = profile::profile_now(true); + let parse_start = profile::profile_now(true); + let target_resolution = (device_plan.scale() != Downscale::None).then_some(( + device_plan + .source_dims() + .0 + .div_ceil(device_plan.scale().denominator()), + device_plan + .source_dims() + .1 + .div_ceil(device_plan.scale().denominator()), + )); + let image = NativeImage::new( + input, + &DecodeSettings { + target_resolution, + ..settings + }, + ) + .map_err(native_decode_error)?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let native_plan = image + .build_direct_color_plan_with_context(native_context) + .map_err(native_decode_error)?; + let plan_us = profile::elapsed_us(plan_start); + let full = Rect::full(device_plan.source_dims()); + let selected_rect = if device_plan.source_rect() == full { + None + } else if device_plan.scale() == Downscale::None { + Some(device_plan.source_rect()) + } else { + Some(device_plan.output_rect()) + }; + + let flatten_start = profile::profile_now(true); + let (payload, components) = flatten_cuda_color_components( + &native_plan, + fmt, + selected_rect.map(|rect| ((rect.x, rect.y), (rect.w, rect.h))), + "j2k CUDA exact classic color plan owners", + )?; + let flatten_us = profile::elapsed_us(flatten_start); + let block_count = components + .iter() + .map(CudaHtj2kDecodePlan::block_count) + .sum::(); + let classic_block_count = components + .iter() + .map(|plan| plan.classic_code_blocks().len()) + .sum::(); + let ht_block_count = components + .iter() + .map(|plan| plan.code_blocks().len()) + .sum::(); + let payload_bytes = payload.len(); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count, + classic_block_count, + ht_block_count, + payload_bytes, + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("classic_prepared_plan"); + Ok(CudaHtj2kColorDecodePlans { + output_index: 0, + dimensions: device_plan.output_dims(), + mct_dimensions: native_plan.dimensions, + bit_depths: rgba_bit_depths_from_rgb(native_plan.bit_depths), + mct: native_plan.mct, + transform: CudaHtj2kTransform::from_native(native_plan.transform), + payload, + components, + report, + }) +} + +#[cfg(feature = "cuda-runtime")] +pub(in crate::decoder) fn build_cuda_htj2k_color_plans_from_bytes_with_profile<'a>( + input: &'a [u8], + fmt: PixelFormat, + native_context: &mut NativeDecoderContext<'a>, +) -> Result { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new(input, &DecodeSettings::default()).map_err(native_decode_error)?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let native_plan = image + .build_direct_color_plan_with_context(native_context) + .map_err(native_decode_error)?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let (payload, components) = + flatten_cuda_color_components(&native_plan, fmt, None, "j2k CUDA color decode plans")?; + let flatten_us = profile::elapsed_us(flatten_start); + let block_count = components + .iter() + .map(CudaHtj2kDecodePlan::block_count) + .sum::(); + let classic_block_count = components + .iter() + .map(|plan| plan.classic_code_blocks().len()) + .sum::(); + let ht_block_count = components + .iter() + .map(|plan| plan.code_blocks().len()) + .sum::(); + let payload_bytes = payload.len(); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count, + classic_block_count, + ht_block_count, + payload_bytes, + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + + Ok(CudaHtj2kColorDecodePlans { + output_index: 0, + dimensions: native_plan.dimensions, + mct_dimensions: native_plan.dimensions, + bit_depths: rgba_bit_depths_from_rgb(native_plan.bit_depths), + mct: native_plan.mct, + transform: CudaHtj2kTransform::from_native(native_plan.transform), + payload, + components, + report, + }) +} diff --git a/crates/j2k-cuda/src/decoder/plan/color_decoder.rs b/crates/j2k-cuda/src/decoder/plan/color_decoder.rs new file mode 100644 index 00000000..1c175295 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/plan/color_decoder.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Region and reduced-resolution color planning on the decoder facade. + +use super::{ + build_cuda_htj2k_color_plans_from_bytes_with_profile, flatten_cuda_color_components, + native_decode_error, profile, rgba_bit_depths_from_rgb, CudaHtj2kColorDecodePlans, + CudaHtj2kDecodePlan, CudaHtj2kDecodeProfileDetail, CudaHtj2kProfileReport, CudaHtj2kTransform, + DecodeSettings, Error, J2kDecoder, NativeDecoderContext, NativeImage, PixelFormat, Rect, +}; + +impl J2kDecoder<'_> { + /// Build flat CUDA HTJ2K RGB component plans and return stage timings. + #[cfg(feature = "cuda-runtime")] + pub(in crate::decoder) fn build_cuda_htj2k_color_plans_with_profile( + &mut self, + fmt: PixelFormat, + ) -> Result { + let mut native_context = NativeDecoderContext::default(); + build_cuda_htj2k_color_plans_from_bytes_with_profile(self.bytes, fmt, &mut native_context) + } + + #[cfg(feature = "cuda-runtime")] + pub(in crate::decoder) fn build_cuda_htj2k_color_scaled_plans_with_profile( + &mut self, + fmt: PixelFormat, + output_dimensions: (u32, u32), + ) -> Result { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new( + self.bytes, + &DecodeSettings { + target_resolution: Some(output_dimensions), + ..DecodeSettings::default() + }, + ) + .map_err(native_decode_error)?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_color_plan_with_context(&mut native_context) + .map_err(native_decode_error)?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let (payload, components) = flatten_cuda_color_components( + &native_plan, + fmt, + None, + "j2k CUDA scaled color decode plans", + )?; + let flatten_us = profile::elapsed_us(flatten_start); + let block_count = components + .iter() + .map(CudaHtj2kDecodePlan::block_count) + .sum::(); + let classic_block_count = components + .iter() + .map(|plan| plan.classic_code_blocks().len()) + .sum::(); + let ht_block_count = components + .iter() + .map(|plan| plan.code_blocks().len()) + .sum::(); + let payload_bytes = payload.len(); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count, + classic_block_count, + ht_block_count, + payload_bytes, + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + + Ok(CudaHtj2kColorDecodePlans { + output_index: 0, + dimensions: native_plan.dimensions, + mct_dimensions: native_plan.dimensions, + bit_depths: rgba_bit_depths_from_rgb(native_plan.bit_depths), + mct: native_plan.mct, + transform: CudaHtj2kTransform::from_native(native_plan.transform), + payload, + components, + report, + }) + } + + #[cfg(feature = "cuda-runtime")] + pub(in crate::decoder) fn build_cuda_htj2k_color_region_plans_with_profile( + &mut self, + fmt: PixelFormat, + roi: Rect, + ) -> Result { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new(self.bytes, &DecodeSettings::default()) + .map_err(native_decode_error)?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_color_plan_with_context(&mut native_context) + .map_err(native_decode_error)?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let (payload, components) = flatten_cuda_color_components( + &native_plan, + fmt, + Some(((roi.x, roi.y), (roi.w, roi.h))), + "j2k CUDA region color decode plans", + )?; + let flatten_us = profile::elapsed_us(flatten_start); + let block_count = components + .iter() + .map(CudaHtj2kDecodePlan::block_count) + .sum::(); + let classic_block_count = components + .iter() + .map(|plan| plan.classic_code_blocks().len()) + .sum::(); + let ht_block_count = components + .iter() + .map(|plan| plan.code_blocks().len()) + .sum::(); + let payload_bytes = payload.len(); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count, + classic_block_count, + ht_block_count, + payload_bytes, + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + + Ok(CudaHtj2kColorDecodePlans { + output_index: 0, + dimensions: (roi.w, roi.h), + mct_dimensions: native_plan.dimensions, + bit_depths: rgba_bit_depths_from_rgb(native_plan.bit_depths), + mct: native_plan.mct, + transform: CudaHtj2kTransform::from_native(native_plan.transform), + payload, + components, + report, + }) + } + + #[cfg(feature = "cuda-runtime")] + pub(in crate::decoder) fn build_cuda_htj2k_color_region_scaled_plans_with_profile( + &mut self, + fmt: PixelFormat, + scaled_roi: Rect, + output_dimensions: (u32, u32), + ) -> Result { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new( + self.bytes, + &DecodeSettings { + target_resolution: Some(output_dimensions), + ..DecodeSettings::default() + }, + ) + .map_err(native_decode_error)?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_color_plan_with_context(&mut native_context) + .map_err(native_decode_error)?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let (payload, components) = flatten_cuda_color_components( + &native_plan, + fmt, + Some(((scaled_roi.x, scaled_roi.y), (scaled_roi.w, scaled_roi.h))), + "j2k CUDA scaled region color decode plans", + )?; + let flatten_us = profile::elapsed_us(flatten_start); + let block_count = components + .iter() + .map(CudaHtj2kDecodePlan::block_count) + .sum::(); + let classic_block_count = components + .iter() + .map(|plan| plan.classic_code_blocks().len()) + .sum::(); + let ht_block_count = components + .iter() + .map(|plan| plan.code_blocks().len()) + .sum::(); + let payload_bytes = payload.len(); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count, + classic_block_count, + ht_block_count, + payload_bytes, + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + + Ok(CudaHtj2kColorDecodePlans { + output_index: 0, + dimensions: (scaled_roi.w, scaled_roi.h), + mct_dimensions: native_plan.dimensions, + bit_depths: rgba_bit_depths_from_rgb(native_plan.bit_depths), + mct: native_plan.mct, + transform: CudaHtj2kTransform::from_native(native_plan.transform), + payload, + components, + report, + }) + } +} diff --git a/crates/j2k-cuda/src/decoder/plan/color_owners.rs b/crates/j2k-cuda/src/decoder/plan/color_owners.rs index 835a3a4c..1e585290 100644 --- a/crates/j2k-cuda/src/decoder/plan/color_owners.rs +++ b/crates/j2k-cuda/src/decoder/plan/color_owners.rs @@ -2,12 +2,19 @@ //! Aggregate ownership for flattened multi-component CUDA decode plans. +mod referenced; + use j2k_core::PixelFormat; use j2k_native::J2kDirectColorPlan; use crate::allocation::HostPhaseBudget; use crate::{CudaHtj2kDecodePlan, Error}; +pub(super) use referenced::{ + flatten_referenced_classic_cuda_color_tile_components, + flatten_referenced_cuda_color_tile_components, +}; + use super::super::CudaHtj2kColorDecodePlans; type OutputRegion = ((u32, u32), (u32, u32)); diff --git a/crates/j2k-cuda/src/decoder/plan/color_owners/referenced.rs b/crates/j2k-cuda/src/decoder/plan/color_owners/referenced.rs new file mode 100644 index 00000000..3cebc02e --- /dev/null +++ b/crates/j2k-cuda/src/decoder/plan/color_owners/referenced.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_core::PixelFormat; +use j2k_native::{ + HtCodeBlockPayloadRanges, J2kClassicCodeBlockPayload, J2kCodestreamRange, + J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, +}; + +use crate::allocation::HostPhaseBudget; +use crate::{CudaHtj2kDecodePlan, Error}; + +#[expect( + clippy::too_many_arguments, + reason = "referenced tile bridge keeps destination geometry and retained owners explicit" +)] +pub(in crate::decoder::plan) fn flatten_referenced_cuda_color_tile_components( + component_plans: &[J2kDirectGrayscalePlan], + payloads: &[HtCodeBlockPayloadRanges], + encoded: &[u8], + format: PixelFormat, + output_origin: (u32, u32), + output_dimensions: (u32, u32), + shared_payload: &mut Vec, + host_budget: &mut HostPhaseBudget, +) -> Result, Error> { + let mut components = host_budget.try_vec_with_capacity(component_plans.len())?; + let mut payload_offset = 0usize; + for component_plan in component_plans { + let component_payload_count = component_plan + .steps + .iter() + .map(|step| match step { + J2kDirectGrayscaleStep::HtSubBand(subband) => subband.jobs.len(), + _ => 0, + }) + .try_fold(0usize, usize::checked_add) + .ok_or(Error::UnsupportedCudaRequest { + reason: super::super::super::CUDA_HTJ2K_BATCH_PAYLOAD_TOO_LARGE, + })?; + let payload_end = payload_offset.checked_add(component_payload_count).ok_or( + Error::UnsupportedCudaRequest { + reason: super::super::super::CUDA_HTJ2K_BATCH_PAYLOAD_TOO_LARGE, + }, + )?; + let component_payloads = + payloads + .get(payload_offset..payload_end) + .ok_or(Error::UnsupportedCudaRequest { + reason: + "prepared CUDA color tile payload ranges do not match component geometry", + })?; + components.push( + CudaHtj2kDecodePlan::from_referenced_tile_grayscale_plan_into_shared( + component_plan, + component_payloads, + encoded, + format, + output_origin, + output_dimensions, + shared_payload, + host_budget, + )?, + ); + payload_offset = payload_end; + } + if payload_offset != payloads.len() { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA color tile payload ranges contain trailing jobs", + }); + } + Ok(components) +} + +#[expect( + clippy::too_many_arguments, + reason = "referenced classic tile bridge keeps ranges and destination geometry explicit" +)] +pub(in crate::decoder::plan) fn flatten_referenced_classic_cuda_color_tile_components( + component_plans: &[J2kDirectGrayscalePlan], + payloads: &[J2kClassicCodeBlockPayload], + ranges: &[J2kCodestreamRange], + encoded: &[u8], + format: PixelFormat, + output_origin: (u32, u32), + output_dimensions: (u32, u32), + shared_payload: &mut Vec, + host_budget: &mut HostPhaseBudget, +) -> Result, Error> { + let mut components = host_budget.try_vec_with_capacity(component_plans.len())?; + let mut payload_offset = 0usize; + for component_plan in component_plans { + let component_payload_count = component_plan + .steps + .iter() + .map(|step| match step { + J2kDirectGrayscaleStep::ClassicSubBand(subband) => subband.jobs.len(), + _ => 0, + }) + .try_fold(0usize, usize::checked_add) + .ok_or(Error::UnsupportedCudaRequest { + reason: super::super::super::CUDA_HTJ2K_BATCH_PAYLOAD_TOO_LARGE, + })?; + let payload_end = payload_offset.checked_add(component_payload_count).ok_or( + Error::UnsupportedCudaRequest { + reason: super::super::super::CUDA_HTJ2K_BATCH_PAYLOAD_TOO_LARGE, + }, + )?; + let component_payloads = + payloads + .get(payload_offset..payload_end) + .ok_or(Error::UnsupportedCudaRequest { + reason: + "prepared CUDA classic color tile payloads do not match component geometry", + })?; + components.push( + CudaHtj2kDecodePlan::from_referenced_classic_tile_grayscale_plan_into_shared( + component_plan, + component_payloads, + ranges, + encoded, + format, + output_origin, + output_dimensions, + shared_payload, + host_budget, + )?, + ); + payload_offset = payload_end; + } + if payload_offset != payloads.len() { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic color tile payloads contain trailing jobs", + }); + } + Ok(components) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use j2k::{prepare_batch, BatchDecodeOptions, EncodedImage}; + use j2k_native::{J2kDirectGrayscaleStep, J2kReferencedHtj2kPlan}; + + use super::*; + + #[test] + fn referenced_color_plan_retains_zero_entropy_components() { + let encoded = Arc::<[u8]>::from(j2k_test_support::openhtj2k_sigprop_overlap_fixture()); + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::clone(&encoded))], + BatchDecodeOptions::default(), + ) + .expect("prepare independent refinement-overlap fixture"); + let image = &prepared.groups()[0].images()[0]; + let referenced = image + .htj2k_plan() + .expect("retained HTJ2K plan") + .adapter_view() + .downcast_ref::() + .expect("native referenced HTJ2K plan adapter"); + let tile = &referenced.tiles()[0]; + let geometry = tile.color_geometry().expect("RGB tile geometry"); + let job_counts = geometry + .component_plans + .iter() + .map(|plan| { + plan.steps + .iter() + .map(|step| match step { + J2kDirectGrayscaleStep::HtSubBand(subband) => subband.jobs.len(), + _ => 0, + }) + .sum::() + }) + .collect::>(); + assert!( + job_counts.contains(&0), + "fixture must retain its zero-entropy component: {job_counts:?}" + ); + + let mut shared_payload = Vec::new(); + let mut budget = HostPhaseBudget::new("referenced zero-entropy color-plan test"); + let span = tile.payload_records(); + let payload_end = span.end_record().expect("tile payload span end"); + let components = flatten_referenced_cuda_color_tile_components( + &geometry.component_plans, + &referenced.payloads()[span.first_record..payload_end], + &encoded, + PixelFormat::Rgb8, + { + let output = image.plan().output_rect(); + (output.x, output.y) + }, + image.plan().output_dims(), + &mut shared_payload, + &mut budget, + ) + .expect("zero-entropy component remains an executable zero-coefficient plan"); + + assert_eq!(components.len(), 3); + assert_eq!( + components + .iter() + .map(CudaHtj2kDecodePlan::block_count) + .collect::>(), + job_counts + ); + } +} diff --git a/crates/j2k-cuda/src/decoder/plan/color_referenced.rs b/crates/j2k-cuda/src/decoder/plan/color_referenced.rs new file mode 100644 index 00000000..91f3b524 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/plan/color_referenced.rs @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Retained HTJ2K and classic color-plan construction. + +use super::{ + flatten_referenced_classic_cuda_color_tile_components, + flatten_referenced_cuda_color_tile_components, profile, rgba_bit_depths_from_rgb, + CudaHtj2kColorDecodePlans, CudaHtj2kDecodePlan, CudaHtj2kDecodeProfileDetail, + CudaHtj2kProfileReport, CudaHtj2kTransform, DeviceDecodePlan, Error, HostPhaseBudget, + J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, PixelFormat, +}; + +#[cfg(feature = "cuda-runtime")] +#[expect( + clippy::too_many_lines, + reason = "tile metadata, payload spans, component owners, and exact output geometry are validated atomically" +)] +pub(in crate::decoder) fn build_cuda_htj2k_color_plans_from_referenced_with_profile( + input: &[u8], + referenced: &J2kReferencedHtj2kPlan, + fmt: PixelFormat, + device_plan: DeviceDecodePlan, + shared_payload: &mut Vec, + host_budget: &mut HostPhaseBudget, +) -> Result, Error> { + let total_start = profile::profile_now(true); + let output_rect = referenced.output_rect(); + let output_dimensions = device_plan.output_dims(); + if (output_rect.width(), output_rect.height()) != output_dimensions { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA color tile output geometry is inconsistent", + }); + } + let mut colors = host_budget.try_vec_with_capacity(referenced.tiles().len())?; + let mut next_payload = 0usize; + for tile in referenced.tiles() { + let (geometry_dimensions, bit_depths, mct, transform, component_plans) = + if let Some(geometry) = tile.color_geometry() { + ( + geometry.dimensions, + rgba_bit_depths_from_rgb(geometry.bit_depths), + geometry.mct, + geometry.transform, + geometry.component_plans.as_slice(), + ) + } else if let Some(geometry) = tile.rgba_geometry() { + ( + geometry.dimensions, + geometry.bit_depths, + geometry.mct, + geometry.transform, + geometry.component_plans.as_slice(), + ) + } else { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA color batch received a grayscale HTJ2K tile", + }); + }; + if component_plans.len() != fmt.channels() { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA color tile component count does not match its output format", + }); + } + let span = tile.payload_records(); + if span.first_record != next_payload { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA color tile payload spans are not contiguous", + }); + } + let payload_end = span.end_record().ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA color tile payload span overflows", + })?; + let payloads = referenced + .payloads() + .get(span.first_record..payload_end) + .ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA color tile payload span is out of bounds", + })?; + let payload_start = shared_payload.len(); + let flatten_start = profile::profile_now(true); + let components = flatten_referenced_cuda_color_tile_components( + component_plans, + payloads, + input, + fmt, + (output_rect.x0, output_rect.y0), + output_dimensions, + shared_payload, + host_budget, + )?; + let block_count = components + .iter() + .map(CudaHtj2kDecodePlan::block_count) + .sum::(); + let report = CudaHtj2kProfileReport { + parse_us: 0, + plan_us: 0, + flatten_us: profile::elapsed_us(flatten_start), + total_us: profile::elapsed_us(total_start), + block_count, + classic_block_count: 0, + ht_block_count: block_count, + payload_bytes: shared_payload.len().saturating_sub(payload_start), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("prepared_color_tile_plan"); + colors.push(CudaHtj2kColorDecodePlans { + output_index: 0, + dimensions: output_dimensions, + mct_dimensions: geometry_dimensions, + bit_depths, + mct, + transform: CudaHtj2kTransform::from_native(transform), + payload: Vec::new(), + components, + report, + }); + next_payload = payload_end; + } + if next_payload != referenced.payloads().len() { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA color tile payloads contain trailing records", + }); + } + Ok(colors) +} + +#[cfg(feature = "cuda-runtime")] +#[expect( + clippy::too_many_lines, + reason = "classic tile metadata, payload spans, component owners, and exact output geometry are validated atomically" +)] +pub(in crate::decoder) fn build_cuda_classic_color_plans_from_referenced_with_profile( + input: &[u8], + referenced: &J2kReferencedClassicPlan, + fmt: PixelFormat, + device_plan: DeviceDecodePlan, + shared_payload: &mut Vec, + host_budget: &mut HostPhaseBudget, +) -> Result, Error> { + let total_start = profile::profile_now(true); + CudaHtj2kDecodePlan::validate_referenced_classic_payload_sequence( + referenced.payloads(), + referenced.ranges(), + )?; + let output_rect = referenced.output_rect(); + let output_dimensions = device_plan.output_dims(); + if (output_rect.width(), output_rect.height()) != output_dimensions { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic color tile output geometry is inconsistent", + }); + } + let mut colors = host_budget.try_vec_with_capacity(referenced.tiles().len())?; + let mut next_payload = 0usize; + for tile in referenced.tiles() { + let (geometry_dimensions, bit_depths, mct, transform, component_plans) = + if let Some(geometry) = tile.color_geometry() { + ( + geometry.dimensions, + rgba_bit_depths_from_rgb(geometry.bit_depths), + geometry.mct, + geometry.transform, + geometry.component_plans.as_slice(), + ) + } else if let Some(geometry) = tile.rgba_geometry() { + ( + geometry.dimensions, + geometry.bit_depths, + geometry.mct, + geometry.transform, + geometry.component_plans.as_slice(), + ) + } else { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic color batch received a grayscale tile", + }); + }; + if component_plans.len() != fmt.channels() { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic color tile component count does not match its output format", + }); + } + let span = tile.payload_records(); + if span.first_record != next_payload { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic color tile payload spans are not contiguous", + }); + } + let payload_end = span.end_record().ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic color tile payload span overflows", + })?; + let payloads = referenced + .payloads() + .get(span.first_record..payload_end) + .ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic color tile payload span is out of bounds", + })?; + let payload_start = shared_payload.len(); + let flatten_start = profile::profile_now(true); + let components = flatten_referenced_classic_cuda_color_tile_components( + component_plans, + payloads, + referenced.ranges(), + input, + fmt, + (output_rect.x0, output_rect.y0), + output_dimensions, + shared_payload, + host_budget, + )?; + let block_count = components + .iter() + .map(CudaHtj2kDecodePlan::block_count) + .sum::(); + let classic_block_count = components + .iter() + .map(|plan| plan.classic_code_blocks().len()) + .sum::(); + let report = CudaHtj2kProfileReport { + parse_us: 0, + plan_us: 0, + flatten_us: profile::elapsed_us(flatten_start), + total_us: profile::elapsed_us(total_start), + block_count, + classic_block_count, + ht_block_count: 0, + payload_bytes: shared_payload.len().saturating_sub(payload_start), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("prepared_classic_color_tile_plan"); + colors.push(CudaHtj2kColorDecodePlans { + output_index: 0, + dimensions: output_dimensions, + mct_dimensions: geometry_dimensions, + bit_depths, + mct, + transform: CudaHtj2kTransform::from_native(transform), + payload: Vec::new(), + components, + report, + }); + next_payload = payload_end; + } + if next_payload != referenced.payloads().len() { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic color tile payloads contain trailing records", + }); + } + Ok(colors) +} diff --git a/crates/j2k-cuda/src/decoder/plan/grayscale.rs b/crates/j2k-cuda/src/decoder/plan/grayscale.rs new file mode 100644 index 00000000..da1089d6 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/plan/grayscale.rs @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Grayscale plan construction for prepared and one-shot inputs. + +use super::{ + native_decode_error, profile, CudaHtj2kDecodePlan, CudaHtj2kDecodeProfileDetail, + CudaHtj2kProfileReport, DecodeSettings, DeviceDecodePlan, DeviceDecodeRequest, Downscale, + Error, HostPhaseBudget, J2kDecoder, J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, + NativeDecoderContext, NativeImage, PixelFormat, Rect, +}; + +pub(in crate::decoder) fn build_cuda_htj2k_grayscale_plan_from_bytes_with_profile<'a>( + input: &'a [u8], + fmt: PixelFormat, + native_context: &mut NativeDecoderContext<'a>, +) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + build_cuda_htj2k_grayscale_plan_from_bytes_for_device_plan_with_profile( + input, + fmt, + None, + DecodeSettings::default(), + native_context, + ) +} + +pub(in crate::decoder) fn build_cuda_htj2k_grayscale_plan_from_bytes_for_device_plan_with_profile< + 'a, +>( + input: &'a [u8], + fmt: PixelFormat, + device_plan: Option, + settings: DecodeSettings, + native_context: &mut NativeDecoderContext<'a>, +) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + let total_start = profile::profile_now(true); + let parse_start = profile::profile_now(true); + let target_resolution = match device_plan { + Some(plan) if plan.scale() != Downscale::None => Some( + DeviceDecodePlan::for_image( + plan.source_dims(), + DeviceDecodeRequest::Scaled { + scale: plan.scale(), + }, + )? + .output_dims(), + ), + _ => settings.target_resolution, + }; + let decode_settings = DecodeSettings { + target_resolution, + ..settings + }; + let image = NativeImage::new(input, &decode_settings).map_err(native_decode_error)?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let selected_rect = device_plan.and_then(|plan| { + let full = Rect::full(plan.source_dims()); + if plan.source_rect() == full { + None + } else if plan.scale() == Downscale::None { + Some(plan.source_rect()) + } else { + Some(plan.output_rect()) + } + }); + let native_plan = match selected_rect { + Some(rect) => image + .build_direct_grayscale_plan_region_with_context( + native_context, + (rect.x, rect.y, rect.w, rect.h), + ) + .map_err(native_decode_error)?, + None => image + .build_direct_grayscale_plan_with_context(native_context) + .map_err(native_decode_error)?, + }; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let cuda_plan = match selected_rect { + Some(rect) => CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( + &native_plan, + fmt, + (rect.x, rect.y), + (rect.w, rect.h), + )?, + None => CudaHtj2kDecodePlan::from_grayscale_direct_plan(&native_plan, fmt, (0, 0))?, + }; + let flatten_us = profile::elapsed_us(flatten_start); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.block_count(), + classic_block_count: cuda_plan.classic_code_blocks().len(), + ht_block_count: cuda_plan.code_blocks().len(), + payload_bytes: cuda_plan.payload().len(), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + Ok((cuda_plan, report)) +} + +#[cfg(feature = "cuda-runtime")] +pub(in crate::decoder) fn build_cuda_htj2k_grayscale_plans_from_referenced_with_profile( + input: &[u8], + referenced: &J2kReferencedHtj2kPlan, + fmt: PixelFormat, + device_plan: DeviceDecodePlan, + shared_payload: &mut Vec, + host_budget: &mut HostPhaseBudget, +) -> Result, Error> { + let total_start = profile::profile_now(true); + let output_rect = referenced.output_rect(); + let output_dimensions = device_plan.output_dims(); + if (output_rect.width(), output_rect.height()) != output_dimensions { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA grayscale tile plan output geometry is inconsistent", + }); + } + let mut plans = host_budget.try_vec_with_capacity(referenced.tiles().len())?; + let mut next_payload = 0usize; + for tile in referenced.tiles() { + let geometry = tile + .grayscale_geometry() + .ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA grayscale batch received a color HTJ2K tile", + })?; + let span = tile.payload_records(); + if span.first_record != next_payload { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA grayscale tile payload spans are not contiguous", + }); + } + let payload_end = span.end_record().ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA grayscale tile payload span overflows", + })?; + let payloads = referenced + .payloads() + .get(span.first_record..payload_end) + .ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA grayscale tile payload span is out of bounds", + })?; + let payload_start = shared_payload.len(); + let flatten_start = profile::profile_now(true); + let cuda_plan = CudaHtj2kDecodePlan::from_referenced_tile_grayscale_plan_into_shared( + geometry, + payloads, + input, + fmt, + (output_rect.x0, output_rect.y0), + output_dimensions, + shared_payload, + host_budget, + )?; + let report = CudaHtj2kProfileReport { + parse_us: 0, + plan_us: 0, + flatten_us: profile::elapsed_us(flatten_start), + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.block_count(), + classic_block_count: 0, + ht_block_count: cuda_plan.code_blocks().len(), + payload_bytes: shared_payload.len().saturating_sub(payload_start), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("prepared_tile_plan"); + plans.push((cuda_plan, report)); + next_payload = payload_end; + } + if next_payload != referenced.payloads().len() { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA grayscale tile payloads contain trailing records", + }); + } + Ok(plans) +} + +#[cfg(feature = "cuda-runtime")] +pub(in crate::decoder) fn build_cuda_classic_grayscale_plans_from_referenced_with_profile( + input: &[u8], + referenced: &J2kReferencedClassicPlan, + fmt: PixelFormat, + device_plan: DeviceDecodePlan, + shared_payload: &mut Vec, + host_budget: &mut HostPhaseBudget, +) -> Result, Error> { + let total_start = profile::profile_now(true); + CudaHtj2kDecodePlan::validate_referenced_classic_payload_sequence( + referenced.payloads(), + referenced.ranges(), + )?; + let output_rect = referenced.output_rect(); + let output_dimensions = device_plan.output_dims(); + if (output_rect.width(), output_rect.height()) != output_dimensions { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic grayscale tile output geometry is inconsistent", + }); + } + let mut plans = host_budget.try_vec_with_capacity(referenced.tiles().len())?; + let mut next_payload = 0usize; + for tile in referenced.tiles() { + let geometry = tile + .grayscale_geometry() + .ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic grayscale batch received a color tile", + })?; + let span = tile.payload_records(); + if span.first_record != next_payload { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic grayscale tile payload spans are not contiguous", + }); + } + let payload_end = span.end_record().ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic grayscale tile payload span overflows", + })?; + let payloads = referenced + .payloads() + .get(span.first_record..payload_end) + .ok_or(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic grayscale tile payload span is out of bounds", + })?; + let payload_start = shared_payload.len(); + let flatten_start = profile::profile_now(true); + let cuda_plan = + CudaHtj2kDecodePlan::from_referenced_classic_tile_grayscale_plan_into_shared( + geometry, + payloads, + referenced.ranges(), + input, + fmt, + (output_rect.x0, output_rect.y0), + output_dimensions, + shared_payload, + host_budget, + )?; + let report = CudaHtj2kProfileReport { + parse_us: 0, + plan_us: 0, + flatten_us: profile::elapsed_us(flatten_start), + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.block_count(), + classic_block_count: cuda_plan.classic_code_blocks().len(), + ht_block_count: 0, + payload_bytes: shared_payload.len().saturating_sub(payload_start), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("prepared_classic_tile_plan"); + plans.push((cuda_plan, report)); + next_payload = payload_end; + } + if next_payload != referenced.payloads().len() { + return Err(Error::UnsupportedCudaRequest { + reason: "prepared CUDA classic grayscale tile payloads contain trailing records", + }); + } + Ok(plans) +} + +impl J2kDecoder<'_> { + /// Build a flat CUDA HTJ2K grayscale decode plan and return stage timings. + pub(crate) fn build_cuda_htj2k_grayscale_plan_with_profile( + &mut self, + fmt: PixelFormat, + ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + let mut native_context = NativeDecoderContext::default(); + build_cuda_htj2k_grayscale_plan_from_bytes_with_profile( + self.bytes, + fmt, + &mut native_context, + ) + } + + /// Build a flat CUDA HTJ2K grayscale region decode plan and return stage timings. + pub(crate) fn build_cuda_htj2k_grayscale_region_plan_with_profile( + &mut self, + fmt: PixelFormat, + roi: Rect, + ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new(self.bytes, &DecodeSettings::default()) + .map_err(native_decode_error)?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_grayscale_plan_region_with_context( + &mut native_context, + (roi.x, roi.y, roi.w, roi.h), + ) + .map_err(native_decode_error)?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( + &native_plan, + fmt, + (roi.x, roi.y), + (roi.w, roi.h), + )?; + let flatten_us = profile::elapsed_us(flatten_start); + + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.block_count(), + classic_block_count: cuda_plan.classic_code_blocks().len(), + ht_block_count: cuda_plan.code_blocks().len(), + payload_bytes: cuda_plan.payload().len(), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + Ok((cuda_plan, report)) + } + + /// Build a flat reduced-resolution CUDA HTJ2K grayscale decode plan and + /// return stage timings. + pub(crate) fn build_cuda_htj2k_grayscale_scaled_plan_with_profile( + &mut self, + fmt: PixelFormat, + output_dimensions: (u32, u32), + ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new( + self.bytes, + &DecodeSettings { + target_resolution: Some(output_dimensions), + ..DecodeSettings::default() + }, + ) + .map_err(native_decode_error)?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_grayscale_plan_with_context(&mut native_context) + .map_err(native_decode_error)?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan(&native_plan, fmt, (0, 0))?; + let flatten_us = profile::elapsed_us(flatten_start); + + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.block_count(), + classic_block_count: cuda_plan.classic_code_blocks().len(), + ht_block_count: cuda_plan.code_blocks().len(), + payload_bytes: cuda_plan.payload().len(), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + Ok((cuda_plan, report)) + } + + /// Build a flat reduced-resolution CUDA HTJ2K grayscale region decode + /// plan and return stage timings. + pub(crate) fn build_cuda_htj2k_grayscale_region_scaled_plan_with_profile( + &mut self, + fmt: PixelFormat, + scaled_roi: Rect, + output_dimensions: (u32, u32), + ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new( + self.bytes, + &DecodeSettings { + target_resolution: Some(output_dimensions), + ..DecodeSettings::default() + }, + ) + .map_err(native_decode_error)?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_grayscale_plan_region_with_context( + &mut native_context, + (scaled_roi.x, scaled_roi.y, scaled_roi.w, scaled_roi.h), + ) + .map_err(native_decode_error)?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( + &native_plan, + fmt, + (scaled_roi.x, scaled_roi.y), + (scaled_roi.w, scaled_roi.h), + )?; + let flatten_us = profile::elapsed_us(flatten_start); + + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.block_count(), + classic_block_count: cuda_plan.classic_code_blocks().len(), + ht_block_count: cuda_plan.code_blocks().len(), + payload_bytes: cuda_plan.payload().len(), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + Ok((cuda_plan, report)) + } +} diff --git a/crates/j2k-cuda/src/decoder/resident.rs b/crates/j2k-cuda/src/decoder/resident.rs index df9f4b76..a4bc99d9 100644 --- a/crates/j2k-cuda/src/decoder/resident.rs +++ b/crates/j2k-cuda/src/decoder/resident.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 mod buffer_access; +#[cfg(feature = "cuda-runtime")] +mod chunked_cleanup; mod cleanup_dequant; mod component; mod error; @@ -9,9 +11,18 @@ mod routing; mod surface; pub(super) use self::buffer_access::pooled_cuda_buffer; +#[cfg(feature = "cuda-runtime")] +pub(super) use self::chunked_cleanup::{ + enqueue_chunked_htj2k_cleanup_dequant, ChunkedHtj2kCleanup, +}; pub(super) use self::cleanup_dequant::run_component_cleanup_dequant_batches; #[cfg(test)] pub(super) use self::cleanup_dequant::split_htj2k_subband_decode_dispatches; +#[cfg(feature = "cuda-runtime")] +pub(super) use self::cleanup_dequant::{ + enqueue_component_classic_batches, enqueue_component_cleanup_dequant_batches, + QueuedComponentClassicDecode, +}; #[cfg(test)] pub(super) use self::cleanup_dequant::{ htj2k_batched_cleanup_dequant_dispatches, htj2k_batched_cleanup_dispatches, diff --git a/crates/j2k-cuda/src/decoder/resident/chunked_cleanup.rs b/crates/j2k-cuda/src/decoder/resident/chunked_cleanup.rs new file mode 100644 index 00000000..6de62bb1 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/resident/chunked_cleanup.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +mod enqueue; +mod planning; + +#[cfg(test)] +mod tests; + +pub(in crate::decoder) use enqueue::enqueue_chunked_htj2k_cleanup_dequant; + +use j2k_cuda_runtime::{CudaHtj2kDecodeResources, CudaQueuedHtj2kCleanupGroup}; + +use super::super::{combine_cuda_cleanup_errors, cuda_error, Error}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Htj2kChunkJobIdentity { + original_job_index: usize, + source_index: usize, +} + +impl Htj2kChunkJobIdentity { + const fn new(original_job_index: usize, source_index: usize) -> Self { + Self { + original_job_index, + source_index, + } + } +} + +/// Pending pass-homogeneous HTJ2K chunks retained through final-store completion. +pub(in crate::decoder) struct ChunkedHtj2kCleanup { + pub(super) group: Option, + pub(super) resources: Vec, + pub(super) identities: Vec, + pub(super) chunk_count: usize, + pub(super) dequant_chunk_count: usize, +} + +impl ChunkedHtj2kCleanup { + pub(in crate::decoder) fn has_status_readback(&self) -> bool { + self.group + .as_ref() + .is_some_and(|group| group.status_count() != 0) + } + + /// Number of bounded compressed/descriptor arenas submitted for this group. + #[cfg(test)] + pub(in crate::decoder) fn chunk_count(&self) -> usize { + self.chunk_count + } + + pub(in crate::decoder) fn finish(mut self) -> Result<(), Error> { + self.finish_remaining() + } + + fn finish_remaining(&mut self) -> Result<(), Error> { + let Some(group) = self.group.take() else { + self.resources.clear(); + return Ok(()); + }; + match group.finish() { + Ok(_) => { + self.resources.clear(); + Ok(()) + } + Err(error) => { + if error.completion_is_uncertain() { + for resources in self.resources.drain(..) { + core::mem::forget(resources); + } + } else { + self.resources.clear(); + } + Err(map_chunk_status_error(error, &self.identities)) + } + } + } + + pub(super) fn finish_after_error(mut self, primary: Error) -> Error { + match self.finish_remaining() { + Ok(()) => primary, + Err(cleanup) => combine_cuda_cleanup_errors(primary, cleanup), + } + } +} + +impl Drop for ChunkedHtj2kCleanup { + fn drop(&mut self) { + let _ = self.finish_remaining(); + } +} + +fn map_chunk_status_error( + error: j2k_cuda_runtime::CudaError, + identities: &[Htj2kChunkJobIdentity], +) -> Error { + let Some(job_index) = error.kernel_job_index() else { + return cuda_error(error); + }; + let Some(identity) = identities.get(job_index) else { + return cuda_error(error); + }; + Error::CudaTier1JobFailed { + source_index: identity.source_index, + original_job_index: identity.original_job_index, + source: error, + } +} diff --git a/crates/j2k-cuda/src/decoder/resident/chunked_cleanup/enqueue.rs b/crates/j2k-cuda/src/decoder/resident/chunked_cleanup/enqueue.rs new file mode 100644 index 00000000..a561e4d9 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/resident/chunked_cleanup/enqueue.rs @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_core::{plan_ht_gpu_job_chunks, HtGpuJobChunkLimits, HtGpuJobPassBucket}; +use j2k_cuda_runtime::{ + CudaHtj2kCleanupTarget, CudaHtj2kDecodeResources, CudaQueuedHtj2kCleanup, + CudaQueuedHtj2kCleanupGroup, +}; + +use super::super::super::{ + cuda_error, CudaBufferPool, CudaComponentDecodeWork, CudaContext, + CudaHtj2kDecodeTableResources, Error, CUDA_HTJ2K_PLAN_INVARIANT_FAILED, +}; +use super::super::pooled_cuda_buffer; +use super::planning::{ + chunk_requests, flatten_job_locations, job_at, pending_at, + sort_selected_jobs_for_coalesced_targets, Htj2kJobLocation, SelectedHtj2kChunkJob, +}; +use super::{ChunkedHtj2kCleanup, Htj2kChunkJobIdentity}; +use crate::allocation::HostPhaseBudget; + +struct SubmittedHtj2kChunk { + cleanup: CudaQueuedHtj2kCleanup, + resources: CudaHtj2kDecodeResources, + identities: Vec, +} + +/// Flatten, pass-bucket, and asynchronously enqueue bounded HTJ2K arenas. +#[expect( + clippy::too_many_arguments, + reason = "the helper keeps explicit ownership of CUDA context, tables, group arena, outputs, and limits" +)] +pub(in crate::decoder) fn enqueue_chunked_htj2k_cleanup_dequant( + context: &CudaContext, + tables: Option<&CudaHtj2kDecodeTableResources>, + shared_payload: &[u8], + component_work: &mut [CudaComponentDecodeWork], + component_source_indices: &[usize], + pool: &CudaBufferPool, + limits: HtGpuJobChunkLimits, + live_host_bytes: usize, +) -> Result { + let locations = flatten_job_locations(component_work, component_source_indices)?; + if locations.is_empty() { + return Ok(ChunkedHtj2kCleanup { + group: None, + resources: Vec::new(), + identities: Vec::new(), + chunk_count: 0, + dequant_chunk_count: 0, + }); + } + let tables = tables.ok_or(Error::UnsupportedCudaRequest { + reason: "CUDA HTJ2K chunks require resident cleanup lookup tables", + })?; + let requests = chunk_requests(component_work, &locations)?; + let plan = plan_ht_gpu_job_chunks(&requests, limits)?; + let status_group = + CudaQueuedHtj2kCleanupGroup::new(context, pool, locations.len()).map_err(cuda_error)?; + let mut owner = ChunkedHtj2kCleanup { + group: Some(status_group), + resources: Vec::new(), + identities: Vec::new(), + chunk_count: 0, + dequant_chunk_count: 0, + }; + owner + .resources + .try_reserve_exact(plan.chunks().len()) + .map_err(|_| Error::HostAllocationFailed { + bytes: plan + .chunks() + .len() + .saturating_mul(core::mem::size_of::()), + what: "CUDA retained HTJ2K chunks", + })?; + owner + .identities + .try_reserve_exact(locations.len()) + .map_err(|_| Error::HostAllocationFailed { + bytes: locations + .len() + .saturating_mul(core::mem::size_of::()), + what: "CUDA retained HTJ2K status identities", + })?; + + for (chunk_index, chunk) in plan.chunks().iter().copied().enumerate() { + let entries = plan + .chunk_entries(chunk_index) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + })?; + let submission = enqueue_one_chunk( + context, + tables, + shared_payload, + component_work, + &locations, + entries, + chunk.bucket(), + chunk.payload_bytes(), + pool, + live_host_bytes, + owner.group.as_ref().ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + })?, + owner.identities.len(), + ); + match submission { + Ok(submitted) => { + let SubmittedHtj2kChunk { + cleanup, + resources, + identities, + } = submitted; + if let Err(error) = owner + .group + .as_mut() + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + })? + .retain(cleanup) + .map_err(cuda_error) + { + return Err(owner.finish_after_error(error)); + } + owner.resources.push(resources); + owner.identities.extend_from_slice(&identities); + owner.chunk_count = owner.chunk_count.saturating_add(1); + if chunk.bucket() != HtGpuJobPassBucket::CleanupOnly { + owner.dequant_chunk_count = owner.dequant_chunk_count.saturating_add(1); + } + } + Err(error) => return Err(owner.finish_after_error(error)), + } + } + account_chunk_dispatches(component_work, &locations, &owner); + for work in component_work { + work.pending_dequant_bands.clear(); + } + Ok(owner) +} + +#[expect( + clippy::too_many_arguments, + reason = "one chunk materializes an explicitly bounded arena against its retained output owners" +)] +#[expect( + clippy::too_many_lines, + reason = "one chunk keeps stable job identity, coalesced destination ownership, arena materialization, and queued completion atomic" +)] +fn enqueue_one_chunk( + context: &CudaContext, + tables: &CudaHtj2kDecodeTableResources, + shared_payload: &[u8], + component_work: &[CudaComponentDecodeWork], + locations: &[Htj2kJobLocation], + entries: &[j2k_core::HtGpuJobChunkEntry], + bucket: HtGpuJobPassBucket, + payload_bytes: usize, + pool: &CudaBufferPool, + live_host_bytes: usize, + status_group: &CudaQueuedHtj2kCleanupGroup, + status_offset: usize, +) -> Result { + let mut budget = HostPhaseBudget::with_live_bytes( + "CUDA bounded HTJ2K chunk materialization", + live_host_bytes, + )?; + let mut selected = budget.try_vec_with_capacity(entries.len())?; + for entry in entries { + let location = + *locations + .get(entry.original_job_index()) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + })?; + if location.source != entry.source_index() { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + }); + } + selected.push(SelectedHtj2kChunkJob { + location, + original_job_index: entry.original_job_index(), + source_index: entry.source_index(), + }); + } + sort_selected_jobs_for_coalesced_targets(&mut selected); + + let mut payload = budget.try_vec_with_capacity(payload_bytes)?; + let mut jobs = budget.try_vec_with_capacity(entries.len())?; + let mut identities = budget.try_vec_with_capacity(entries.len())?; + for selected in &selected { + let location = selected.location; + let mut job = *job_at(component_work, location)?; + let start = + usize::try_from(job.payload_offset).map_err(|_| Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + })?; + let end = + start + .checked_add(job.payload_len as usize) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + })?; + let bytes = shared_payload + .get(start..end) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + })?; + job.payload_offset = + u64::try_from(payload.len()).map_err(|_| Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + })?; + payload.extend_from_slice(bytes); + jobs.push(job); + identities.push(Htj2kChunkJobIdentity::new( + selected.original_job_index, + selected.source_index, + )); + } + if payload.len() != payload_bytes { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + }); + } + let resources = context + .upload_htj2k_decode_resources_with_tables_and_pool(&payload, tables, pool) + .map_err(cuda_error)?; + let mut targets = budget.try_vec_with_capacity(entries.len())?; + let mut start = 0usize; + while start < selected.len() { + let location = selected[start].location; + let mut end = start + 1; + while end < selected.len() + && selected[end].location.work == location.work + && selected[end].location.pending == location.pending + { + end += 1; + } + let pending = pending_at(component_work, location)?; + targets.push(CudaHtj2kCleanupTarget { + coefficients: pooled_cuda_buffer( + &component_work[location.work].bands[pending.band_index].buffer, + )?, + jobs: &jobs[start..end], + output_words: pending.output_words, + }); + start = end; + } + // SAFETY: the returned guard retains uploaded descriptor owners; the + // group owns the shared status allocation, `SubmittedHtj2kChunk` retains + // payload/tables, and component_work owns every disjoint destination. + let cleanup = unsafe { + match bucket { + HtGpuJobPassBucket::CleanupOnly => context + .decode_htj2k_codeblocks_cleanup_dequantize_multi_enqueue_into_status_group( + &resources, + &targets, + pool, + budget.live_bytes(), + status_group, + status_offset, + ), + HtGpuJobPassBucket::SigProp | HtGpuJobPassBucket::MagRef => context + .decode_htj2k_codeblocks_cleanup_multi_enqueue_into_status_group( + &resources, + &targets, + pool, + budget.live_bytes(), + status_group, + status_offset, + ), + } + } + .map_err(cuda_error)?; + if bucket != HtGpuJobPassBucket::CleanupOnly { + // SAFETY: the cleanup guard retains its device descriptors, and the + // same default stream orders this dequantization before later IDWT. + unsafe { context.j2k_dequantize_queued_htj2k_cleanup_enqueue(&cleanup) } + .map_err(cuda_error)?; + } + Ok(SubmittedHtj2kChunk { + cleanup, + resources, + identities, + }) +} + +fn account_chunk_dispatches( + component_work: &mut [CudaComponentDecodeWork], + locations: &[Htj2kJobLocation], + owner: &ChunkedHtj2kCleanup, +) { + let Some(first) = locations.first() else { + return; + }; + let cleanup_dispatches = owner.chunk_count; + let dequant_dispatches = owner.dequant_chunk_count; + let Some(accounting) = component_work.get_mut(first.work) else { + return; + }; + accounting.dispatches = accounting + .dispatches + .saturating_add(cleanup_dispatches) + .saturating_add(dequant_dispatches); + accounting.decode_dispatches = accounting + .decode_dispatches + .saturating_add(cleanup_dispatches); + accounting.timings.ht_dispatch_count = accounting + .timings + .ht_dispatch_count + .saturating_add(cleanup_dispatches); + accounting.timings.dequant_dispatch_count = accounting + .timings + .dequant_dispatch_count + .saturating_add(dequant_dispatches); +} diff --git a/crates/j2k-cuda/src/decoder/resident/chunked_cleanup/planning.rs b/crates/j2k-cuda/src/decoder/resident/chunked_cleanup/planning.rs new file mode 100644 index 00000000..1debcdde --- /dev/null +++ b/crates/j2k-cuda/src/decoder/resident/chunked_cleanup/planning.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_core::HtGpuJobChunkRequest; +use j2k_cuda_runtime::{htj2k_cleanup_multi_descriptor_bytes, CudaHtj2kCodeBlockJob}; + +use super::super::super::{CudaComponentDecodeWork, Error, CUDA_HTJ2K_PLAN_INVARIANT_FAILED}; +use crate::allocation::HostPhaseBudget; + +#[derive(Clone, Copy)] +pub(super) struct Htj2kJobLocation { + pub(super) work: usize, + pub(super) pending: usize, + pub(super) job: usize, + pub(super) source: usize, +} + +#[derive(Clone, Copy)] +pub(super) struct SelectedHtj2kChunkJob { + pub(super) location: Htj2kJobLocation, + pub(super) original_job_index: usize, + pub(super) source_index: usize, +} + +pub(super) fn sort_selected_jobs_for_coalesced_targets(selected: &mut [SelectedHtj2kChunkJob]) { + selected.sort_unstable_by_key(|selected| { + ( + selected.location.work, + selected.location.pending, + selected.location.job, + ) + }); +} + +pub(super) fn flatten_job_locations( + component_work: &[CudaComponentDecodeWork], + component_source_indices: &[usize], +) -> Result, Error> { + if component_work.len() != component_source_indices.len() { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + }); + } + let job_count = component_work + .iter() + .flat_map(|work| work.pending_dequant_bands.iter()) + .try_fold(0usize, |count, pending| { + count.checked_add(pending.jobs.len()) + }) + .ok_or(Error::HostAllocationFailed { + bytes: usize::MAX, + what: "CUDA HTJ2K flattened job locations", + })?; + let mut budget = HostPhaseBudget::new("CUDA HTJ2K flattened job locations"); + let mut locations = budget.try_vec_with_capacity(job_count)?; + for (work_index, (work, source_index)) in component_work + .iter() + .zip(component_source_indices.iter().copied()) + .enumerate() + { + for (pending_index, pending) in work.pending_dequant_bands.iter().enumerate() { + for job_index in 0..pending.jobs.len() { + locations.push(Htj2kJobLocation { + work: work_index, + pending: pending_index, + job: job_index, + source: source_index, + }); + } + } + } + Ok(locations) +} + +pub(super) fn chunk_requests( + component_work: &[CudaComponentDecodeWork], + locations: &[Htj2kJobLocation], +) -> Result, Error> { + let mut budget = HostPhaseBudget::new("CUDA HTJ2K shared chunk requests"); + let mut requests = budget.try_vec_with_capacity(locations.len())?; + let descriptor_bytes = htj2k_cleanup_multi_descriptor_bytes(); + for location in locations { + let job = job_at(component_work, *location)?; + requests.push(HtGpuJobChunkRequest::new( + location.source, + job.number_of_coding_passes, + job.payload_len as usize, + descriptor_bytes, + )); + } + Ok(requests) +} + +pub(super) fn pending_at( + component_work: &[CudaComponentDecodeWork], + location: Htj2kJobLocation, +) -> Result<&super::super::super::CudaPendingDequantBand, Error> { + component_work + .get(location.work) + .and_then(|work| work.pending_dequant_bands.get(location.pending)) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + }) +} + +pub(super) fn job_at( + component_work: &[CudaComponentDecodeWork], + location: Htj2kJobLocation, +) -> Result<&CudaHtj2kCodeBlockJob, Error> { + pending_at(component_work, location)? + .jobs + .get(location.job) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + }) +} diff --git a/crates/j2k-cuda/src/decoder/resident/chunked_cleanup/tests.rs b/crates/j2k-cuda/src/decoder/resident/chunked_cleanup/tests.rs new file mode 100644 index 00000000..74f2bcfe --- /dev/null +++ b/crates/j2k-cuda/src/decoder/resident/chunked_cleanup/tests.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::planning::{ + sort_selected_jobs_for_coalesced_targets, Htj2kJobLocation, SelectedHtj2kChunkJob, +}; +use super::{map_chunk_status_error, Htj2kChunkJobIdentity}; +use crate::Error; +use j2k_cuda_runtime::CudaError; + +#[test] +fn chunk_local_kernel_failure_maps_to_original_job_and_source() { + let identities = [ + Htj2kChunkJobIdentity::new(11, 4), + Htj2kChunkJobIdentity::new(29, 8), + ]; + let mapped = map_chunk_status_error( + CudaError::KernelJobStatus { + kernel: "injected", + job_index: 1, + code: 7, + detail: 13, + }, + &identities, + ); + assert!(matches!( + mapped, + Error::CudaTier1JobFailed { + source_index: 8, + original_job_index: 29, + .. + } + )); +} + +#[test] +fn chunk_jobs_for_one_coefficient_band_are_adjacent_for_one_runtime_target() { + let mut selected = [ + selected_job(0, 2, 1, 7), + selected_job(0, 1, 4, 3), + selected_job(0, 2, 0, 6), + selected_job(1, 0, 0, 8), + ]; + + sort_selected_jobs_for_coalesced_targets(&mut selected); + + assert_eq!( + selected.map(|selected| ( + selected.location.work, + selected.location.pending, + selected.location.job, + )), + [(0, 1, 4), (0, 2, 0), (0, 2, 1), (1, 0, 0)] + ); +} + +fn selected_job( + work_index: usize, + pending_index: usize, + job_index: usize, + original_job_index: usize, +) -> SelectedHtj2kChunkJob { + SelectedHtj2kChunkJob { + location: Htj2kJobLocation { + work: work_index, + pending: pending_index, + job: job_index, + source: 5, + }, + original_job_index, + source_index: 5, + } +} diff --git a/crates/j2k-cuda/src/decoder/resident/cleanup_dequant.rs b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant.rs index 4d9973e3..05e13ec7 100644 --- a/crates/j2k-cuda/src/decoder/resident/cleanup_dequant.rs +++ b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant.rs @@ -11,7 +11,13 @@ use crate::allocation::HostPhaseBudget; #[cfg(feature = "cuda-runtime")] mod classic; #[cfg(feature = "cuda-runtime")] -use classic::run_component_classic_batches; +pub(in crate::decoder) use classic::{ + enqueue_component_classic_batches, run_component_classic_batches, QueuedComponentClassicDecode, +}; +#[cfg(feature = "cuda-runtime")] +mod enqueue; +#[cfg(feature = "cuda-runtime")] +pub(in crate::decoder) use enqueue::enqueue_component_cleanup_dequant_batches; #[cfg(test)] pub(in crate::decoder) fn split_htj2k_subband_decode_dispatches( @@ -234,6 +240,12 @@ pub(in crate::decoder) fn run_component_cleanup_dequant_batches( || context.j2k_dequantize_queued_htj2k_cleanup_with_pool(queued), ) } else { + if !collect_stage_timings { + return Err(Error::UnsupportedCudaRequest { + reason: + "normal CUDA HTJ2K refinement requires retained queued cleanup metadata", + }); + } let mut dequant_budget = HostPhaseBudget::with_live_bytes( "j2k CUDA dequantization target phase", live_host_bytes, diff --git a/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic.rs b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic.rs index a4d3daba..310438e4 100644 --- a/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic.rs +++ b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic.rs @@ -7,7 +7,12 @@ use super::super::super::{ use super::super::buffer_access::pooled_cuda_buffer; use crate::allocation::HostPhaseBudget; -pub(super) fn run_component_classic_batches( +mod queued; +pub(in crate::decoder) use queued::{ + enqueue_component_classic_batches, QueuedComponentClassicDecode, +}; + +pub(in crate::decoder) fn run_component_classic_batches( context: &j2k_cuda_runtime::CudaContext, decode_resources: &CudaHtj2kDecodeResources, component_work: &mut [CudaComponentDecodeWork], diff --git a/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued.rs b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued.rs new file mode 100644 index 00000000..91c0898f --- /dev/null +++ b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::super::super::{ + cuda_error, CudaBufferPool, CudaClassicDecodeTarget, CudaComponentDecodeWork, + CudaHtj2kDecodeResources, Error, +}; +use super::super::super::buffer_access::pooled_cuda_buffer; +use crate::allocation::HostPhaseBudget; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ClassicJobIdentity { + source_index: usize, + original_job_index: usize, +} + +pub(in crate::decoder) struct QueuedComponentClassicDecode { + queued: Option, + identities: Vec, +} + +impl QueuedComponentClassicDecode { + pub(in crate::decoder) fn has_status_readback(&self) -> bool { + self.queued + .as_ref() + .is_some_and(|queued| queued.status_count() != 0) + } + + pub(in crate::decoder) fn finish(mut self) -> Result<(), Error> { + let Some(queued) = self.queued.take() else { + return Ok(()); + }; + queued + .finish() + .map(|_| ()) + .map_err(|error| map_classic_status_error(error, &self.identities)) + } +} + +impl Drop for QueuedComponentClassicDecode { + fn drop(&mut self) { + if let Some(queued) = self.queued.take() { + let _ = queued.finish(); + } + } +} + +pub(in crate::decoder) fn enqueue_component_classic_batches( + context: &j2k_cuda_runtime::CudaContext, + decode_resources: &CudaHtj2kDecodeResources, + table_resources: &j2k_cuda_runtime::CudaClassicDecodeTableResources, + component_work: &mut [CudaComponentDecodeWork], + component_source_indices: &[usize], + pool: &CudaBufferPool, + live_host_bytes: usize, +) -> Result, Error> { + if component_work.len() != component_source_indices.len() { + return Err(Error::UnsupportedCudaRequest { + reason: "CUDA classic source identity count does not match component work", + }); + } + let pending_count = component_work + .iter() + .map(|work| work.pending_classic_bands.len()) + .sum::(); + if pending_count == 0 { + return Ok(None); + } + let job_count = component_work + .iter() + .flat_map(|work| &work.pending_classic_bands) + .try_fold(0usize, |count, pending| { + count.checked_add(pending.jobs.len()) + }) + .ok_or(Error::HostAllocationFailed { + bytes: usize::MAX, + what: "CUDA classic job source identities", + })?; + let mut budget = + HostPhaseBudget::with_live_bytes("j2k CUDA queued classic targets", live_host_bytes)?; + let mut targets = budget.try_vec_with_capacity(pending_count)?; + let mut identities = budget.try_vec_with_capacity(job_count)?; + for (work, source_index) in component_work + .iter() + .zip(component_source_indices.iter().copied()) + { + for pending in &work.pending_classic_bands { + targets.push(CudaClassicDecodeTarget { + coefficients: pooled_cuda_buffer(&work.bands[pending.band_index].buffer)?, + jobs: &pending.jobs, + segments: &pending.segments, + output_words: pending.output_words, + }); + for _ in &pending.jobs { + identities.push(ClassicJobIdentity { + source_index, + original_job_index: identities.len(), + }); + } + } + } + // SAFETY: component work retains disjoint coefficient targets; the + // high-level pending owner retains payload/tables through this guard. + let queued = unsafe { + context.decode_classic_codeblocks_multi_enqueue_with_resources_and_pool( + decode_resources, + table_resources, + &targets, + pool, + budget.live_bytes(), + ) + } + .map_err(cuda_error)?; + let stats = queued.execution(); + if let Some(accounting) = component_work + .iter_mut() + .find(|work| !work.pending_classic_bands.is_empty()) + { + accounting.timings.classic_dispatch_count = accounting + .timings + .classic_dispatch_count + .saturating_add(stats.kernel_dispatches()); + accounting.dispatches = accounting + .dispatches + .saturating_add(stats.kernel_dispatches()); + accounting.decode_dispatches = accounting + .decode_dispatches + .saturating_add(stats.decode_kernel_dispatches()); + } + for work in component_work { + work.pending_classic_bands.clear(); + } + Ok(Some(QueuedComponentClassicDecode { + queued: Some(queued), + identities, + })) +} + +fn map_classic_status_error( + error: j2k_cuda_runtime::CudaError, + identities: &[ClassicJobIdentity], +) -> Error { + let Some(job_index) = error.kernel_job_index() else { + return cuda_error(error); + }; + let Some(identity) = identities.get(job_index) else { + return cuda_error(error); + }; + Error::CudaTier1JobFailed { + source_index: identity.source_index, + original_job_index: identity.original_job_index, + source: error, + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued/tests.rs b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued/tests.rs new file mode 100644 index 00000000..b72ad67a --- /dev/null +++ b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued/tests.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{map_classic_status_error, ClassicJobIdentity}; + +#[test] +fn classic_kernel_failure_maps_to_responsible_source_index() { + let identities = [ + ClassicJobIdentity { + source_index: 2, + original_job_index: 0, + }, + ClassicJobIdentity { + source_index: 6, + original_job_index: 1, + }, + ]; + let mapped = map_classic_status_error( + j2k_cuda_runtime::CudaError::KernelJobStatus { + kernel: "injected", + job_index: 1, + code: 5, + detail: 13, + }, + &identities, + ); + assert!(matches!( + mapped, + crate::Error::CudaTier1JobFailed { + source_index: 6, + original_job_index: 1, + .. + } + )); +} diff --git a/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/enqueue.rs b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/enqueue.rs new file mode 100644 index 00000000..fb52e3e2 --- /dev/null +++ b/crates/j2k-cuda/src/decoder/resident/cleanup_dequant/enqueue.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + cuda_error, pooled_cuda_buffer, run_component_cleanup_dequant_batches, CudaBufferPool, + CudaComponentDecodeWork, CudaHtj2kCleanupTarget, CudaHtj2kDecodeResources, + CudaQueuedHtj2kCleanup, Error, HostPhaseBudget, CUDA_HTJ2K_KERNELS_NOT_READY, +}; + +/// Enqueue the HT cleanup/dequantization portion of a batch without a host +/// completion boundary. Classic blocks retain their existing completed path. +pub(in crate::decoder) fn enqueue_component_cleanup_dequant_batches( + context: &j2k_cuda_runtime::CudaContext, + decode_resources: &CudaHtj2kDecodeResources, + component_work: &mut [CudaComponentDecodeWork], + pool: &CudaBufferPool, + live_host_bytes: usize, +) -> Result, Error> { + if component_work + .iter() + .any(|work| !work.pending_classic_bands.is_empty()) + { + run_component_cleanup_dequant_batches( + context, + decode_resources, + component_work, + pool, + false, + live_host_bytes, + )?; + return Ok(None); + } + + let pending_count = component_work + .iter() + .map(|work| work.pending_dequant_bands.len()) + .sum::(); + if pending_count == 0 { + return Ok(None); + } + let accounting_index = component_work + .iter() + .position(|work| !work.pending_dequant_bands.is_empty()) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + let has_refinement = component_work.iter().any(|work| { + work.pending_dequant_bands.iter().any(|pending| { + pending + .jobs + .iter() + .any(|job| job.refinement_length > 0 || job.number_of_coding_passes > 1) + }) + }); + let mut budget = + HostPhaseBudget::with_live_bytes("j2k CUDA queued cleanup targets", live_host_bytes)?; + let mut targets = budget.try_vec_with_capacity(pending_count)?; + for work in component_work.iter() { + for pending in &work.pending_dequant_bands { + targets.push(CudaHtj2kCleanupTarget { + coefficients: pooled_cuda_buffer(&work.bands[pending.band_index].buffer)?, + jobs: &pending.jobs, + output_words: pending.output_words, + }); + } + } + + // SAFETY: component_work retains every coefficient target, decode_resources + // retains payload/tables, and the returned typed guard is kept until the + // final same-stream store has established completion. + let queued = unsafe { + if has_refinement { + context + .decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool_and_live_host_bytes( + decode_resources, + &targets, + pool, + budget.live_bytes(), + ) + .map_err(cuda_error)? + } else { + context + .decode_htj2k_codeblocks_cleanup_dequantize_multi_enqueue_with_resources_and_pool( + decode_resources, + &targets, + pool, + budget.live_bytes(), + ) + .map_err(cuda_error)? + } + }; + let cleanup_stats = queued.execution(); + let dequant_stats = if has_refinement { + // SAFETY: the queued guard owns the uploaded job metadata and remains + // live through final store completion; coefficient owners remain in + // component_work for the same duration. + unsafe { context.j2k_dequantize_queued_htj2k_cleanup_enqueue(&queued) } + .map_err(cuda_error)? + } else { + j2k_cuda_runtime::CudaExecutionStats::default() + }; + { + let accounting = &mut component_work[accounting_index]; + accounting.dispatches = accounting + .dispatches + .saturating_add(cleanup_stats.kernel_dispatches()) + .saturating_add(dequant_stats.kernel_dispatches()); + accounting.decode_dispatches = accounting + .decode_dispatches + .saturating_add(cleanup_stats.decode_kernel_dispatches()) + .saturating_add(dequant_stats.decode_kernel_dispatches()); + accounting.timings.ht_dispatch_count = accounting + .timings + .ht_dispatch_count + .saturating_add(cleanup_stats.kernel_dispatches()); + accounting.timings.dequant_dispatch_count = accounting + .timings + .dequant_dispatch_count + .saturating_add(dequant_stats.kernel_dispatches()); + } + for work in component_work { + work.pending_dequant_bands.clear(); + } + Ok(Some(queued)) +} diff --git a/crates/j2k-cuda/src/decoder/resident/routing.rs b/crates/j2k-cuda/src/decoder/resident/routing.rs index 104cd88b..057f0a18 100644 --- a/crates/j2k-cuda/src/decoder/resident/routing.rs +++ b/crates/j2k-cuda/src/decoder/resident/routing.rs @@ -8,6 +8,8 @@ use super::super::color_batch::{ decode_color_cuda_resident_region_scaled_surface, decode_color_cuda_resident_region_surface, decode_color_cuda_resident_scaled_surface, decode_color_cuda_resident_surface_with_profile, }; +#[cfg(feature = "cuda-runtime")] +use super::super::grayscale_batch::decode_grayscale_cuda_resident_batch_surfaces_with_profile; use super::super::{ profile, CudaHtj2kProfileReport, CudaSession, DeviceDecodePlan, DeviceDecodeRequest, Downscale, Error, J2kDecoder, PixelFormat, Rect, Surface, SurfaceResidency, @@ -90,6 +92,14 @@ pub(in crate::decoder) fn decode_batch_to_cuda_resident_surface_with_profile_con )); } match fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 | PixelFormat::GrayI16 => { + decode_grayscale_cuda_resident_batch_surfaces_with_profile( + inputs, + session, + fmt, + collect_stage_timings, + ) + } PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 | PixelFormat::Rgba16 => { decode_color_cuda_resident_batch_surfaces_with_profile( inputs, diff --git a/crates/j2k-cuda/src/direct_plan.rs b/crates/j2k-cuda/src/direct_plan.rs index a1791539..1ce791b1 100644 --- a/crates/j2k-cuda/src/direct_plan.rs +++ b/crates/j2k-cuda/src/direct_plan.rs @@ -1,5 +1,9 @@ use j2k_core::PixelFormat; -use j2k_native::{J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kWaveletTransform}; +use j2k_native::{ + HtCodeBlockPayloadRanges, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kWaveletTransform, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_native::{J2kClassicCodeBlockPayload, J2kCodestreamRange}; use crate::{allocation::HostPhaseBudget, Error}; @@ -11,18 +15,29 @@ mod shared; #[cfg(test)] mod tests; +#[cfg(feature = "cuda-runtime")] +use self::classic::referenced::{ + append_referenced_classic_subband, referenced_classic_payload_bytes, +}; use self::{ classic::append_classic_subband, - ht::append_ht_subband, + ht::{append_ht_subband, append_referenced_ht_subband, referenced_payload_bytes}, required_regions::required_regions_for_direct_plan, shared::{convert_store_step, CudaPlanOwners}, }; -const EMPTY_CUDA_PLAN: &str = "strict CUDA plan contains no entropy code blocks"; +const EMPTY_CUDA_COEFFICIENT_PLAN: &str = "strict CUDA plan contains no coefficient bands"; const MIXED_TRANSFORMS_UNSUPPORTED: &str = "strict CUDA HTJ2K plan contains mixed DWT transforms"; const PLAN_PAYLOAD_TOO_LARGE: &str = "strict CUDA HTJ2K plan payload is too large"; const PLAN_OUTPUT_RECT_MISMATCH: &str = "strict CUDA HTJ2K plan store does not fit the requested output rectangle"; +const REFERENCED_PLAN_CLASSIC_UNSUPPORTED: &str = + "prepared CUDA HTJ2K plan unexpectedly contains classic code blocks"; +#[cfg(feature = "cuda-runtime")] +const REFERENCED_CLASSIC_PLAN_HT_UNSUPPORTED: &str = + "prepared CUDA classic plan unexpectedly contains HT code blocks"; +const REFERENCED_PLAN_PAYLOAD_MISMATCH: &str = + "prepared CUDA HTJ2K geometry does not match referenced payload ranges"; /// CUDA-side DWT transform selector for a flat HTJ2K plan. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -306,6 +321,121 @@ pub(crate) struct CudaHtj2kDecodePlan { } impl CudaHtj2kDecodePlan { + #[cfg(feature = "cuda-runtime")] + #[expect( + clippy::too_many_arguments, + reason = "explicit retained classic tile inputs" + )] + pub(crate) fn from_referenced_classic_tile_grayscale_plan_into_shared( + plan: &J2kDirectGrayscalePlan, + payloads: &[J2kClassicCodeBlockPayload], + ranges: &[J2kCodestreamRange], + encoded: &[u8], + output_format: PixelFormat, + output_origin: (u32, u32), + output_dimensions: (u32, u32), + shared_payload: &mut Vec, + host_budget: &mut HostPhaseBudget, + ) -> Result { + let payload_bytes = referenced_classic_payload_bytes(encoded, payloads, ranges)?; + if payload_bytes != 0 { + host_budget.try_vec_reserve(shared_payload, payload_bytes)?; + } + let (mut owners, _) = CudaPlanOwners::from_referenced_plan(plan)?; + let mut payloads = payloads.iter(); + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::HtSubBand(_) => { + return Err(Error::UnsupportedCudaRequest { + reason: REFERENCED_CLASSIC_PLAN_HT_UNSUPPORTED, + }); + } + J2kDirectGrayscaleStep::ClassicSubBand(subband) => { + append_referenced_classic_subband( + &mut owners, + subband, + None, + &mut payloads, + ranges, + encoded, + shared_payload, + )?; + } + J2kDirectGrayscaleStep::Idwt(step) => owners.append_idwt(*step)?, + J2kDirectGrayscaleStep::Store(step) => { + owners + .store_steps + .push(shared::convert_referenced_tile_store_step( + *step, + output_dimensions, + )?); + } + } + } + if payloads.next().is_some() { + return Err(Error::UnsupportedCudaRequest { + reason: REFERENCED_PLAN_PAYLOAD_MISMATCH, + }); + } + owners.finish(plan, output_format, output_origin, output_dimensions) + } + + #[expect( + clippy::too_many_arguments, + reason = "the tile adapter explicitly carries source bytes, output geometry, shared arena, and allocation budget" + )] + pub(crate) fn from_referenced_tile_grayscale_plan_into_shared( + plan: &J2kDirectGrayscalePlan, + payloads: &[HtCodeBlockPayloadRanges], + encoded: &[u8], + output_format: PixelFormat, + output_origin: (u32, u32), + output_dimensions: (u32, u32), + shared_payload: &mut Vec, + host_budget: &mut HostPhaseBudget, + ) -> Result { + let payload_bytes = referenced_payload_bytes(encoded, payloads)?; + if payload_bytes != 0 { + host_budget.try_vec_reserve(shared_payload, payload_bytes)?; + } + let (mut owners, _) = CudaPlanOwners::from_referenced_plan(plan)?; + let mut payloads = payloads.iter(); + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::HtSubBand(subband) => { + append_referenced_ht_subband( + &mut owners, + subband, + None, + &mut payloads, + encoded, + shared_payload, + )?; + } + J2kDirectGrayscaleStep::ClassicSubBand(_) => { + return Err(Error::UnsupportedCudaRequest { + reason: REFERENCED_PLAN_CLASSIC_UNSUPPORTED, + }); + } + J2kDirectGrayscaleStep::Idwt(step) => owners.append_idwt(*step)?, + J2kDirectGrayscaleStep::Store(step) => { + owners + .store_steps + .push(shared::convert_referenced_tile_store_step( + *step, + output_dimensions, + )?); + } + } + } + if payloads.next().is_some() { + return Err(Error::UnsupportedCudaRequest { + reason: REFERENCED_PLAN_PAYLOAD_MISMATCH, + }); + } + owners.finish(plan, output_format, output_origin, output_dimensions) + } + pub(crate) fn from_grayscale_direct_plan( plan: &J2kDirectGrayscalePlan, output_format: PixelFormat, diff --git a/crates/j2k-cuda/src/direct_plan/accessors.rs b/crates/j2k-cuda/src/direct_plan/accessors.rs index 8a7fb6ea..f4739c26 100644 --- a/crates/j2k-cuda/src/direct_plan/accessors.rs +++ b/crates/j2k-cuda/src/direct_plan/accessors.rs @@ -12,13 +12,6 @@ use super::{ impl CudaHtj2kDecodePlan { #[cfg(feature = "cuda-runtime")] - #[cfg_attr( - not(test), - expect( - dead_code, - reason = "output dimensions accessor supports CUDA plan tests" - ) - )] pub(crate) fn dimensions(&self) -> (u32, u32) { self.dimensions } diff --git a/crates/j2k-cuda/src/direct_plan/classic.rs b/crates/j2k-cuda/src/direct_plan/classic.rs index 4fb762c7..b0c8b5a2 100644 --- a/crates/j2k-cuda/src/direct_plan/classic.rs +++ b/crates/j2k-cuda/src/direct_plan/classic.rs @@ -17,6 +17,9 @@ const STYLE_VERTICALLY_CAUSAL_CONTEXT: u32 = 1 << 2; const STYLE_SEGMENTATION_SYMBOLS: u32 = 1 << 3; const STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS: u32 = 1 << 4; +#[cfg(feature = "cuda-runtime")] +pub(super) mod referenced; + pub(super) fn append_classic_subband( owners: &mut CudaPlanOwners, subband: &J2kOwnedSubBandPlan, @@ -51,12 +54,23 @@ fn append_classic_job( subband_index: u32, job: &J2kOwnedCodeBlockBatchJob, ) -> Result<(), Error> { - validate_classic_job(job)?; + validate_classic_job(job, job.data.len())?; let payload_offset = checked_u64(owners.payload.len())?; let payload_len = checked_u32(job.data.len())?; + append_classic_job_metadata(owners, subband_index, job, payload_offset, payload_len)?; + owners.payload.extend_from_slice(&job.data); + Ok(()) +} + +fn append_classic_job_metadata( + owners: &mut CudaPlanOwners, + subband_index: u32, + job: &J2kOwnedCodeBlockBatchJob, + payload_offset: u64, + payload_len: u32, +) -> Result<(), Error> { let segment_start = checked_u32(owners.classic_segments.len())?; let output_stride = checked_u32(job.output_stride)?; - owners.payload.extend_from_slice(&job.data); owners .classic_segments .extend(job.segments.iter().map(convert_classic_segment)); @@ -82,7 +96,7 @@ fn append_classic_job( Ok(()) } -fn validate_classic_job(job: &J2kOwnedCodeBlockBatchJob) -> Result<(), Error> { +fn validate_classic_job(job: &J2kOwnedCodeBlockBatchJob, payload_len: usize) -> Result<(), Error> { if job.roi_shift != 0 || !(1..=64).contains(&job.width) || !(1..=64).contains(&job.height) @@ -113,7 +127,7 @@ fn validate_classic_job(job: &J2kOwnedCodeBlockBatchJob) -> Result<(), Error> { )?; } if expected_pass != job.number_of_coding_passes - || usize::try_from(expected_offset).ok() != Some(job.data.len()) + || usize::try_from(expected_offset).ok() != Some(payload_len) { return invalid_classic_plan(); } diff --git a/crates/j2k-cuda/src/direct_plan/classic/referenced.rs b/crates/j2k-cuda/src/direct_plan/classic/referenced.rs new file mode 100644 index 00000000..a6d78353 --- /dev/null +++ b/crates/j2k-cuda/src/direct_plan/classic/referenced.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_native::{ + J2kClassicCodeBlockPayload, J2kCodestreamRange, J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, +}; + +use super::{ + append_classic_job_metadata, checked_u32, checked_u64, invalid_classic_plan, + validate_classic_job, CLASSIC_PLAN_INVALID, +}; +use crate::direct_plan::{ + required_regions::RequiredBandRegions, shared::CudaPlanOwners, CudaClassicSubband, + CudaHtj2kDecodePlan, Error, PLAN_PAYLOAD_TOO_LARGE, +}; + +impl CudaHtj2kDecodePlan { + pub(crate) fn validate_referenced_classic_payload_sequence( + payloads: &[J2kClassicCodeBlockPayload], + ranges: &[J2kCodestreamRange], + ) -> Result<(), Error> { + validate_referenced_classic_payload_sequence(payloads, ranges) + } +} + +pub(in crate::direct_plan) fn append_referenced_classic_subband<'a>( + owners: &mut CudaPlanOwners, + subband: &J2kOwnedSubBandPlan, + required_regions: Option<&RequiredBandRegions>, + payloads: &mut impl Iterator, + ranges: &[J2kCodestreamRange], + encoded: &[u8], + shared_payload: &mut Vec, +) -> Result<(), Error> { + let subband_index = checked_u32(owners.classic_subbands.len())?; + let code_block_start = checked_u32(owners.classic_code_blocks.len())?; + for job in &subband.jobs { + let payload = payloads.next().ok_or(Error::UnsupportedCudaRequest { + reason: CLASSIC_PLAN_INVALID, + })?; + let fragments = referenced_classic_ranges(encoded, *payload, ranges)?; + validate_referenced_classic_job(job, payload.combined_length)?; + if required_regions.is_some_and(|regions| { + !regions.get(subband.band_id).is_some_and(|required| { + required.intersects(job.output_x, job.output_y, job.width, job.height) + }) + }) { + continue; + } + let payload_offset = checked_u64(shared_payload.len())?; + for range in fragments { + shared_payload.extend_from_slice(referenced_classic_slice(encoded, *range)?); + } + append_classic_job_metadata( + owners, + subband_index, + job, + payload_offset, + checked_u32(payload.combined_length)?, + )?; + } + owners.classic_subbands.push(CudaClassicSubband { + band_id: subband.band_id, + width: subband.width, + height: subband.height, + code_block_start, + code_block_count: checked_u32( + owners.classic_code_blocks.len() - code_block_start as usize, + )?, + }); + Ok(()) +} + +pub(in crate::direct_plan) fn referenced_classic_payload_bytes( + encoded: &[u8], + payloads: &[J2kClassicCodeBlockPayload], + ranges: &[J2kCodestreamRange], +) -> Result { + payloads.iter().try_fold(0usize, |total, payload| { + referenced_classic_ranges(encoded, *payload, ranges)?; + total + .checked_add(payload.combined_length) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + }) + }) +} + +pub(in crate::direct_plan) fn validate_referenced_classic_payload_sequence( + payloads: &[J2kClassicCodeBlockPayload], + ranges: &[J2kCodestreamRange], +) -> Result<(), Error> { + let mut next_range = 0usize; + for payload in payloads { + if payload.first_range != next_range { + return invalid_classic_plan(); + } + next_range = payload.end_range().ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + } + if next_range != ranges.len() { + return invalid_classic_plan(); + } + Ok(()) +} + +fn validate_referenced_classic_job( + job: &J2kOwnedCodeBlockBatchJob, + payload_len: usize, +) -> Result<(), Error> { + if !job.data.is_empty() { + return invalid_classic_plan(); + } + validate_classic_job(job, payload_len) +} + +fn referenced_classic_ranges<'a>( + encoded: &[u8], + payload: J2kClassicCodeBlockPayload, + ranges: &'a [J2kCodestreamRange], +) -> Result<&'a [J2kCodestreamRange], Error> { + let end_range = payload.end_range().ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + let selected = + ranges + .get(payload.first_range..end_range) + .ok_or(Error::UnsupportedCudaRequest { + reason: CLASSIC_PLAN_INVALID, + })?; + let mut combined = 0usize; + for range in selected { + combined = combined + .checked_add(referenced_classic_slice(encoded, *range)?.len()) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + } + if combined != payload.combined_length { + return invalid_classic_plan(); + } + Ok(selected) +} + +fn referenced_classic_slice(encoded: &[u8], range: J2kCodestreamRange) -> Result<&[u8], Error> { + let end = range.end().ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + encoded + .get(range.offset..end) + .ok_or(Error::UnsupportedCudaRequest { + reason: CLASSIC_PLAN_INVALID, + }) +} diff --git a/crates/j2k-cuda/src/direct_plan/ht.rs b/crates/j2k-cuda/src/direct_plan/ht.rs index 56bc75c8..1a3b73e7 100644 --- a/crates/j2k-cuda/src/direct_plan/ht.rs +++ b/crates/j2k-cuda/src/direct_plan/ht.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use j2k_native::HtOwnedSubBandPlan; +use j2k_native::{HtCodeBlockPayloadRanges, HtOwnedSubBandPlan, J2kCodestreamRange}; use super::{ required_regions::RequiredBandRegions, shared::CudaPlanOwners, CudaHtj2kCodeBlock, @@ -79,6 +79,126 @@ pub(super) fn append_ht_subband( Ok(()) } +pub(super) fn append_referenced_ht_subband<'a>( + owners: &mut CudaPlanOwners, + subband: &HtOwnedSubBandPlan, + required_regions: Option<&RequiredBandRegions>, + payloads: &mut impl Iterator, + encoded: &[u8], + shared_payload: &mut Vec, +) -> Result<(), Error> { + let subband_index = checked_u32(owners.subbands.len())?; + let code_block_start = checked_u32(owners.code_blocks.len())?; + for job in &subband.jobs { + let ranges = payloads.next().ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_BLOCK_LENGTH_MISMATCH, + })?; + if required_regions.is_some_and(|regions| { + !regions.get(subband.band_id).is_some_and(|required| { + required.intersects(job.output_x, job.output_y, job.width, job.height) + }) + }) { + continue; + } + if !job.data.is_empty() || job.roi_shift != 0 { + return Err(Error::UnsupportedCudaRequest { + reason: if job.roi_shift != 0 { + ROI_MAXSHIFT_UNSUPPORTED + } else { + PLAN_BLOCK_LENGTH_MISMATCH + }, + }); + } + let cleanup = referenced_slice(encoded, ranges.cleanup)?; + let refinement = match (job.refinement_length, ranges.refinement) { + (0, None) => &[][..], + (0, Some(range)) if range.length == 0 => &[][..], + (_, Some(range)) => referenced_slice(encoded, range)?, + (_, None) => { + return Err(Error::UnsupportedCudaRequest { + reason: PLAN_BLOCK_LENGTH_MISMATCH, + }); + } + }; + if cleanup.len() != job.cleanup_length as usize + || refinement.len() != job.refinement_length as usize + { + return Err(Error::UnsupportedCudaRequest { + reason: PLAN_BLOCK_LENGTH_MISMATCH, + }); + } + let payload_offset = checked_u64(shared_payload.len())?; + let payload_len = + cleanup + .len() + .checked_add(refinement.len()) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + let payload_len = checked_u32(payload_len)?; + shared_payload.extend_from_slice(cleanup); + shared_payload.extend_from_slice(refinement); + owners.code_blocks.push(CudaHtj2kCodeBlock { + subband_index, + payload_offset, + payload_len, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + output_x: job.output_x, + output_y: job.output_y, + width: job.width, + height: job.height, + output_stride: checked_u32(job.output_stride)?, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + stripe_causal: u8::from(job.stripe_causal), + dequantization_step: job.dequantization_step, + }); + } + owners.subbands.push(CudaHtj2kSubband { + band_id: subband.band_id, + x0: subband.rect.x0, + y0: subband.rect.y0, + x1: subband.rect.x1, + y1: subband.rect.y1, + width: subband.width, + height: subband.height, + code_block_start, + code_block_count: checked_u32(owners.code_blocks.len() - code_block_start as usize)?, + }); + Ok(()) +} + +pub(super) fn referenced_payload_bytes( + encoded: &[u8], + payloads: &[HtCodeBlockPayloadRanges], +) -> Result { + payloads.iter().try_fold(0usize, |total, payload| { + let cleanup = referenced_slice(encoded, payload.cleanup)?.len(); + let refinement = payload.refinement.map_or(Ok(0), |range| { + referenced_slice(encoded, range).map(<[u8]>::len) + })?; + total + .checked_add(cleanup) + .and_then(|value| value.checked_add(refinement)) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + }) + }) +} + +fn referenced_slice(encoded: &[u8], range: J2kCodestreamRange) -> Result<&[u8], Error> { + let end = range.end().ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + encoded + .get(range.offset..end) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_BLOCK_LENGTH_MISMATCH, + }) +} + fn checked_u32(value: usize) -> Result { u32::try_from(value).map_err(|_| Error::UnsupportedCudaRequest { reason: PLAN_PAYLOAD_TOO_LARGE, diff --git a/crates/j2k-cuda/src/direct_plan/shared.rs b/crates/j2k-cuda/src/direct_plan/shared.rs index 8f1360c9..cab98c88 100644 --- a/crates/j2k-cuda/src/direct_plan/shared.rs +++ b/crates/j2k-cuda/src/direct_plan/shared.rs @@ -7,7 +7,7 @@ use j2k_native::{ use super::{ CudaClassicCodeBlock, CudaClassicSegment, CudaClassicSubband, CudaHtj2kCodeBlock, CudaHtj2kDecodePlan, CudaHtj2kIdwtStep, CudaHtj2kRect, CudaHtj2kStoreStep, CudaHtj2kSubband, - CudaHtj2kTransform, Error, EMPTY_CUDA_PLAN, MIXED_TRANSFORMS_UNSUPPORTED, + CudaHtj2kTransform, Error, EMPTY_CUDA_COEFFICIENT_PLAN, MIXED_TRANSFORMS_UNSUPPORTED, PLAN_OUTPUT_RECT_MISMATCH, PLAN_PAYLOAD_TOO_LARGE, }; use crate::allocation::HostPhaseBudget; @@ -38,7 +38,23 @@ pub(super) struct CudaPlanOwners { impl CudaPlanOwners { pub(super) fn from_plan(plan: &J2kDirectGrayscalePlan) -> Result<(Self, usize), Error> { - let hint = cuda_plan_capacity_hint(plan)?; + Self::from_plan_with_payload_capacity(plan, None) + } + + pub(super) fn from_referenced_plan( + plan: &J2kDirectGrayscalePlan, + ) -> Result<(Self, usize), Error> { + Self::from_plan_with_payload_capacity(plan, Some(0)) + } + + fn from_plan_with_payload_capacity( + plan: &J2kDirectGrayscalePlan, + payload_capacity: Option, + ) -> Result<(Self, usize), Error> { + let mut hint = cuda_plan_capacity_hint(plan)?; + if let Some(payload_capacity) = payload_capacity { + hint.payload_bytes = payload_capacity; + } let mut budget = HostPhaseBudget::new("CUDA direct-plan owner graph"); let owners = Self { payload: budget.try_vec_with_capacity(hint.payload_bytes)?, @@ -76,9 +92,12 @@ impl CudaPlanOwners { output_origin: (u32, u32), dimensions: (u32, u32), ) -> Result { - if self.code_blocks.is_empty() && self.classic_code_blocks.is_empty() { + // An omitted packet is a valid all-zero coefficient band. Keep that + // executable graph so the zero-filled band can flow through IDWT and + // final color reconstruction even when it has no entropy jobs. + if self.subbands.is_empty() && self.classic_subbands.is_empty() { return Err(Error::UnsupportedCudaRequest { - reason: EMPTY_CUDA_PLAN, + reason: EMPTY_CUDA_COEFFICIENT_PLAN, }); } Ok(CudaHtj2kDecodePlan { @@ -159,6 +178,18 @@ pub(super) fn convert_store_step( output_origin: (u32, u32), output_dimensions: (u32, u32), ) -> Result { + let already_dense = (step.output_width, step.output_height) == output_dimensions + && step + .output_x + .checked_add(step.copy_width) + .is_some_and(|end| end <= output_dimensions.0) + && step + .output_y + .checked_add(step.copy_height) + .is_some_and(|end| end <= output_dimensions.1); + if already_dense { + return convert_referenced_tile_store_step(step, output_dimensions); + } if output_dimensions.0 == 0 || output_dimensions.1 == 0 { return output_rect_error(); } @@ -190,6 +221,33 @@ pub(super) fn convert_store_step( }) } +pub(super) fn convert_referenced_tile_store_step( + step: J2kDirectStoreStep, + output_dimensions: (u32, u32), +) -> Result { + if output_dimensions.0 == 0 + || output_dimensions.1 == 0 + || (step.output_width, step.output_height) != output_dimensions + || checked_end(step.output_x, step.copy_width)? > output_dimensions.0 + || checked_end(step.output_y, step.copy_height)? > output_dimensions.1 + { + return output_rect_error(); + } + Ok(CudaHtj2kStoreStep { + input_band_id: step.input_band_id, + input_rect: convert_rect(step.input_rect), + source_x: step.source_x, + source_y: step.source_y, + copy_width: step.copy_width, + copy_height: step.copy_height, + output_width: step.output_width, + output_height: step.output_height, + output_x: step.output_x, + output_y: step.output_y, + addend: step.addend, + }) +} + fn checked_end(start: u32, len: u32) -> Result { start.checked_add(len).ok_or(Error::UnsupportedCudaRequest { reason: PLAN_OUTPUT_RECT_MISMATCH, diff --git a/crates/j2k-cuda/src/direct_plan/tests.rs b/crates/j2k-cuda/src/direct_plan/tests.rs index d8b039c4..27b8d34c 100644 --- a/crates/j2k-cuda/src/direct_plan/tests.rs +++ b/crates/j2k-cuda/src/direct_plan/tests.rs @@ -327,3 +327,70 @@ fn region_plan_rejects_store_outside_output_rect() { "unexpected error: {error}" ); } + +#[test] +fn referenced_prepared_plan_materializes_only_the_execution_arena() { + let encoded = std::sync::Arc::<[u8]>::from( + j2k_native::encode_htj2k( + &(0_u8..64).collect::>(), + 8, + 8, + 1, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode referenced-plan fixture"), + ); + let prepared = j2k::prepare_batch( + vec![j2k::EncodedImage::full(std::sync::Arc::clone(&encoded))], + j2k::BatchDecodeOptions::default(), + ) + .expect("prepare referenced-plan fixture"); + let image = &prepared.groups()[0].images()[0]; + let prepared_plan = image.htj2k_plan().expect("retained HTJ2K plan"); + let referenced = prepared_plan + .adapter_view() + .downcast_ref::() + .expect("native referenced HTJ2K plan adapter"); + let tile = &referenced.tiles()[0]; + let geometry = tile + .grayscale_geometry() + .expect("grayscale referenced tile geometry"); + let span = tile.payload_records(); + let payload_end = span.end_record().expect("payload span end"); + let tile_payloads = &referenced.payloads()[span.first_record..payload_end]; + let mut expected = Vec::new(); + for payload in referenced.payloads() { + let cleanup_end = payload.cleanup.end().expect("cleanup range end"); + expected.extend_from_slice(&encoded[payload.cleanup.offset..cleanup_end]); + if let Some(refinement) = payload.refinement { + let refinement_end = refinement.end().expect("refinement range end"); + expected.extend_from_slice(&encoded[refinement.offset..refinement_end]); + } + } + + let output = image.plan().output_rect(); + let mut shared = Vec::new(); + let mut budget = crate::allocation::HostPhaseBudget::new("referenced CUDA plan test"); + let cuda = CudaHtj2kDecodePlan::from_referenced_tile_grayscale_plan_into_shared( + geometry, + tile_payloads, + &encoded, + PixelFormat::Gray8, + (output.x, output.y), + (output.w, output.h), + &mut shared, + &mut budget, + ) + .expect("flatten referenced CUDA plan"); + + assert_eq!(shared, expected); + assert!(cuda.payload().is_empty()); + assert_eq!(cuda.code_blocks().len(), referenced.payloads().len()); + assert_eq!(cuda.code_blocks()[0].payload_offset, 0); +} diff --git a/crates/j2k-cuda/src/error.rs b/crates/j2k-cuda/src/error.rs index 72ef98bd..cb15ea68 100644 --- a/crates/j2k-cuda/src/error.rs +++ b/crates/j2k-cuda/src/error.rs @@ -51,6 +51,9 @@ pub enum Error { /// Logical phase or owner graph. what: &'static str, }, + /// Bounded HTJ2K GPU job planning rejected one source job. + #[error(transparent)] + HtJobChunkPlan(#[from] j2k_core::HtGpuJobChunkPlanError), /// Backend request is unsupported by this adapter. #[error("backend request {request:?} is not supported by j2k-cuda")] UnsupportedBackend { @@ -75,6 +78,20 @@ pub enum Error { source: j2k_cuda_runtime::CudaError, }, #[cfg(feature = "cuda-runtime")] + /// A classic JPEG 2000 or HTJ2K tier-1 descriptor failed during GPU execution. + #[error( + "CUDA tier-1 source {source_index} job {original_job_index} failed during device execution: {source}" + )] + CudaTier1JobFailed { + /// Original caller input index that owns the failed job. + source_index: usize, + /// Stable job index before pass-bucket ordering and chunk splitting. + original_job_index: usize, + /// Indexed CUDA kernel failure. + #[source] + source: j2k_cuda_runtime::CudaError, + }, + #[cfg(feature = "cuda-runtime")] /// A CUDA operation failed and the required resource cleanup also failed. #[error("CUDA operation failed ({primary}); CUDA cleanup also failed ({cleanup})")] CudaCleanupFailed { @@ -85,6 +102,42 @@ pub enum Error { }, } +impl Error { + /// Whether this error can leave submitted CUDA work referencing an + /// external allocation whose completion was not established. + #[cfg(feature = "cuda-runtime")] + #[doc(hidden)] + pub fn completion_is_uncertain(&self) -> bool { + match self { + Self::CudaRuntime { source } | Self::CudaTier1JobFailed { source, .. } => { + source.completion_is_uncertain() + } + Self::CudaCleanupFailed { primary, cleanup } => { + primary.completion_is_uncertain() || cleanup.completion_is_uncertain() + } + _ => false, + } + } + + /// Whether the persistent CUDA session cannot safely execute later groups. + #[doc(hidden)] + #[must_use] + pub fn session_is_unusable(&self) -> bool { + match self { + Self::CudaUnavailable => true, + #[cfg(feature = "cuda-runtime")] + Self::CudaRuntime { source } | Self::CudaTier1JobFailed { source, .. } => { + source.session_is_unusable() + } + #[cfg(feature = "cuda-runtime")] + Self::CudaCleanupFailed { primary, cleanup } => { + primary.session_is_unusable() || cleanup.session_is_unusable() + } + _ => false, + } + } +} + #[cfg(feature = "cuda-runtime")] pub(crate) fn combine_cuda_cleanup_errors(primary_error: Error, cleanup_error: Error) -> Error { Error::CudaCleanupFailed { @@ -111,9 +164,12 @@ impl AdapterErrorParts for Error { Self::Decode(_) | Self::NativeDecode { .. } | Self::HostAllocationFailed { .. } - | Self::HostAllocationTooLarge { .. } => AdapterErrorKind::Other, + | Self::HostAllocationTooLarge { .. } + | Self::HtJobChunkPlan(_) => AdapterErrorKind::Other, #[cfg(feature = "cuda-runtime")] - Self::CudaRuntime { .. } | Self::CudaCleanupFailed { .. } => AdapterErrorKind::Other, + Self::CudaRuntime { .. } + | Self::CudaTier1JobFailed { .. } + | Self::CudaCleanupFailed { .. } => AdapterErrorKind::Other, } } } diff --git a/crates/j2k-cuda/src/lib.rs b/crates/j2k-cuda/src/lib.rs index f918d5ff..15f1c556 100644 --- a/crates/j2k-cuda/src/lib.rs +++ b/crates/j2k-cuda/src/lib.rs @@ -12,6 +12,7 @@ #![warn(unreachable_pub)] mod allocation; +mod batch; mod codec; mod decoder; #[cfg(any(feature = "cuda-runtime", test))] @@ -23,6 +24,14 @@ mod runtime; mod session; mod surface; +pub use batch::{ + CudaBatchDecodeResult, CudaBatchDecoder, CudaBatchError, CudaBatchGroup, CudaBatchGroupError, +}; +#[cfg(feature = "cuda-runtime")] +pub use batch::{ + CudaExternalBatchGroup, CudaExternalBatchTryFinish, CudaResidentBatchBuffer, + SubmittedCudaExternalBatch, SubmittedCudaResidentBatch, +}; pub use codec::Codec; pub use decoder::J2kDecoder; #[cfg(feature = "cuda-runtime")] @@ -53,4 +62,6 @@ pub use profile::{ CudaHtj2kDecodeProfileDetail, CudaHtj2kEncodeProfileReport, CudaHtj2kProfileReport, }; pub use session::CudaSession; +#[cfg(feature = "cuda-runtime")] +pub use session::{CudaDecodePoolDiagnostics, CudaDecodePoolSnapshot, CudaSessionDiagnostics}; pub use surface::{CudaSurface, CudaSurfaceStats, Surface, SurfaceResidency}; diff --git a/crates/j2k-cuda/src/session.rs b/crates/j2k-cuda/src/session.rs index 55aaa28f..7ddab13c 100644 --- a/crates/j2k-cuda/src/session.rs +++ b/crates/j2k-cuda/src/session.rs @@ -2,11 +2,13 @@ #[cfg(feature = "cuda-runtime")] use j2k_cuda_runtime::{ - CudaBufferPool, CudaContext, CudaHtj2kDecodeTableResources, CudaHtj2kDecodeTables, - CudaHtj2kEncodeResources, + CudaBufferPool, CudaClassicDecodeTableResources, CudaContext, CudaContextDiagnostics, + CudaHtj2kDecodeTableResources, CudaHtj2kDecodeTables, CudaHtj2kEncodeResources, }; #[cfg(feature = "cuda-runtime")] use j2k_native::{ht_uvlc_table0, ht_uvlc_table1, ht_vlc_table0, ht_vlc_table1}; +#[cfg(feature = "cuda-runtime")] +use std::num::NonZeroUsize; #[cfg(all(test, feature = "cuda-runtime"))] use std::sync::atomic::{AtomicUsize, Ordering}; #[cfg(feature = "cuda-runtime")] @@ -19,6 +21,105 @@ use crate::Error; #[cfg(all(test, feature = "cuda-runtime"))] static HTJ2K_DECODE_TABLE_UPLOADS: AtomicUsize = AtomicUsize::new(0); +#[cfg(all(test, feature = "cuda-runtime"))] +static CLASSIC_DECODE_TABLE_UPLOADS: AtomicUsize = AtomicUsize::new(0); + +/// Stable retention snapshot for one internal CUDA decode buffer pool. +#[cfg(feature = "cuda-runtime")] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaDecodePoolSnapshot { + /// Completed device buffers immediately available for reuse. + pub cached_buffers: usize, + /// Completed device-allocation bytes immediately available for reuse. + pub cached_bytes: usize, + /// Buffers held until queued work establishes completion. + pub deferred_buffers: usize, + /// Device-allocation bytes held until queued work establishes completion. + pub deferred_bytes: usize, + /// Active completion guards preventing deferred-buffer reuse. + pub reuse_holds: usize, + /// Highest completed allocation-byte total observed by this pool. + pub peak_cached_bytes: usize, + /// Highest deferred allocation-byte total observed by this pool. + pub peak_deferred_bytes: usize, +} + +#[cfg(feature = "cuda-runtime")] +impl CudaDecodePoolSnapshot { + /// Device bytes currently retained either for reuse or pending completion. + #[must_use] + pub const fn retained_bytes(self) -> usize { + self.cached_bytes.saturating_add(self.deferred_bytes) + } + + /// Conservative upper bound obtained by adding the independent pool peaks. + #[must_use] + pub const fn peak_retained_bytes_upper_bound(self) -> usize { + self.peak_cached_bytes + .saturating_add(self.peak_deferred_bytes) + } +} + +#[cfg(feature = "cuda-runtime")] +impl From for CudaDecodePoolSnapshot { + fn from(value: j2k_cuda_runtime::CudaBufferPoolDiagnostics) -> Self { + Self { + cached_buffers: value.cached_buffers, + cached_bytes: value.cached_bytes, + deferred_buffers: value.deferred_buffers, + deferred_bytes: value.deferred_bytes, + reuse_holds: value.reuse_holds, + peak_cached_bytes: value.peak_cached_bytes, + peak_deferred_bytes: value.peak_deferred_bytes, + } + } +} + +/// Diagnostics for the two private pools retained by one CUDA codec session. +#[cfg(feature = "cuda-runtime")] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaDecodePoolDiagnostics { + /// General single-image/component decode pool, when initialized. + pub decode: Option, + /// Best-fit dense batch decode pool, when initialized. + pub batch_decode: Option, +} + +/// Combined runtime work counters and retained decode-pool state for one session. +#[cfg(feature = "cuda-runtime")] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaSessionDiagnostics { + /// Context-level transfer, launch, allocation, and synchronization counters, + /// when CUDA was initialized. + pub runtime: Option, + /// Private decode-buffer pool state. + pub pools: CudaDecodePoolDiagnostics, +} + +#[cfg(feature = "cuda-runtime")] +impl CudaDecodePoolDiagnostics { + /// Total device bytes currently retained by both initialized decode pools. + #[must_use] + pub fn retained_bytes(self) -> usize { + self.decode + .map_or(0, CudaDecodePoolSnapshot::retained_bytes) + .saturating_add( + self.batch_decode + .map_or(0, CudaDecodePoolSnapshot::retained_bytes), + ) + } + + /// Conservative sum of the independent high-water bounds for both pools. + #[must_use] + pub fn peak_retained_bytes_upper_bound(self) -> usize { + self.decode + .map_or(0, CudaDecodePoolSnapshot::peak_retained_bytes_upper_bound) + .saturating_add( + self.batch_decode + .map_or(0, CudaDecodePoolSnapshot::peak_retained_bytes_upper_bound), + ) + } +} #[cfg(feature = "cuda-runtime")] mod encode_resources; @@ -35,6 +136,8 @@ pub struct CudaSession { #[cfg(feature = "cuda-runtime")] htj2k_decode_tables: Option, #[cfg(feature = "cuda-runtime")] + classic_decode_tables: Option, + #[cfg(feature = "cuda-runtime")] htj2k_encode_resources: Option>, #[cfg(all(test, feature = "cuda-runtime"))] htj2k_encode_resource_uploads: usize, @@ -42,6 +145,10 @@ pub struct CudaSession { decode_buffer_pool: Option, #[cfg(feature = "cuda-runtime")] decode_batch_buffer_pool: Option, + #[cfg(feature = "cuda-runtime")] + htj2k_decode_chunk_limits: Option, + #[cfg(all(test, feature = "cuda-runtime"))] + last_htj2k_decode_chunk_count: usize, } impl CudaSession { @@ -66,6 +173,31 @@ impl CudaSession { self.context.is_some() } + /// Return the session-owned context for a framework-owned destination. + /// + /// The first call retains the requested device's primary context. Later + /// calls reject a different device ordinal, so adapters cannot construct a + /// destination view against a context other than the one this session owns. + #[cfg(feature = "cuda-runtime")] + #[doc(hidden)] + pub fn context_for_device_interop( + &mut self, + device_ordinal: usize, + ) -> Result { + if let Some(context) = &self.context { + if context.device_ordinal() != device_ordinal { + return Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA interop device does not match the persistent session", + }); + } + return Ok(context.clone()); + } + + let context = CudaContext::retain_primary(device_ordinal).map_err(cuda_error)?; + self.context = Some(context.clone()); + Ok(context) + } + #[cfg(feature = "cuda-runtime")] pub(crate) fn cuda_context(&mut self) -> Result { if self.context.is_none() { @@ -98,6 +230,23 @@ impl CudaSession { Ok(resources) } + #[cfg(feature = "cuda-runtime")] + pub(crate) fn classic_decode_table_resources( + &mut self, + ) -> Result { + if let Some(tables) = &self.classic_decode_tables { + return Ok(tables.clone()); + } + let context = self.cuda_context()?; + let tables = context + .upload_classic_decode_table_resources() + .map_err(cuda_error)?; + #[cfg(test)] + CLASSIC_DECODE_TABLE_UPLOADS.fetch_add(1, Ordering::Relaxed); + self.classic_decode_tables = Some(tables.clone()); + Ok(tables) + } + #[cfg(feature = "cuda-runtime")] pub(crate) fn htj2k_encode_resources( &mut self, @@ -153,6 +302,76 @@ impl CudaSession { self.decode_batch_buffer_pool = Some(pool.clone()); Ok(pool) } + + /// Snapshot only the two device-buffer pools retained for decode work. + /// + /// Calling this method does not initialize a CUDA context or either pool. + #[cfg(feature = "cuda-runtime")] + pub fn decode_pool_diagnostics(&self) -> Result { + Ok(CudaDecodePoolDiagnostics { + decode: self + .decode_buffer_pool + .as_ref() + .map(|pool| pool.diagnostics().map(CudaDecodePoolSnapshot::from)) + .transpose() + .map_err(cuda_error)?, + batch_decode: self + .decode_batch_buffer_pool + .as_ref() + .map(|pool| pool.diagnostics().map(CudaDecodePoolSnapshot::from)) + .transpose() + .map_err(cuda_error)?, + }) + } + + /// Snapshot transfer/event counters and decode-pool retention without initializing CUDA. + #[cfg(feature = "cuda-runtime")] + pub fn diagnostics(&self) -> Result { + Ok(CudaSessionDiagnostics { + runtime: self + .context + .as_ref() + .map(CudaContext::diagnostics) + .transpose() + .map_err(cuda_error)?, + pools: self.decode_pool_diagnostics()?, + }) + } + + #[cfg(feature = "cuda-runtime")] + pub(crate) fn htj2k_decode_chunk_limits(&self) -> j2k_core::HtGpuJobChunkLimits { + if let Some(limits) = self.htj2k_decode_chunk_limits { + return limits; + } + let Some(max_jobs) = NonZeroUsize::new(65_536) else { + return j2k_core::HtGpuJobChunkLimits::new(NonZeroUsize::MIN, 0, 0); + }; + j2k_core::HtGpuJobChunkLimits::new( + max_jobs, + 64 * 1024 * 1024, + max_jobs + .get() + .saturating_mul(j2k_cuda_runtime::htj2k_cleanup_multi_descriptor_bytes()), + ) + } + + #[cfg(all(test, feature = "cuda-runtime"))] + pub(crate) fn set_htj2k_decode_chunk_limits_for_test( + &mut self, + limits: j2k_core::HtGpuJobChunkLimits, + ) { + self.htj2k_decode_chunk_limits = Some(limits); + } + + #[cfg(all(test, feature = "cuda-runtime"))] + pub(crate) fn record_htj2k_decode_chunk_count_for_test(&mut self, count: usize) { + self.last_htj2k_decode_chunk_count = count; + } + + #[cfg(all(test, feature = "cuda-runtime"))] + pub(crate) fn last_htj2k_decode_chunk_count_for_test(&self) -> usize { + self.last_htj2k_decode_chunk_count + } } impl j2k_core::DeviceSubmitSession for CudaSession { @@ -185,6 +404,16 @@ pub(crate) fn htj2k_decode_table_uploads_for_test() -> usize { HTJ2K_DECODE_TABLE_UPLOADS.load(Ordering::Relaxed) } +#[cfg(all(test, feature = "cuda-runtime"))] +pub(crate) fn reset_classic_decode_table_uploads_for_test() { + CLASSIC_DECODE_TABLE_UPLOADS.store(0, Ordering::Relaxed); +} + +#[cfg(all(test, feature = "cuda-runtime"))] +pub(crate) fn classic_decode_table_uploads_for_test() -> usize { + CLASSIC_DECODE_TABLE_UPLOADS.load(Ordering::Relaxed) +} + impl std::fmt::Debug for CudaSession { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut debug = f.debug_struct("CudaSession"); @@ -197,6 +426,11 @@ impl std::fmt::Debug for CudaSession { &self.htj2k_decode_tables.is_some(), ); #[cfg(feature = "cuda-runtime")] + debug.field( + "classic_decode_tables_cached", + &self.classic_decode_tables.is_some(), + ); + #[cfg(feature = "cuda-runtime")] debug.field( "htj2k_encode_resources_cached", &self.htj2k_encode_resources.is_some(), @@ -216,56 +450,4 @@ impl std::fmt::Debug for CudaSession { } #[cfg(all(test, feature = "cuda-runtime"))] -mod tests { - use super::CudaSession; - use crate::Error; - - fn cuda_required() -> bool { - std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some() - } - - #[test] - fn htj2k_decode_tables_are_uploaded_once_per_session() { - crate::session::reset_htj2k_decode_table_uploads_for_test(); - let mut session = CudaSession::default(); - - let first = session.htj2k_decode_table_resources(); - if matches!( - first, - Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) - ) && !cuda_required() - { - return; - } - first.expect("first HTJ2K decode table upload"); - session - .htj2k_decode_table_resources() - .expect("cached HTJ2K decode tables"); - - assert_eq!(crate::session::htj2k_decode_table_uploads_for_test(), 1); - } - - #[test] - fn cuda_session_reuses_one_decode_buffer_pool_when_required() { - let mut session = CudaSession::default(); - - let first = session.decode_buffer_pool(); - if matches!( - first, - Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) - ) && !cuda_required() - { - return; - } - let first = first.expect("first decode buffer pool"); - let second = session - .decode_buffer_pool() - .expect("cached decode buffer pool"); - { - let buffer = first.take(16).expect("pooled decode buffer"); - assert_eq!(buffer.byte_len(), 16); - } - - assert!(second.cached_count().expect("shared pool cached count") >= 1); - } -} +mod tests; diff --git a/crates/j2k-cuda/src/session/tests.rs b/crates/j2k-cuda/src/session/tests.rs new file mode 100644 index 00000000..b54888aa --- /dev/null +++ b/crates/j2k-cuda/src/session/tests.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::CudaSession; +use crate::Error; + +fn cuda_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some() +} + +#[test] +fn uninitialized_decode_pool_diagnostics_are_empty() { + let diagnostics = CudaSession::default() + .decode_pool_diagnostics() + .expect("empty session diagnostics"); + assert!(diagnostics.decode.is_none()); + assert!(diagnostics.batch_decode.is_none()); + assert_eq!(diagnostics.retained_bytes(), 0); +} + +#[test] +fn htj2k_decode_tables_are_uploaded_once_per_session() { + crate::session::reset_htj2k_decode_table_uploads_for_test(); + let mut session = CudaSession::default(); + + let first = session.htj2k_decode_table_resources(); + if matches!( + first, + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) + ) && !cuda_required() + { + return; + } + first.expect("first HTJ2K decode table upload"); + session + .htj2k_decode_table_resources() + .expect("cached HTJ2K decode tables"); + + assert_eq!(crate::session::htj2k_decode_table_uploads_for_test(), 1); +} + +#[test] +fn classic_decode_tables_are_uploaded_once_per_session() { + crate::session::reset_classic_decode_table_uploads_for_test(); + let mut session = CudaSession::default(); + + let first = session.classic_decode_table_resources(); + if matches!( + first, + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) + ) && !cuda_required() + { + return; + } + first.expect("first classic decode table upload"); + session + .classic_decode_table_resources() + .expect("cached classic decode tables"); + + assert_eq!(crate::session::classic_decode_table_uploads_for_test(), 1); +} + +#[test] +fn cuda_session_reuses_one_decode_buffer_pool_when_required() { + let mut session = CudaSession::default(); + + let first = session.decode_buffer_pool(); + if matches!( + first, + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) + ) && !cuda_required() + { + return; + } + let first = first.expect("first decode buffer pool"); + let second = session + .decode_buffer_pool() + .expect("cached decode buffer pool"); + { + let buffer = first.take(16).expect("pooled decode buffer"); + assert_eq!(buffer.byte_len(), 16); + } + + assert!(second.cached_count().expect("shared pool cached count") >= 1); +} diff --git a/crates/j2k-cuda/src/surface.rs b/crates/j2k-cuda/src/surface.rs index 6549c855..123dd0c5 100644 --- a/crates/j2k-cuda/src/surface.rs +++ b/crates/j2k-cuda/src/surface.rs @@ -62,34 +62,15 @@ impl CudaSurfaceStats { /// Borrowed view of a CUDA-resident surface. #[derive(Clone, Copy, Debug)] pub struct CudaSurface<'a> { - #[cfg(feature = "cuda-runtime")] - buffer: &'a CudaDeviceBuffer, - #[cfg(feature = "cuda-runtime")] - offset: usize, - #[cfg(not(feature = "cuda-runtime"))] + device_ptr: u64, _marker: core::marker::PhantomData<&'a ()>, pub(crate) stats: CudaSurfaceStats, } impl CudaSurface<'_> { /// Raw CUDA device pointer value. - /// - /// # Panics - /// - /// Panics if the internally validated surface range no longer fits in the - /// CUDA `u64` address space, which indicates a construction invariant bug. pub fn device_ptr(&self) -> u64 { - #[cfg(feature = "cuda-runtime")] - { - self.buffer - .device_ptr() - .checked_add(self.offset as u64) - .expect("validated CUDA surface range pointer must fit in u64") - } - #[cfg(not(feature = "cuda-runtime"))] - { - unreachable!("CudaSurface cannot be constructed without cuda-runtime support") - } + self.device_ptr } /// Execution counters for this surface. @@ -168,7 +149,13 @@ impl Surface { len, } => { let byte_len = self.byte_len(); - debug_assert_eq!(*len, byte_len); + if *len < byte_len { + return Err(BufferError::InputTooSmall { + required: byte_len, + have: *len, + } + .into()); + } if let Some(len) = tight_cuda_download_len(byte_len, self.pitch_bytes, stride, out.len()) { @@ -191,15 +178,19 @@ impl Surface { #[cfg(feature = "cuda-runtime")] match &self.storage { Storage::Cuda(buffer) => Some(CudaSurface { - buffer, - offset: 0, - stats: self.stats, - }), - Storage::CudaRange { buffer, offset, .. } => Some(CudaSurface { - buffer, - offset: *offset, + device_ptr: buffer.device_ptr(), + _marker: core::marker::PhantomData, stats: self.stats, }), + Storage::CudaRange { buffer, offset, .. } => { + let offset = u64::try_from(*offset).ok()?; + let device_ptr = buffer.device_ptr().checked_add(offset)?; + Some(CudaSurface { + device_ptr, + _marker: core::marker::PhantomData, + stats: self.stats, + }) + } Storage::Host(_) => None, } #[cfg(not(feature = "cuda-runtime"))] diff --git a/crates/j2k-cuda/tests/batch_decoder_api.rs b/crates/j2k-cuda/tests/batch_decoder_api.rs new file mode 100644 index 00000000..53e6f723 --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[path = "batch_decoder_api/basic_contracts.rs"] +mod basic_contracts; + +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/async_resident.rs"] +mod async_resident; +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/classic_native.rs"] +mod classic_native; +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/exact_color.rs"] +mod exact_color; +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/external_lifecycle.rs"] +mod external_lifecycle; +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/multitile.rs"] +mod multitile; +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/refinement_overlap.rs"] +mod refinement_overlap; +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/rgba.rs"] +mod rgba; +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/session_soak.rs"] +mod session_soak; +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/signed_rgb.rs"] +mod signed_rgb; +#[cfg(feature = "cuda-runtime")] +#[path = "batch_decoder_api/support.rs"] +mod support; diff --git a/crates/j2k-cuda/tests/batch_decoder_api/async_resident.rs b/crates/j2k-cuda/tests/batch_decoder_api/async_resident.rs new file mode 100644 index 00000000..2ffe2579 --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/async_resident.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{prepare_batch, BatchDecodeOptions, BatchLayout, EncodedImage}; +use j2k_cuda::{CudaBatchDecoder, CudaSession, Surface}; +use j2k_cuda_runtime::CudaContext; + +#[test] +fn dropped_resident_submission_retires_work_and_preserves_session_reuse_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-rgb-s12-53-single-raw") + .expect("independent signed RGB12 fixture"); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(fixture.encoded))], + options, + ) + .expect("prepare independent signed RGB12 fixture"); + let context = CudaContext::system_default().expect("CUDA context"); + let session = CudaSession::with_context(context); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + + let dropped = decoder + .submit_prepared(&prepared) + .expect("submit first resident batch"); + assert_eq!(dropped.pending_group_count(), 1); + drop(dropped); + + let completed = decoder + .submit_prepared(&prepared) + .expect("reuse session after dropped submission") + .wait() + .expect("wait reused resident submission"); + let [group] = completed.groups() else { + panic!("expected one completed signed RGB group") + }; + assert_eq!(group.source_indices(), [0]); + let dense = group.dense_output(); + let mut actual = vec![0_u8; fixture.oracle.len()]; + dense + .buffer() + .copy_range_to_host(dense.ranges()[0].offset, &mut actual) + .expect("download completed signed RGB output"); + assert_eq!(actual, fixture.oracle); +} + +#[test] +fn asynchronous_multitile_rgb_and_grayscale_groups_complete_together_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let fixtures = j2k_test_support::openjph_batch_fixtures(); + let rgb = fixtures + .iter() + .find(|fixture| fixture.name == "openjph-rgb-u12-53-raw") + .expect("independent multi-tile RGB fixture"); + let gray_pixels = (0_u8..64).collect::>(); + let supported_gray = Arc::<[u8]>::from( + j2k_native::encode_htj2k( + &gray_pixels, + 8, + 8, + 1, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode supported grayscale fixture"), + ); + // The independent OpenJPH oracle is stored in interleaved sample order. + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![ + EncodedImage::full(Arc::from(rgb.encoded)), + EncodedImage::full(supported_gray), + ], + options, + ) + .expect("prepare heterogeneous async batch"); + assert_eq!(prepared.groups().len(), 2); + + let context = CudaContext::system_default().expect("CUDA context"); + let session = CudaSession::with_context(context); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + let submitted = decoder + .submit_prepared(&prepared) + .expect("submit heterogeneous async batch"); + assert_eq!(submitted.pending_group_count(), 2); + + let completed = submitted.wait().expect("wait heterogeneous async batch"); + assert!(completed.group_errors().is_empty()); + assert_eq!(completed.groups().len(), 2); + let rgb_group = completed + .groups() + .iter() + .find(|group| group.source_indices() == [0]) + .expect("completed multi-tile RGB group"); + let dense = rgb_group.dense_output(); + let mut actual_rgb = vec![0_u8; rgb.oracle.len()]; + dense + .buffer() + .copy_range_to_host(dense.ranges()[0].offset, &mut actual_rgb) + .expect("download completed RGB output"); + assert_eq!(actual_rgb, rgb.oracle); + let gray = completed + .groups() + .iter() + .find(|group| group.source_indices() == [1]) + .expect("completed grayscale group"); + assert_eq!( + Surface::download_batch_tight(gray.surfaces()).expect("download grayscale output"), + gray_pixels + ); +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/basic_contracts.rs b/crates/j2k-cuda/tests/batch_decoder_api/basic_contracts.rs new file mode 100644 index 00000000..5c6e6c3d --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/basic_contracts.rs @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +#[cfg(not(feature = "cuda-runtime"))] +use j2k::DecodeRequest; +use j2k::{ + prepare_batch, BatchDecodeOptions, BatchDecoder, BatchItemError, DecodeSettings, EncodedImage, +}; +#[cfg(not(feature = "cuda-runtime"))] +use j2k_cuda::Error; +use j2k_cuda::{CudaBatchDecodeResult, CudaBatchDecoder, CudaBatchError}; + +#[test] +fn persistent_batch_decoder_accepts_an_empty_batch_without_cuda() { + let mut decoder = CudaBatchDecoder::new(); + + let output = decoder + .decode_batch(Vec::new()) + .expect("empty batch must not initialize CUDA"); + + assert!(output.groups().is_empty()); + assert!(output.errors().is_empty()); + assert!(output.group_errors().is_empty()); + assert_eq!(decoder.session().submissions(), 0); +} + +#[test] +fn cuda_prepared_image_regrouping_reports_indexed_settings_mismatch_without_cuda() { + let lenient_options = BatchDecodeOptions { + settings: DecodeSettings::lenient(), + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from( + j2k_test_support::htj2k_gray8_large_fixture(4, 4), + ))], + lenient_options, + ) + .expect("prepare lenient CUDA input"); + let image = prepared.groups()[0].images()[0].clone(); + let mut decoder = CudaBatchDecoder::with_options(BatchDecodeOptions::default()); + + let regrouped = decoder + .prepare_prepared_images(vec![image.clone()]) + .expect("settings mismatch remains indexed preflight data"); + assert!(regrouped.groups().is_empty()); + assert_eq!(regrouped.errors()[0].index, 0); + assert!(matches!( + regrouped.errors()[0].source, + BatchItemError::PreparedDecodeSettingsMismatch { + prepared, + requested, + } if prepared == DecodeSettings::lenient() && requested == DecodeSettings::strict() + )); + + let output = decoder + .decode_prepared_images(vec![image]) + .expect("all-preflight-error batch must not initialize CUDA"); + assert!(output.groups().is_empty()); + assert_eq!(output.errors()[0].index, 0); + assert_eq!(decoder.session().submissions(), 0); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn asynchronous_resident_decoder_accepts_an_empty_batch_without_cuda() { + let mut decoder = CudaBatchDecoder::new(); + + let submitted = decoder + .submit_batch(Vec::new()) + .expect("empty submission must not initialize CUDA"); + assert_eq!(submitted.pending_group_count(), 0); + assert!(submitted.is_complete().expect("query empty completion")); + + let output = submitted.wait().expect("wait empty submission"); + assert!(output.groups().is_empty()); + assert!(output.errors().is_empty()); + assert!(output.group_errors().is_empty()); + assert_eq!(decoder.session().submissions(), 0); +} + +#[test] +fn persistent_batch_decoder_implements_shared_prepared_contract() { + fn decode_through_trait( + decoder: &mut D, + prepared: &j2k::PreparedBatch, + ) -> Result + where + D: BatchDecoder, + { + decoder.decode_prepared(prepared) + } + + let prepared = prepare_batch(Vec::new(), BatchDecodeOptions::default()) + .expect("prepare empty shared batch"); + let mut decoder = CudaBatchDecoder::new(); + let output = decode_through_trait(&mut decoder, &prepared) + .expect("empty prepared batch must not initialize CUDA"); + + assert!(output.groups().is_empty()); + assert!(output.errors().is_empty()); + assert!(output.group_errors().is_empty()); + assert_eq!(decoder.session().submissions(), 0); +} + +#[cfg(not(feature = "cuda-runtime"))] +#[test] +fn nonempty_batch_fails_closed_without_cuda_runtime() { + let pixels = (0_u8..64).collect::>(); + let encoded = j2k_native::encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode valid HTJ2K batch fixture"); + let input = EncodedImage::new(Arc::from(encoded), DecodeRequest::Full); + let mut decoder = CudaBatchDecoder::new(); + + let error = decoder + .decode_batch(vec![input]) + .expect_err("nonempty CUDA batch requires the CUDA runtime feature"); + let CudaBatchError::GroupExecution { source, .. } = error else { + panic!("expected group execution error") + }; + assert!(matches!(*source, Error::CudaUnavailable)); +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/classic_native.rs b/crates/j2k-cuda/tests/batch_decoder_api/classic_native.rs new file mode 100644 index 00000000..279eba77 --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/classic_native.rs @@ -0,0 +1,416 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{ + prepare_batch, BatchColor, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, + DecodeRequest, EncodedImage, NativeSampleType, Rect, +}; +use j2k_core::Downscale; +use j2k_cuda::{CudaBatchDecoder, CudaBatchGroup, CudaSession, Surface}; +use j2k_cuda_runtime::{CudaContext, CudaExternalDeviceBufferViewMut}; + +#[test] +fn prepared_classic_gray_and_rgb_native_batches_match_cpu_for_all_requests_and_layouts() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { + roi: Rect { + x: 2, + y: 3, + w: 9, + h: 7, + }, + }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi: Rect { + x: 2, + y: 4, + w: 10, + h: 8, + }, + scale: Downscale::Half, + }, + ]; + let context = CudaContext::system_default().expect("CUDA context"); + for channels in [1, 3] { + for encoded in classic_fixtures(channels) { + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + for request in requests { + assert_classic_case(&context, &encoded, layout, request); + } + } + } + } +} + +#[test] +fn prepared_classic_multitile_gray_and_rgb_are_resident_and_external_bit_exact() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let context = CudaContext::system_default().expect("CUDA context"); + let requests = [ + DecodeRequest::Full, + DecodeRequest::RegionReduced { + roi: Rect { + x: 2, + y: 4, + w: 10, + h: 8, + }, + scale: Downscale::Half, + }, + ]; + for channels in [1, 3] { + let encoded = encode_classic_multitile(channels); + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::clone(&encoded))], + BatchDecodeOptions::default(), + ) + .expect("prepare classic multi-tile fixture"); + let native = prepared.groups()[0].images()[0] + .classic_plan() + .expect("prepared classic plan") + .adapter_view() + .downcast_ref::() + .expect("native referenced classic adapter"); + assert!(native.tiles().len() > 1); + + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + for request in requests { + assert_classic_case(&context, &encoded, layout, request); + } + } + } +} + +#[test] +fn classic_irreversible_gray_and_rgb_match_cpu_within_one_lsb_for_all_requests_and_layouts() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { + roi: Rect { + x: 2, + y: 3, + w: 9, + h: 7, + }, + }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi: Rect { + x: 2, + y: 4, + w: 10, + h: 8, + }, + scale: Downscale::Half, + }, + ]; + let context = CudaContext::system_default().expect("CUDA context"); + for channels in [1, 3] { + let encoded = classic_irreversible_fixture(channels); + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + for request in requests { + assert_classic_irreversible_case(&context, &encoded, layout, request); + } + } + } +} + +fn classic_fixtures(channels: u16) -> [Arc<[u8]>; 3] { + let sample_count = 16 * 16 * channels as usize; + let u8_samples = (0..sample_count) + .map(|index| u8::try_from((index * 29 + 7) & 0xff).expect("masked sample fits u8")) + .collect::>(); + let u16_samples = (0..sample_count) + .map(|index| u16::try_from((index * 977 + 31) & 0x0fff).expect("masked sample fits u16")) + .collect::>(); + let i16_samples = (0..sample_count) + .map(|index| { + let index = i32::try_from(index).expect("fixture index fits i32"); + i16::try_from((index * 811 + 19) % 65_536 - 32_768).expect("bounded sample fits i16") + }) + .collect::>(); + [ + encode_classic(&u8_samples, channels, 8, false), + encode_classic(native_bytes(&u16_samples), channels, 12, false), + encode_classic(native_bytes(&i16_samples), channels, 16, true), + ] +} + +fn encode_classic( + samples: impl AsRef<[u8]>, + channels: u16, + bit_depth: u8, + signed: bool, +) -> Arc<[u8]> { + Arc::from( + j2k_native::encode( + samples.as_ref(), + 16, + 16, + channels, + bit_depth, + signed, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + use_mct: channels == 3, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode classic native fixture"), + ) +} + +fn encode_classic_multitile(channels: u16) -> Arc<[u8]> { + let sample_count = 16 * 16 * channels as usize; + let samples = (0..sample_count) + .map(|index| u16::try_from((index * 977 + 31) & 0x0fff).expect("masked sample fits u16")) + .collect::>(); + Arc::from( + j2k_native::encode( + native_bytes(&samples).as_slice(), + 16, + 16, + channels, + 12, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + use_mct: channels == 3, + tile_size: Some((9, 7)), + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode classic multi-tile fixture"), + ) +} + +fn classic_irreversible_fixture(channels: u16) -> Arc<[u8]> { + let sample_count = 16 * 16 * channels as usize; + let samples = (0..sample_count) + .map(|index| { + u8::try_from((index * 47 + index / 7 + 13) & 0xff).expect("masked sample fits u8") + }) + .collect::>(); + Arc::from( + j2k_native::encode( + &samples, + 16, + 16, + channels, + 8, + false, + &j2k_native::EncodeOptions { + reversible: false, + num_decomposition_levels: 2, + use_mct: channels == 3, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode classic irreversible fixture"), + ) +} + +fn native_bytes(samples: &[T]) -> Vec { + // SAFETY: integers have no invalid bit patterns and the returned bytes are copied. + unsafe { + std::slice::from_raw_parts( + samples.as_ptr().cast::(), + std::mem::size_of_val(samples), + ) + } + .to_vec() +} + +fn assert_classic_case( + context: &CudaContext, + encoded: &Arc<[u8]>, + layout: BatchLayout, + request: DecodeRequest, +) { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::new(Arc::clone(encoded), request)], + options, + ) + .expect("prepare classic native fixture"); + assert!(prepared.errors().is_empty()); + let [group] = prepared.groups() else { + panic!("expected one classic native group") + }; + assert!(group.images()[0].classic_plan().is_some()); + assert!(group.images()[0].htj2k_plan().is_none()); + + let mut cpu = CpuBatchDecoder::new(options); + let oracle = cpu + .decode_prepared(&prepared) + .expect("CPU classic native oracle"); + let expected = samples_as_bytes(oracle.groups()[0].samples()); + let alignment = match group.info().sample_type { + NativeSampleType::U8 => 1, + NativeSampleType::U16 | NativeSampleType::I16 => 2, + _ => panic!("unsupported native sample type"), + }; + + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + let mut allocation = context + .allocate(expected.len()) + .expect("classic external destination"); + let submitted = { + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: destination is fresh, exclusive, and retained until wait. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + context, + ptr, + len, + alignment, + &mut allocation, + ) + } + .expect("classic external view"); + // SAFETY: allocation is not accessed while submitted work is pending. + unsafe { decoder.submit_batch_into(group, &mut destination) } + .expect("submit classic external batch") + }; + submitted.wait().expect("wait classic external batch"); + let mut external = vec![0_u8; expected.len()]; + allocation + .copy_to_host(&mut external) + .expect("download classic external output"); + assert_eq!(external, expected, "external {layout:?} {request:?}"); + + let resident = decoder + .decode_prepared(&prepared) + .expect("decode classic resident batch"); + assert!(resident.group_errors().is_empty()); + let actual = download_resident_bytes(&resident.groups()[0], expected.len()); + assert_eq!(actual, expected, "resident {layout:?} {request:?}"); +} + +fn assert_classic_irreversible_case( + context: &CudaContext, + encoded: &Arc<[u8]>, + layout: BatchLayout, + request: DecodeRequest, +) { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::new(Arc::clone(encoded), request)], + options, + ) + .expect("prepare classic irreversible fixture"); + assert!(prepared.errors().is_empty()); + let [group] = prepared.groups() else { + panic!("expected one classic irreversible group") + }; + assert!(group.images()[0].classic_plan().is_some()); + + let mut cpu = CpuBatchDecoder::new(options); + let oracle = cpu + .decode_prepared(&prepared) + .expect("CPU classic irreversible oracle"); + let CpuBatchSamples::U8(expected) = oracle.groups()[0].samples() else { + panic!("classic irreversible fixture must use U8 storage") + }; + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + let mut allocation = context + .allocate(expected.len()) + .expect("classic irreversible external destination"); + let submitted = { + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: destination is fresh, exclusive, and retained until wait. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts(context, ptr, len, 1, &mut allocation) + } + .expect("classic irreversible external view"); + // SAFETY: allocation is not accessed while submitted work is pending. + unsafe { decoder.submit_batch_into(group, &mut destination) } + .expect("submit classic irreversible external batch") + }; + submitted + .wait() + .expect("wait classic irreversible external batch"); + let mut external = vec![0_u8; expected.len()]; + allocation + .copy_to_host(&mut external) + .expect("download classic irreversible external output"); + assert_within_one_lsb(&external, expected, "external", layout, request); + + let resident = decoder + .decode_prepared(&prepared) + .expect("decode classic irreversible resident batch"); + let actual = download_resident_bytes(&resident.groups()[0], expected.len()); + assert_within_one_lsb(&actual, expected, "resident", layout, request); +} + +fn download_resident_bytes(group: &CudaBatchGroup, expected_len: usize) -> Vec { + if group.info().color == BatchColor::Gray { + return Surface::download_batch_tight(group.surfaces()) + .expect("download classic grayscale resident surfaces"); + } + let dense = group.dense_output(); + let mut actual = vec![0_u8; expected_len]; + dense + .buffer() + .copy_range_to_host(dense.ranges()[0].offset, &mut actual) + .expect("download classic color resident output"); + actual +} + +fn assert_within_one_lsb( + actual: &[u8], + expected: &[u8], + route: &str, + layout: BatchLayout, + request: DecodeRequest, +) { + assert_eq!(actual.len(), expected.len()); + for (index, (&actual, &expected)) in actual.iter().zip(expected).enumerate() { + assert!( + actual.abs_diff(expected) <= 1, + "{route} {layout:?} {request:?} sample {index}: actual={actual}, expected={expected}" + ); + } +} + +fn samples_as_bytes(samples: &CpuBatchSamples) -> Vec { + match samples { + CpuBatchSamples::U8(samples) => samples.clone(), + CpuBatchSamples::U16(samples) => samples + .iter() + .flat_map(|sample| sample.to_ne_bytes()) + .collect(), + CpuBatchSamples::I16(samples) => samples + .iter() + .flat_map(|sample| sample.to_ne_bytes()) + .collect(), + other => panic!("unsupported classic oracle type: {other:?}"), + } +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/exact_color.rs b/crates/j2k-cuda/tests/batch_decoder_api/exact_color.rs new file mode 100644 index 00000000..99770ede --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/exact_color.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{ + prepare_batch, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, + DecodeRequest, Downscale, EncodedImage, Rect, +}; +use j2k_cuda::{CudaBatchDecoder, CudaSession, Surface}; +use j2k_cuda_runtime::{CudaContext, CudaExternalDeviceBufferViewMut}; + +#[test] +fn exact_rgb_external_and_resident_batches_match_cpu_for_geometry_and_layout_when_runtime_required() +{ + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { + roi: Rect { + x: 2, + y: 3, + w: 9, + h: 7, + }, + }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi: Rect { + x: 2, + y: 4, + w: 10, + h: 8, + }, + scale: Downscale::Half, + }, + ]; + let context = CudaContext::system_default().expect("CUDA context"); + + for (encoded, expected_u16) in exact_rgb_fixtures() { + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + for request in requests { + assert_exact_rgb_case(&context, &encoded, expected_u16, layout, request); + } + } + } +} + +fn exact_rgb_fixtures() -> [(Arc<[u8]>, bool); 2] { + let encoded_u8 = Arc::<[u8]>::from( + j2k_native::encode_htj2k( + &(0_u32..16 * 16 * 3) + .map(|value| ((value * 29 + 7) & 0x7f) as u8) + .collect::>(), + 16, + 16, + 3, + 7, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode RGB7 HTJ2K"), + ); + let samples_u16 = (0_u32..16 * 16 * 3) + .map(|value| ((value * 977 + 31) & 0x0fff) as u16) + .collect::>(); + // SAFETY: the encoder copies this native integer fixture immediately. + let samples_u16_bytes = unsafe { + std::slice::from_raw_parts( + samples_u16.as_ptr().cast::(), + std::mem::size_of_val(samples_u16.as_slice()), + ) + }; + let encoded_u16 = Arc::<[u8]>::from( + j2k_native::encode_htj2k( + samples_u16_bytes, + 16, + 16, + 3, + 12, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode RGB12 HTJ2K"), + ); + [(encoded_u8, false), (encoded_u16, true)] +} + +fn assert_exact_rgb_case( + context: &CudaContext, + encoded: &Arc<[u8]>, + expected_u16: bool, + layout: BatchLayout, + request: DecodeRequest, +) { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::new(Arc::clone(encoded), request)], + options, + ) + .expect("prepare exact RGB group"); + assert!(prepared.errors().is_empty()); + let [prepared_group] = prepared.groups() else { + panic!("expected one exact RGB group") + }; + assert!(prepared_group.images()[0].htj2k_plan().is_some()); + + let mut cpu = CpuBatchDecoder::new(options); + let oracle = cpu + .decode_prepared(&prepared) + .expect("CPU exact RGB oracle"); + let expected = match oracle.groups()[0].samples() { + CpuBatchSamples::U8(samples) => samples.clone(), + CpuBatchSamples::U16(samples) => samples + .iter() + .flat_map(|sample| sample.to_ne_bytes()) + .collect(), + other => panic!("unexpected exact RGB oracle type: {other:?}"), + }; + assert_eq!( + expected_u16, + prepared_group.info().sample_type == j2k::NativeSampleType::U16 + ); + + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + assert_exact_external( + context, + &mut decoder, + prepared_group, + &expected, + expected_u16, + layout, + request, + ); + assert_exact_resident(&mut decoder, &prepared, &expected, layout, request); +} + +fn assert_exact_external( + context: &CudaContext, + decoder: &mut CudaBatchDecoder, + group: &j2k::PreparedBatchGroup, + expected: &[u8], + expected_u16: bool, + layout: BatchLayout, + request: DecodeRequest, +) { + let mut allocation = context + .allocate(expected.len()) + .expect("external exact RGB destination"); + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: the allocation remains live and exclusively owned until the + // returned completion owner is waited below. + let submitted = unsafe { + let mut destination = CudaExternalDeviceBufferViewMut::from_raw_parts( + context, + ptr, + len, + if expected_u16 { 2 } else { 1 }, + &mut allocation, + ) + .expect("exact RGB external view"); + decoder + .submit_batch_into(group, &mut destination) + .expect("submit exact RGB external batch") + }; + let external = submitted.wait().expect("wait exact RGB external batch"); + assert_eq!(external.ranges().len(), 1); + let mut actual = vec![0_u8; expected.len()]; + allocation + .copy_to_host(&mut actual) + .expect("download exact RGB external output"); + assert_eq!(actual, expected, "{layout:?} {request:?}"); +} + +fn assert_exact_resident( + decoder: &mut CudaBatchDecoder, + prepared: &j2k::PreparedBatch, + expected: &[u8], + layout: BatchLayout, + request: DecodeRequest, +) { + let resident = decoder + .decode_prepared(prepared) + .expect("decode exact RGB resident batch"); + let [resident_group] = resident.groups() else { + panic!("expected one resident RGB group") + }; + let dense = resident_group.dense_output(); + assert_eq!(dense.ranges().len(), 1); + let mut dense_bytes = vec![0_u8; expected.len()]; + dense + .buffer() + .copy_range_to_host(dense.ranges()[0].offset, &mut dense_bytes) + .expect("download dense exact RGB output"); + assert_eq!(dense_bytes, expected, "dense {layout:?} {request:?}"); + if layout == BatchLayout::Nhwc { + assert_eq!(resident_group.surfaces().len(), 1); + assert_eq!( + Surface::download_batch_tight(resident_group.surfaces()) + .expect("download NHWC surface"), + expected + ); + } else { + assert!(resident_group.surfaces().is_empty()); + } +} + +#[test] +fn irreversible_rgb8_external_and_resident_outputs_match_cpu_within_one_lsb_when_runtime_required() +{ + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let pixels = (0_u32..16 * 16 * 3) + .map(|value| ((value * 47 + value / 7 + 13) & 0xff) as u8) + .collect::>(); + let encoded = Arc::<[u8]>::from( + j2k_native::encode_htj2k( + &pixels, + 16, + 16, + 3, + 8, + false, + &j2k_native::EncodeOptions { + reversible: false, + num_decomposition_levels: 2, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode irreversible RGB8 HTJ2K fixture"), + ); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch(vec![EncodedImage::full(encoded)], options) + .expect("prepare irreversible RGB8 CUDA group"); + let [group] = prepared.groups() else { + panic!("expected one irreversible RGB8 group") + }; + assert!(group.images()[0].htj2k_plan().is_some()); + + let mut cpu = CpuBatchDecoder::new(options); + let oracle = cpu + .decode_prepared(&prepared) + .expect("decode irreversible RGB8 CPU oracle"); + let expected = match oracle.groups()[0].samples() { + CpuBatchSamples::U8(samples) => samples, + other => panic!("expected U8 CPU oracle, got {other:?}"), + }; + + let context = CudaContext::system_default().expect("CUDA context"); + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + let mut allocation = context + .allocate(expected.len()) + .expect("irreversible external destination"); + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: the allocation remains live and inaccessible until completion. + let submitted = unsafe { + let mut destination = CudaExternalDeviceBufferViewMut::from_raw_parts( + &context, + ptr, + len, + std::mem::align_of::(), + &mut allocation, + ) + .expect("irreversible external view"); + decoder + .submit_batch_into(group, &mut destination) + .expect("submit irreversible RGB8 external batch") + }; + submitted + .wait() + .expect("wait irreversible RGB8 external batch"); + let mut external = vec![0_u8; expected.len()]; + allocation + .copy_to_host(&mut external) + .expect("download irreversible RGB8 external output"); + assert_u8_within_one_lsb(&external, expected, "external"); + + let resident = decoder + .decode_prepared(&prepared) + .expect("decode irreversible RGB8 resident batch"); + let [resident_group] = resident.groups() else { + panic!("expected one irreversible resident group") + }; + let dense = resident_group.dense_output(); + let mut resident_bytes = vec![0_u8; expected.len()]; + dense + .buffer() + .copy_range_to_host(dense.ranges()[0].offset, &mut resident_bytes) + .expect("download irreversible RGB8 resident output"); + assert_u8_within_one_lsb(&resident_bytes, expected, "resident"); +} + +fn assert_u8_within_one_lsb(actual: &[u8], expected: &[u8], route: &str) { + assert_eq!(actual.len(), expected.len(), "{route} output length"); + for (index, (&actual, &expected)) in actual.iter().zip(expected).enumerate() { + assert!( + actual.abs_diff(expected) <= 1, + "{route} sample {index}: actual={actual}, expected={expected}" + ); + } +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/external_lifecycle.rs b/crates/j2k-cuda/tests/batch_decoder_api/external_lifecycle.rs new file mode 100644 index 00000000..b139ffcb --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/external_lifecycle.rs @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{ + prepare_batch, BatchDecodeOptions, CpuBatchDecoder, CpuBatchSamples, DecodeRequest, Downscale, + EncodedImage, PreparedBatchGroup, Rect, +}; +use j2k_cuda::{CudaBatchDecoder, CudaExternalBatchTryFinish, CudaSession}; +use j2k_cuda_runtime::{CudaContext, CudaExternalDeviceBufferViewMut}; + +use super::support::submit_external_for_test; + +#[test] +fn external_batch_handles_roi_reduction_and_drop_safe_session_reuse_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + + let pixels = (0_u8..=255).collect::>(); + let encoded = Arc::<[u8]>::from( + j2k_native::encode_htj2k( + &pixels, + 16, + 16, + 1, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode HTJ2K ROI/reduction fixture"), + ); + let inputs = vec![ + EncodedImage::new( + Arc::clone(&encoded), + DecodeRequest::Region { + roi: Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }, + }, + ), + EncodedImage::new( + encoded, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + ), + ]; + let options = BatchDecodeOptions::default(); + let prepared = prepare_batch(inputs, options).expect("prepare shared batch"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 2); + assert_eq!(prepared.groups()[0].source_indices(), &[0]); + assert_eq!(prepared.groups()[1].source_indices(), &[1]); + + let mut cpu = CpuBatchDecoder::new(options); + let cpu_output = cpu + .decode_prepared(&prepared) + .expect("CPU oracle batch decode"); + assert_eq!(cpu_output.groups().len(), prepared.groups().len()); + + let context = CudaContext::system_default().expect("CUDA context"); + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + for (prepared_group, oracle_group) in prepared.groups().iter().zip(cpu_output.groups().iter()) { + assert_eq!( + prepared_group.source_indices(), + oracle_group.source_indices() + ); + let expected = match oracle_group.samples() { + CpuBatchSamples::U8(samples) => samples, + samples => panic!("expected U8 CPU oracle, got {samples:?}"), + }; + assert_drop_retirement_and_reuse(&context, &mut decoder, prepared_group, expected); + } +} + +fn assert_drop_retirement_and_reuse( + context: &CudaContext, + decoder: &mut CudaBatchDecoder, + prepared_group: &PreparedBatchGroup, + expected: &[u8], +) { + let mut allocation = context + .allocate(expected.len()) + .expect("external destination allocation"); + + // The first submission is intentionally dropped unfinished. Drop must + // retire its resources before the persistent session is reused. + { + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: `allocation` is exclusively borrowed through submission. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + context, + ptr, + len, + std::mem::align_of::(), + &mut allocation, + ) + } + .expect("external destination view"); + // SAFETY: dropping the completion owner retires work before reuse. + let submitted = unsafe { decoder.submit_batch_into(prepared_group, &mut destination) } + .expect("submit external batch for drop retirement"); + drop(submitted); + } + + let submitted = { + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: the allocation remains live and inaccessible until the + // returned completion owner is retired below. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + context, + ptr, + len, + std::mem::align_of::(), + &mut allocation, + ) + } + .expect("reused external destination view"); + // SAFETY: there is no overlapping access before completion. + unsafe { decoder.submit_batch_into(prepared_group, &mut destination) } + .expect("submit external batch after drop retirement") + }; + let group = match submitted.try_finish().expect("poll external batch") { + CudaExternalBatchTryFinish::Pending(submitted) => { + submitted.wait().expect("wait external batch") + } + CudaExternalBatchTryFinish::Complete(group) => group, + }; + assert_eq!(group.source_indices(), prepared_group.source_indices()); + assert_eq!(group.ranges().len(), 1); + + let mut actual = vec![0_u8; expected.len()]; + allocation + .copy_to_host(&mut actual) + .expect("download completed external output"); + assert_eq!(actual, expected); +} + +#[test] +fn mixed_decomposition_shapes_keep_async_pool_resources_live_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + + let pixels_a = (0_u8..=255).collect::>(); + let pixels_b = pixels_a + .iter() + .map(|value| value.wrapping_mul(29).wrapping_add(7)) + .collect::>(); + let encode = |pixels: &[u8], levels| { + Arc::<[u8]>::from( + j2k_native::encode_htj2k( + pixels, + 16, + 16, + 1, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: levels, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode mixed-decomposition HTJ2K fixture"), + ) + }; + let prepared = prepare_batch( + vec![ + EncodedImage::full(encode(&pixels_a, 1)), + EncodedImage::full(encode(&pixels_b, 2)), + ], + BatchDecodeOptions::default(), + ) + .expect("prepare mixed-decomposition batch"); + assert_eq!(prepared.groups().len(), 2); + assert_eq!(prepared.groups()[0].source_indices(), [0]); + assert_eq!(prepared.groups()[1].source_indices(), [1]); + + let mut cpu = CpuBatchDecoder::new(BatchDecodeOptions::default()); + let oracle = cpu + .decode_prepared(&prepared) + .expect("CPU mixed-decomposition oracle"); + let expected = oracle + .groups() + .iter() + .map(|group| match group.samples() { + CpuBatchSamples::U8(samples) => samples, + samples => panic!("expected U8 CPU oracle, got {samples:?}"), + }) + .collect::>(); + + let context = CudaContext::system_default().expect("CUDA context"); + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session(session); + let mut first_output = context + .allocate(expected[0].len()) + .expect("first mixed-shape output"); + let mut second_output = context + .allocate(expected[1].len()) + .expect("second mixed-shape output"); + // SAFETY: both allocations remain live and untouched until both pending + // owners are waited below. Distinct destinations do not overlap. + let first = unsafe { + submit_external_for_test( + &mut decoder, + &prepared.groups()[0], + &context, + &mut first_output, + ) + }; + // Submit again before waiting the first batch. This catches recycling of + // IDWT scratch still referenced by the first default-stream graph. + // SAFETY: same lifetime guarantee as the first distinct allocation. + let second = unsafe { + submit_external_for_test( + &mut decoder, + &prepared.groups()[1], + &context, + &mut second_output, + ) + }; + second.wait().expect("wait second mixed-shape batch"); + first.wait().expect("wait first mixed-shape batch"); + + let mut first_actual = vec![0_u8; expected[0].len()]; + let mut second_actual = vec![0_u8; expected[1].len()]; + first_output + .copy_to_host(&mut first_actual) + .expect("download first mixed-shape batch"); + second_output + .copy_to_host(&mut second_actual) + .expect("download second mixed-shape batch"); + assert_eq!(first_actual, *expected[0]); + assert_eq!(second_actual, *expected[1]); +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/multitile.rs b/crates/j2k-cuda/tests/batch_decoder_api/multitile.rs new file mode 100644 index 00000000..4535de98 --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/multitile.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{prepare_batch, BatchDecodeOptions, BatchLayout, EncodedImage}; +use j2k_cuda::{CudaBatchDecoder, CudaSession, Surface}; +use j2k_cuda_runtime::CudaContext; + +use super::support::submit_external_for_test; + +#[test] +fn independent_openjph_multitile_rgb_retains_every_prepared_tile_without_cuda() { + let encoded = openjph_multitile_rgb_u12(); + let prepared = prepare_batch( + vec![EncodedImage::full(encoded)], + BatchDecodeOptions::default(), + ) + .expect("prepare independent OpenJPH RGB fixture"); + let [group] = prepared.groups() else { + panic!("expected one OpenJPH RGB group") + }; + let native = group.images()[0] + .htj2k_plan() + .expect("prepared multi-tile HTJ2K plan") + .adapter_view() + .downcast_ref::() + .expect("native referenced HTJ2K adapter"); + assert_eq!(native.tiles().len(), 4); +} + +#[test] +fn independent_openjph_multitile_gray_and_rgb_are_resident_and_external_bit_exact_when_runtime_required( +) { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let fixtures = j2k_test_support::openjph_batch_fixtures(); + let rgb = fixtures + .iter() + .find(|fixture| fixture.name == "openjph-rgb-u12-53-raw") + .expect("independent multi-tile RGB12 fixture"); + let gray = fixtures + .iter() + .find(|fixture| fixture.name == "openjph-gray-u12-53-raw") + .expect("independent multi-tile Gray12 fixture"); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![ + EncodedImage::full(Arc::from(rgb.encoded)), + EncodedImage::full(Arc::from(gray.encoded)), + ], + options, + ) + .expect("prepare heterogeneous multi-tile CUDA batch"); + assert_eq!(prepared.groups().len(), 2); + + let context = CudaContext::system_default().expect("CUDA context"); + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + for group in prepared.groups() { + let expected = match group.source_indices() { + [0] => rgb.oracle, + [1] => gray.oracle, + other => panic!("unexpected source indices: {other:?}"), + }; + let mut allocation = context + .allocate(expected.len()) + .expect("multi-tile external destination"); + // SAFETY: `allocation` remains live and inaccessible until `wait`. + let submitted = + unsafe { submit_external_for_test(&mut decoder, group, &context, &mut allocation) }; + let external = submitted.wait().expect("wait multi-tile external decode"); + assert_eq!(external.ranges().len(), 1); + let mut actual = vec![0_u8; expected.len()]; + allocation + .copy_to_host(&mut actual) + .expect("download multi-tile external output"); + assert_eq!(actual, expected); + } + + let output = decoder + .decode_prepared(&prepared) + .expect("decode multi-tile resident batch"); + assert!(output.group_errors().is_empty()); + assert_eq!(output.groups().len(), 2); + for group in output.groups() { + let (expected, is_color) = match group.source_indices() { + [0] => (rgb.oracle, true), + [1] => (gray.oracle, false), + other => panic!("unexpected resident source indices: {other:?}"), + }; + let dense = group.dense_output(); + assert_eq!(dense.ranges().len(), 1); + let mut actual = vec![0_u8; expected.len()]; + dense + .buffer() + .copy_range_to_host(dense.ranges()[0].offset, &mut actual) + .expect("download multi-tile resident output"); + assert_eq!(actual, expected); + if !is_color { + assert_eq!(group.surfaces().len(), 1); + let surface_actual = Surface::download_batch_tight(group.surfaces()) + .expect("download multi-tile resident grayscale output"); + assert_eq!(surface_actual, expected); + } + } +} + +fn openjph_multitile_rgb_u12() -> Arc<[u8]> { + Arc::from( + j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-rgb-u12-53-raw") + .expect("checked-in OpenJPH RGB12 fixture") + .encoded, + ) +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/refinement_overlap.rs b/crates/j2k-cuda/tests/batch_decoder_api/refinement_overlap.rs new file mode 100644 index 00000000..89ea43e1 --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/refinement_overlap.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{prepare_batch, BatchDecodeOptions, BatchLayout, EncodedImage}; +use j2k_cuda::{CudaBatchDecoder, CudaSession}; +use j2k_cuda_runtime::{CudaContext, CudaExternalDeviceBufferViewMut}; + +#[test] +fn independent_sigprop_magref_overlap_matches_openhtj2k_for_resident_and_external_output() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let expected = j2k_test_support::openhtj2k_sigprop_overlap_pixels(); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from( + j2k_test_support::openhtj2k_sigprop_overlap_fixture(), + ))], + options, + ) + .expect("prepare independent refinement-overlap fixture"); + let [group] = prepared.groups() else { + panic!("expected one refinement-overlap group") + }; + assert!(group.images()[0].htj2k_plan().is_some()); + + let context = CudaContext::system_default().expect("CUDA context"); + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + let mut allocation = context + .allocate(expected.len()) + .expect("refinement external destination"); + let submitted = { + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: destination is exclusive and retained through completion. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts(&context, ptr, len, 1, &mut allocation) + } + .expect("refinement external view"); + // SAFETY: allocation is not read until the pending owner is waited. + unsafe { decoder.submit_batch_into(group, &mut destination) } + .expect("submit refinement external decode") + }; + submitted.wait().expect("wait refinement external decode"); + let mut external = vec![0_u8; expected.len()]; + allocation + .copy_to_host(&mut external) + .expect("download refinement external output"); + assert_within_one_lsb(&external, expected, "external"); + + let resident = decoder + .decode_prepared(&prepared) + .expect("decode refinement resident output"); + let dense = resident.groups()[0].dense_output(); + let mut actual = vec![0_u8; expected.len()]; + dense + .buffer() + .copy_range_to_host(dense.ranges()[0].offset, &mut actual) + .expect("download refinement resident output"); + assert_within_one_lsb(&actual, expected, "resident"); +} + +fn assert_within_one_lsb(actual: &[u8], expected: &[u8], route: &str) { + assert_eq!(actual.len(), expected.len(), "{route} length"); + for (index, (&actual, &expected)) in actual.iter().zip(expected).enumerate() { + assert!( + actual.abs_diff(expected) <= 1, + "{route} sample {index}: actual={actual}, expected={expected}" + ); + } +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/rgba.rs b/crates/j2k-cuda/tests/batch_decoder_api/rgba.rs new file mode 100644 index 00000000..7dc01835 --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/rgba.rs @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{ + prepare_batch, wrap_j2k_codestream, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, + CpuBatchSamples, DecodeRequest, EncodedImage, J2kChannelAssociation, J2kChannelDefinition, + J2kChannelType, J2kFileBoxMetadata, J2kFileColorSpec, J2kFileWrapOptions, NativeSampleType, + Rect, +}; +use j2k_core::{Colorspace, Downscale}; +use j2k_cuda::{CudaBatchDecoder, CudaSession}; +use j2k_cuda_runtime::{CudaContext, CudaExternalDeviceBufferViewMut}; +use j2k_test_support::{ + generated_htj2k_rgba_fixture, Htj2kRgbaAlpha, Htj2kRgbaFixture, Htj2kRgbaSampleProfile, + Htj2kRgbaSamples, +}; + +#[test] +fn exact_rgba_resident_and_external_match_cpu_across_codecs_types_geometry_and_layouts() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { + roi: Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }, + }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi: Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }, + scale: Downscale::Half, + }, + ]; + let context = CudaContext::system_default().expect("CUDA context"); + for profile in [ + Htj2kRgbaSampleProfile::U8Rct, + Htj2kRgbaSampleProfile::U12, + Htj2kRgbaSampleProfile::I16, + ] { + let fixture = generated_htj2k_rgba_fixture(profile, Htj2kRgbaAlpha::Straight); + let encodings = [ + ("HTJ2K", Arc::<[u8]>::from(wrap_rgba_jph(&fixture))), + ( + "classic JPEG 2000", + Arc::<[u8]>::from(wrap_classic_rgba_jp2(&fixture)), + ), + ]; + for (codec, encoded) in encodings { + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + for request in requests { + assert_one_case(&context, codec, &encoded, layout, request); + } + } + } + } +} + +fn assert_one_case( + context: &CudaContext, + codec: &str, + encoded: &Arc<[u8]>, + layout: BatchLayout, + request: DecodeRequest, +) { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::new(Arc::clone(encoded), request)], + options, + ) + .unwrap_or_else(|error| panic!("prepare {codec} RGBA: {error}")); + assert!(prepared.errors().is_empty(), "{codec} preflight errors"); + let [group] = prepared.groups() else { + panic!("expected one {codec} RGBA group") + }; + if codec == "HTJ2K" { + assert!(group.images()[0].htj2k_plan().is_some()); + } else { + assert!(group.images()[0].classic_plan().is_some()); + } + + let mut cpu = CpuBatchDecoder::new(options); + let oracle = cpu + .decode_prepared(&prepared) + .unwrap_or_else(|error| panic!("CPU {codec} RGBA oracle: {error}")); + let expected = samples_as_bytes(oracle.groups()[0].samples()); + + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + let mut allocation = context + .allocate(expected.len()) + .expect("RGBA external destination"); + let submitted = { + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + let alignment = match group.info().sample_type { + NativeSampleType::U8 => 1, + NativeSampleType::U16 | NativeSampleType::I16 => 2, + _ => panic!("unsupported RGBA sample type"), + }; + // SAFETY: the allocation is exclusively borrowed until submission + // completion and is not read while codec work is pending. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + context, + ptr, + len, + alignment, + &mut allocation, + ) + } + .expect("RGBA external view"); + // SAFETY: the destination owner remains live and inaccessible until wait. + unsafe { decoder.submit_batch_into(group, &mut destination) } + .unwrap_or_else(|error| panic!("submit external {codec} RGBA: {error}")) + }; + submitted + .wait() + .unwrap_or_else(|error| panic!("wait external {codec} RGBA: {error}")); + let mut external = vec![0_u8; expected.len()]; + allocation + .copy_to_host(&mut external) + .expect("download external RGBA"); + assert_eq!( + external, expected, + "external {codec} {layout:?} {request:?}" + ); + + let resident = decoder + .decode_prepared(&prepared) + .unwrap_or_else(|error| panic!("decode resident {codec} RGBA: {error}")); + assert!( + resident.group_errors().is_empty(), + "{codec} resident errors" + ); + let dense = resident.groups()[0].dense_output(); + let mut actual = vec![0_u8; expected.len()]; + dense + .buffer() + .copy_range_to_host(dense.ranges()[0].offset, &mut actual) + .expect("download resident RGBA"); + assert_eq!(actual, expected, "resident {codec} {layout:?} {request:?}"); +} + +fn samples_as_bytes(samples: &CpuBatchSamples) -> Vec { + match samples { + CpuBatchSamples::U8(samples) => samples.clone(), + CpuBatchSamples::U16(samples) => samples + .iter() + .flat_map(|sample| sample.to_ne_bytes()) + .collect(), + CpuBatchSamples::I16(samples) => samples + .iter() + .flat_map(|sample| sample.to_ne_bytes()) + .collect(), + other => panic!("unsupported RGBA oracle type: {other:?}"), + } +} + +fn wrap_rgba_jph(fixture: &Htj2kRgbaFixture) -> Vec { + wrap_rgba_container(&fixture.encoded, fixture.alpha, J2kFileWrapOptions::jph()) +} + +fn wrap_classic_rgba_jp2(fixture: &Htj2kRgbaFixture) -> Vec { + let pixels = samples_as_encoded_bytes(&fixture.samples); + let codestream = j2k_native::encode( + &pixels, + fixture.width, + fixture.height, + 4, + fixture.bit_depth, + fixture.signed, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + use_mct: fixture.use_mct, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode classic RGBA fixture"); + wrap_rgba_container(&codestream, fixture.alpha, J2kFileWrapOptions::jp2()) +} + +fn samples_as_encoded_bytes(samples: &Htj2kRgbaSamples) -> Vec { + match samples { + Htj2kRgbaSamples::U8(samples) => samples.clone(), + Htj2kRgbaSamples::U16(samples) => samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect(), + Htj2kRgbaSamples::I16(samples) => samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect(), + } +} + +fn wrap_rgba_container( + codestream: &[u8], + alpha: Htj2kRgbaAlpha, + options: J2kFileWrapOptions<'_>, +) -> Vec { + let alpha_type = match alpha { + Htj2kRgbaAlpha::Straight => J2kChannelType::Opacity, + Htj2kRgbaAlpha::Premultiplied => J2kChannelType::PremultipliedOpacity, + }; + let definitions = [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 3, + channel_type: alpha_type, + association: J2kChannelAssociation::WholeImage, + }, + ]; + wrap_j2k_codestream( + codestream, + options + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &definitions, + }), + ) + .expect("wrap RGBA container") +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/session_soak.rs b/crates/j2k-cuda/tests/batch_decoder_api/session_soak.rs new file mode 100644 index 00000000..95648d2a --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/session_soak.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{BatchDecodeOptions, BatchLayout, EncodedImage}; +use j2k_cuda::{CudaBatchDecoder, CudaSession}; +use j2k_cuda_runtime::CudaContext; + +use super::support::submit_external_for_test; + +#[test] +#[expect( + clippy::too_many_lines, + reason = "one soak scenario audits pool, transfer, event, synchronization, and output invariants together" +)] +fn resident_and_external_decode_pools_stabilize_for_one_thousand_batches_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-rgb-s12-53-single-raw") + .expect("independent signed RGB12 single-tile fixture"); + let encoded = Arc::<[u8]>::from(fixture.encoded); + let inputs = (0..8) + .map(|_| EncodedImage::full(Arc::clone(&encoded))) + .collect::>(); + let context = CudaContext::system_default().expect("CUDA context"); + let session = CudaSession::with_context(context.clone()); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + let prepared = decoder + .prepare(inputs) + .expect("prepare reusable RGB12 batch"); + let [group] = prepared.groups() else { + panic!("expected one homogeneous signed RGB12 group") + }; + let external_bytes = fixture + .oracle + .len() + .checked_mul(8) + .expect("external batch byte length"); + let mut external = context + .allocate(external_bytes) + .expect("persistent external destination"); + + for _ in 0..16 { + drop( + decoder + .decode_prepared(&prepared) + .expect("warm persistent CUDA batch pools"), + ); + // SAFETY: the allocation remains live and inaccessible until wait. + unsafe { submit_external_for_test(&mut decoder, group, &context, &mut external) } + .wait() + .expect("warm persistent external CUDA batch pools"); + } + let warm = decoder + .diagnostics() + .expect("warm CUDA session diagnostics"); + let warm_runtime = warm.runtime.expect("CUDA runtime must be initialized"); + let warm_batch = warm + .pools + .batch_decode + .expect("dense batch decode pool must be initialized"); + assert_eq!(warm_batch.deferred_bytes, 0); + assert_eq!(warm_batch.reuse_holds, 0); + assert!(warm.pools.retained_bytes() > 0); + + for _ in 0..1_000 { + drop( + decoder + .decode_prepared(&prepared) + .expect("reuse prepared CUDA batch"), + ); + } + for _ in 0..1_000 { + // SAFETY: the allocation remains live and inaccessible until wait. + unsafe { submit_external_for_test(&mut decoder, group, &context, &mut external) } + .wait() + .expect("reuse external CUDA destination"); + } + + let after = decoder + .diagnostics() + .expect("post-soak CUDA session diagnostics"); + let after_runtime = after.runtime.expect("CUDA runtime remains initialized"); + let after_batch = after + .pools + .batch_decode + .expect("batch pool remains initialized"); + assert_eq!(after_batch.deferred_bytes, 0); + assert_eq!(after_batch.reuse_holds, 0); + assert_eq!(after.pools.retained_bytes(), warm.pools.retained_bytes()); + assert_eq!( + after.pools.peak_retained_bytes_upper_bound(), + warm.pools.peak_retained_bytes_upper_bound() + ); + assert_eq!( + after_runtime.status_device_to_host_operations + - warm_runtime.status_device_to_host_operations, + 2_000, + "each resident/external group must use one status readback" + ); + assert_eq!( + after_runtime.device_to_host_operations - warm_runtime.device_to_host_operations, + 2_000, + "the soak must not download decoded pixels" + ); + assert_eq!( + after_runtime.device_to_host_bytes - warm_runtime.device_to_host_bytes, + after_runtime.status_device_to_host_bytes - warm_runtime.status_device_to_host_bytes, + "all pre-verification D2H bytes must be the one small status transfer" + ); + assert_eq!( + after_runtime.event_driver_allocations, warm_runtime.event_driver_allocations, + "completion events must stabilize after warmup" + ); + assert!(after_runtime.event_reuses > warm_runtime.event_reuses); + assert_eq!( + after_runtime.event_host_synchronizations, warm_runtime.event_host_synchronizations, + "the group status readback must replace a second final-store event wait" + ); + assert_eq!( + after_runtime.context_host_synchronizations, warm_runtime.context_host_synchronizations, + "normal batch completion must not synchronize the whole context" + ); + + let mut actual = vec![0_u8; external_bytes]; + external + .copy_to_host(&mut actual) + .expect("download final external soak output"); + for image in actual.chunks_exact(fixture.oracle.len()) { + assert_eq!(image, fixture.oracle); + } +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/signed_rgb.rs b/crates/j2k-cuda/tests/batch_decoder_api/signed_rgb.rs new file mode 100644 index 00000000..7e845fe4 --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/signed_rgb.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{ + prepare_batch, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, + DecodeRequest, Downscale, EncodedImage, Rect, +}; +use j2k_cuda::{CudaBatchDecoder, CudaSession}; +use j2k_cuda_runtime::{CudaContext, CudaExternalDeviceBufferViewMut}; +use j2k_test_support::OpenJphBatchFixture; + +#[test] +#[expect( + clippy::too_many_lines, + reason = "the CUDA-only matrix keeps independent-fixture, external, and resident parity together" +)] +fn signed_rgb_i16_external_and_resident_batches_match_cpu_for_geometry_and_layout_when_runtime_required( +) { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + + let fixtures = j2k_test_support::openjph_batch_fixtures() + .iter() + .filter(|fixture| { + matches!( + fixture.name, + "openjph-rgb-s8-53-single-raw" + | "openjph-rgb-s12-53-single-raw" + | "openjph-rgb-s16-53-single-raw" + ) + }) + .collect::>(); + assert_eq!(fixtures.len(), 3); + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { + roi: Rect { + x: 2, + y: 3, + w: 9, + h: 7, + }, + }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi: Rect { + x: 2, + y: 4, + w: 10, + h: 8, + }, + scale: Downscale::Half, + }, + ]; + let context = CudaContext::system_default().expect("CUDA context"); + + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let session = CudaSession::with_context(context.clone()); + let mut decoder = CudaBatchDecoder::with_session_and_options(session, options); + let mut cpu = CpuBatchDecoder::new(options); + + for fixture in &fixtures { + let encoded = Arc::<[u8]>::from(fixture.encoded); + for request in requests { + let prepared = prepare_batch( + vec![EncodedImage::new(Arc::clone(&encoded), request)], + options, + ) + .unwrap_or_else(|error| panic!("{} prepare: {error}", fixture.name)); + assert!(prepared.errors().is_empty(), "{}", fixture.name); + let [prepared_group] = prepared.groups() else { + panic!("{}: expected one signed RGB group", fixture.name) + }; + assert!( + prepared_group.images()[0].htj2k_plan().is_some(), + "{}", + fixture.name + ); + + let oracle = cpu + .decode_prepared(&prepared) + .unwrap_or_else(|error| panic!("{} CPU oracle: {error}", fixture.name)); + let expected_samples = match oracle.groups()[0].samples() { + CpuBatchSamples::I16(samples) => samples, + other => panic!( + "{}: unexpected signed RGB oracle type: {other:?}", + fixture.name + ), + }; + if layout == BatchLayout::Nhwc && request == DecodeRequest::Full { + assert_eq!( + expected_samples, + &openjph_i16_oracle(fixture), + "{} independent OpenJPH oracle", + fixture.name + ); + } + let expected = expected_samples + .iter() + .flat_map(|sample| sample.to_ne_bytes()) + .collect::>(); + + let mut allocation = context + .allocate(expected.len()) + .expect("signed RGB external destination"); + let submitted = { + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: the allocation remains live and inaccessible until + // the returned completion owner is waited below. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + &context, + ptr, + len, + std::mem::align_of::(), + &mut allocation, + ) + } + .expect("signed RGB external view"); + // SAFETY: the view owns the unique destination borrow through + // submission and completion is established before host read. + unsafe { decoder.submit_batch_into(prepared_group, &mut destination) } + .unwrap_or_else(|error| panic!("{} submit external: {error}", fixture.name)) + }; + submitted + .wait() + .unwrap_or_else(|error| panic!("{} wait external: {error}", fixture.name)); + let mut external = vec![0_u8; expected.len()]; + allocation + .copy_to_host(&mut external) + .expect("download signed RGB external output"); + assert_eq!( + external, expected, + "{} external {layout:?} {request:?}", + fixture.name + ); + + let resident = decoder + .decode_prepared(&prepared) + .unwrap_or_else(|error| panic!("{} resident: {error}", fixture.name)); + let [resident_group] = resident.groups() else { + panic!("{}: expected one signed RGB resident group", fixture.name) + }; + let dense = resident_group.dense_output(); + let mut resident_bytes = vec![0_u8; expected.len()]; + dense + .buffer() + .copy_range_to_host(dense.ranges()[0].offset, &mut resident_bytes) + .expect("download signed RGB resident output"); + assert_eq!( + resident_bytes, expected, + "{} resident {layout:?} {request:?}", + fixture.name + ); + } + } + } +} + +fn openjph_i16_oracle(fixture: &OpenJphBatchFixture) -> Vec { + if fixture.precision <= 8 { + fixture + .oracle + .iter() + .map(|sample| i16::from(i8::from_ne_bytes([*sample]))) + .collect() + } else { + fixture + .oracle + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]])) + .collect() + } +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/support.rs b/crates/j2k-cuda/tests/batch_decoder_api/support.rs new file mode 100644 index 00000000..6a631465 --- /dev/null +++ b/crates/j2k-cuda/tests/batch_decoder_api/support.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::PreparedBatchGroup; +use j2k_cuda::{CudaBatchDecoder, SubmittedCudaExternalBatch}; +use j2k_cuda_runtime::{CudaContext, CudaDeviceBuffer, CudaExternalDeviceBufferViewMut}; + +pub(super) unsafe fn submit_external_for_test( + decoder: &mut CudaBatchDecoder, + group: &PreparedBatchGroup, + context: &CudaContext, + allocation: &mut CudaDeviceBuffer, +) -> SubmittedCudaExternalBatch { + let ptr = allocation.device_ptr(); + let len = allocation.byte_len(); + // SAFETY: the caller guarantees `allocation` stays live and inaccessible + // until the returned completion owner is retired. + let mut destination = unsafe { + CudaExternalDeviceBufferViewMut::from_raw_parts( + context, + ptr, + len, + std::mem::align_of::(), + allocation, + ) + } + .expect("external destination view"); + // SAFETY: forwarded from this helper's caller; the view exclusively + // borrows the live allocation for submission. + unsafe { decoder.submit_batch_into(group, &mut destination) } + .expect("submit external test batch") +} diff --git a/crates/j2k-cuda/tests/bench_harness.rs b/crates/j2k-cuda/tests/bench_harness.rs index 43171839..1cb190bf 100644 --- a/crates/j2k-cuda/tests/bench_harness.rs +++ b/crates/j2k-cuda/tests/bench_harness.rs @@ -326,6 +326,7 @@ fn read_cuda_runtime_sources() -> String { "htj2k_decode.rs", "htj2k_decode/api.rs", "htj2k_decode/completion.rs", + "htj2k_decode/completion/cleanup_enqueue.rs", "htj2k_decode/context_validation.rs", "htj2k_decode/launch.rs", "htj2k_decode/output_regions.rs", diff --git a/crates/j2k-metal-support/src/lib.rs b/crates/j2k-metal-support/src/lib.rs index ed305cea..479d65af 100644 --- a/crates/j2k-metal-support/src/lib.rs +++ b/crates/j2k-metal-support/src/lib.rs @@ -48,7 +48,9 @@ pub use dispatch::{ #[cfg(target_os = "macos")] pub use pipeline::{named_pipeline, shader_library, MetalPipelineLoader}; #[cfg(target_os = "macos")] -pub use resident::{MetalImageLayout, ResidentMetalImage, SubmittedMetalImages}; +pub use resident::{ + MetalImageDestination, MetalImageLayout, ResidentMetalImage, SubmittedMetalImages, +}; #[cfg(target_os = "macos")] pub use runtime::{ checked_blit_command_encoder, checked_command_buffer, checked_command_queue, diff --git a/crates/j2k-metal-support/src/resident.rs b/crates/j2k-metal-support/src/resident.rs index e1b91aa6..a1c027bf 100644 --- a/crates/j2k-metal-support/src/resident.rs +++ b/crates/j2k-metal-support/src/resident.rs @@ -7,95 +7,9 @@ use metal::{Buffer, BufferRef, CommandBuffer, DeviceRef, MTLCommandBufferStatus} use crate::{wait_for_completion, MetalSupportError}; -/// Validated byte layout for one image stored in a Metal buffer. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct MetalImageLayout { - byte_offset: usize, - dimensions: (u32, u32), - pitch_bytes: usize, - pixel_format: PixelFormat, - byte_len: usize, -} +mod destination; -impl MetalImageLayout { - /// Validate image geometry and construct a Metal buffer layout. - /// - /// # Errors - /// - /// Returns [`MetalSupportError::MetalImageLayout`] when row geometry or - /// the represented byte range overflows, or when the pitch is shorter than - /// one pixel row. - pub fn new( - byte_offset: usize, - dimensions: (u32, u32), - pitch_bytes: usize, - pixel_format: PixelFormat, - ) -> Result { - if dimensions.0 == 0 || dimensions.1 == 0 { - return Err(MetalSupportError::MetalImageLayout { - reason: "image dimensions must be nonzero", - }); - } - let row_bytes = usize::try_from(dimensions.0) - .ok() - .and_then(|width| width.checked_mul(pixel_format.bytes_per_pixel())) - .ok_or(MetalSupportError::MetalImageLayout { - reason: "pixel row byte count overflows usize", - })?; - if pitch_bytes < row_bytes { - return Err(MetalSupportError::MetalImageLayout { - reason: "pitch is shorter than one pixel row", - }); - } - let byte_len = pitch_bytes.checked_mul(dimensions.1 as usize).ok_or( - MetalSupportError::MetalImageLayout { - reason: "pitched image byte length overflows usize", - }, - )?; - byte_offset - .checked_add(byte_len) - .ok_or(MetalSupportError::MetalImageLayout { - reason: "image byte range overflows usize", - })?; - Ok(Self { - byte_offset, - dimensions, - pitch_bytes, - pixel_format, - byte_len, - }) - } - - /// Byte offset of the first pixel row. - #[must_use] - pub const fn byte_offset(self) -> usize { - self.byte_offset - } - - /// Image dimensions in pixels. - #[must_use] - pub const fn dimensions(self) -> (u32, u32) { - self.dimensions - } - - /// Number of bytes between consecutive image rows. - #[must_use] - pub const fn pitch_bytes(self) -> usize { - self.pitch_bytes - } - - /// Pixel format stored in the image range. - #[must_use] - pub const fn pixel_format(self) -> PixelFormat { - self.pixel_format - } - - /// Number of bytes represented by the pitched image. - #[must_use] - pub const fn byte_len(self) -> usize { - self.byte_len - } -} +pub use self::destination::{MetalImageDestination, MetalImageLayout}; /// Owned, logically immutable Metal-resident image. /// diff --git a/crates/j2k-metal-support/src/resident/destination.rs b/crates/j2k-metal-support/src/resident/destination.rs new file mode 100644 index 00000000..b7227238 --- /dev/null +++ b/crates/j2k-metal-support/src/resident/destination.rs @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::fmt; + +use j2k_core::PixelFormat; +use metal::{Buffer, DeviceRef}; + +use super::{buffer_len, validate_bounds, validate_registry_id}; +use crate::MetalSupportError; + +/// Validated byte layout for one image stored in a Metal buffer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MetalImageLayout { + byte_offset: usize, + dimensions: (u32, u32), + pitch_bytes: usize, + pixel_format: PixelFormat, + image_count: usize, + image_stride_bytes: usize, + byte_len: usize, +} + +const METAL_BUFFER_OFFSET_ALIGNMENT: usize = 4; + +/// Exclusively writable image subrange in a caller-owned Metal buffer. +/// +/// This type is the checked handoff boundary for decoders that write directly +/// into storage allocated by another runtime. It retains the Metal allocation, +/// records its device identity, and exposes only the validated image layout. +/// Constructing it is unsafe because the Rust type system cannot prove that a +/// framework holding another `MTLBuffer` reference will honor exclusive GPU +/// write access to the selected subrange. +pub struct MetalImageDestination { + buffer: Buffer, + device_registry_id: u64, + buffer_len: usize, + layout: MetalImageLayout, +} + +impl fmt::Debug for MetalImageDestination { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MetalImageDestination") + .field("device_registry_id", &self.device_registry_id) + .field("buffer_len", &self.buffer_len) + .field("layout", &self.layout) + .finish_non_exhaustive() + } +} + +impl MetalImageDestination { + /// Validate and retain an exclusively writable Metal image subrange. + /// + /// # Safety + /// + /// Until codec completion or an explicit GPU dependency has been + /// established, the caller must prevent every CPU access and every GPU + /// read or write that overlaps `layout`. Once the codec has inserted a + /// same-device consumer-queue wait, later reads on that queue are allowed; + /// unsynchronized access and overlapping writes remain forbidden. A + /// decoder may bind the raw buffer only for writes inside the validated + /// range and must retain this value until completion or dependency + /// registration makes consumer access safe. + /// + /// # Errors + /// + /// Returns a typed bounds or alignment error when the image subrange cannot + /// be safely bound as a Metal kernel destination. + pub unsafe fn from_exclusive_buffer( + buffer: Buffer, + layout: MetalImageLayout, + ) -> Result { + let buffer_len = buffer_len(&buffer)?; + validate_bounds(layout, buffer_len)?; + if !layout + .byte_offset() + .is_multiple_of(METAL_BUFFER_OFFSET_ALIGNMENT) + { + return Err(MetalSupportError::BufferAlignment { + offset_bytes: layout.byte_offset(), + align: METAL_BUFFER_OFFSET_ALIGNMENT, + }); + } + Ok(Self { + device_registry_id: buffer.device().registry_id(), + buffer, + buffer_len, + layout, + }) + } + + /// Validated image layout within the destination allocation. + #[must_use] + pub const fn layout(&self) -> MetalImageLayout { + self.layout + } + + /// Total allocation length retained by this destination. + #[must_use] + pub const fn buffer_len(&self) -> usize { + self.buffer_len + } + + /// Registry identifier of the Metal device that owns the allocation. + #[must_use] + pub const fn device_registry_id(&self) -> u64 { + self.device_registry_id + } + + /// Validate that a Metal device can write this destination. + /// + /// # Errors + /// + /// Returns [`MetalSupportError::MetalImageDeviceMismatch`] when the + /// destination allocation belongs to another Metal device. + pub fn validate_device(&self, device: &DeviceRef) -> Result<(), MetalSupportError> { + validate_registry_id(self.device_registry_id, device.registry_id()) + } + + /// Validate the dimensions and pixel format expected by a decoder store. + /// + /// # Errors + /// + /// Returns [`MetalSupportError::MetalImageLayout`] when the decoder output + /// geometry or pixel format differs from the validated destination layout. + pub fn validate_image( + &self, + dimensions: (u32, u32), + pixel_format: PixelFormat, + ) -> Result<(), MetalSupportError> { + if self.layout.dimensions() != dimensions { + return Err(MetalSupportError::MetalImageLayout { + reason: "destination dimensions do not match decoded image", + }); + } + if self.layout.pixel_format() != pixel_format { + return Err(MetalSupportError::MetalImageLayout { + reason: "destination pixel format does not match decoded image", + }); + } + Ok(()) + } + + /// Validate a complete dense image group stored in this destination. + /// + /// # Errors + /// + /// Returns [`MetalSupportError::MetalImageLayout`] when the decoded image + /// geometry, pixel format, or group count differs from this range. + pub fn validate_batch( + &self, + dimensions: (u32, u32), + pixel_format: PixelFormat, + image_count: usize, + ) -> Result<(), MetalSupportError> { + self.validate_image(dimensions, pixel_format)?; + if self.layout.image_count() != image_count { + return Err(MetalSupportError::MetalImageLayout { + reason: "destination image count does not match decoded group", + }); + } + Ok(()) + } + + /// Borrow the allocation for an audited backend write operation. + /// + /// # Safety + /// + /// The returned handle may only be bound for writes within [`Self::layout`] + /// under the exclusive-access and completion guarantees established by the + /// constructor. It must not escape the audited backend operation. + #[must_use] + pub unsafe fn raw_buffer(&self) -> &Buffer { + &self.buffer + } +} + +impl MetalImageLayout { + /// Validate image geometry and construct a Metal buffer layout. + /// + /// # Errors + /// + /// Returns [`MetalSupportError::MetalImageLayout`] when row geometry or + /// the represented byte range overflows, or when the pitch is shorter than + /// one pixel row. + pub fn new( + byte_offset: usize, + dimensions: (u32, u32), + pitch_bytes: usize, + pixel_format: PixelFormat, + ) -> Result { + let image_byte_len = checked_image_byte_len(dimensions, pitch_bytes, pixel_format)?; + Self::new_batch( + byte_offset, + dimensions, + pitch_bytes, + pixel_format, + 1, + image_byte_len, + ) + } + + /// Validate a homogeneous image group inside one Metal allocation. + /// + /// Only `byte_offset`, the group base, must satisfy Metal buffer-binding + /// alignment. `image_stride_bytes` may place later Gray8 images at byte + /// offsets that are not independently bindable; final-store kernels bind + /// the allocation once and apply those item offsets in-kernel. + /// + /// # Errors + /// + /// Returns [`MetalSupportError::MetalImageLayout`] for zero image count, + /// short or sample-misaligned strides, or overflowing byte geometry. + pub fn new_batch( + byte_offset: usize, + dimensions: (u32, u32), + pitch_bytes: usize, + pixel_format: PixelFormat, + image_count: usize, + image_stride_bytes: usize, + ) -> Result { + let image_byte_len = checked_image_byte_len(dimensions, pitch_bytes, pixel_format)?; + if image_count == 0 { + return Err(MetalSupportError::MetalImageLayout { + reason: "image count must be nonzero", + }); + } + if image_stride_bytes < image_byte_len { + return Err(MetalSupportError::MetalImageLayout { + reason: "image stride is shorter than one pitched image", + }); + } + if !pitch_bytes.is_multiple_of(pixel_format.bytes_per_sample()) + || !image_stride_bytes.is_multiple_of(pixel_format.bytes_per_sample()) + { + return Err(MetalSupportError::MetalImageLayout { + reason: "row and image strides must be aligned to the sample width", + }); + } + let prior_images = image_count - 1; + let byte_len = image_stride_bytes + .checked_mul(prior_images) + .and_then(|offset| offset.checked_add(image_byte_len)) + .ok_or(MetalSupportError::MetalImageLayout { + reason: "pitched image group byte length overflows usize", + })?; + byte_offset + .checked_add(byte_len) + .ok_or(MetalSupportError::MetalImageLayout { + reason: "image group byte range overflows usize", + })?; + Ok(Self { + byte_offset, + dimensions, + pitch_bytes, + pixel_format, + image_count, + image_stride_bytes, + byte_len, + }) + } + + /// Number of homogeneous images represented by this allocation range. + #[must_use] + pub const fn image_count(self) -> usize { + self.image_count + } + + /// Byte distance between the first pixel of consecutive images. + #[must_use] + pub const fn image_stride_bytes(self) -> usize { + self.image_stride_bytes + } + + /// Byte offset of one image relative to the validated group base. + /// + /// Returns `None` when `image_index` is outside the group or multiplication + /// overflows. + #[must_use] + pub const fn image_offset_bytes(self, image_index: usize) -> Option { + if image_index >= self.image_count { + return None; + } + self.image_stride_bytes.checked_mul(image_index) + } + + /// Byte offset of the first pixel row. + #[must_use] + pub const fn byte_offset(self) -> usize { + self.byte_offset + } + + /// Image dimensions in pixels. + #[must_use] + pub const fn dimensions(self) -> (u32, u32) { + self.dimensions + } + + /// Number of bytes between consecutive image rows. + #[must_use] + pub const fn pitch_bytes(self) -> usize { + self.pitch_bytes + } + + /// Pixel format stored in the image range. + #[must_use] + pub const fn pixel_format(self) -> PixelFormat { + self.pixel_format + } + + /// Number of bytes represented by the pitched image or image group. + #[must_use] + pub const fn byte_len(self) -> usize { + self.byte_len + } +} + +fn checked_image_byte_len( + dimensions: (u32, u32), + pitch_bytes: usize, + pixel_format: PixelFormat, +) -> Result { + if dimensions.0 == 0 || dimensions.1 == 0 { + return Err(MetalSupportError::MetalImageLayout { + reason: "image dimensions must be nonzero", + }); + } + let row_bytes = usize::try_from(dimensions.0) + .ok() + .and_then(|width| width.checked_mul(pixel_format.bytes_per_pixel())) + .ok_or(MetalSupportError::MetalImageLayout { + reason: "pixel row byte count overflows usize", + })?; + if pitch_bytes < row_bytes { + return Err(MetalSupportError::MetalImageLayout { + reason: "pitch is shorter than one pixel row", + }); + } + pitch_bytes + .checked_mul(dimensions.1 as usize) + .ok_or(MetalSupportError::MetalImageLayout { + reason: "pitched image byte length overflows usize", + }) +} diff --git a/crates/j2k-metal-support/src/tests/resident.rs b/crates/j2k-metal-support/src/tests/resident.rs index d2fac6e7..2916ed7b 100644 --- a/crates/j2k-metal-support/src/tests/resident.rs +++ b/crates/j2k-metal-support/src/tests/resident.rs @@ -3,8 +3,8 @@ use crate::{ checked_blit_command_encoder, checked_buffer_read_vec, checked_command_buffer, checked_command_queue, checked_shared_buffer_for_len, checked_shared_buffer_with_slice, - system_default_device, MetalImageLayout, MetalSupportError, ResidentMetalImage, - SubmittedMetalImages, + system_default_device, MetalImageDestination, MetalImageLayout, MetalSupportError, + ResidentMetalImage, SubmittedMetalImages, }; use j2k_core::{DeviceSubmission, PixelFormat}; @@ -28,6 +28,28 @@ fn metal_image_layout_rejects_short_pitch_and_overflow() { )); } +#[test] +fn metal_image_batch_layout_allows_unaligned_gray8_item_offsets_from_aligned_base() { + let layout = MetalImageLayout::new_batch(4, (3, 3), 3, PixelFormat::Gray8, 2, 9) + .expect("valid odd Gray8 batch layout"); + + assert_eq!(layout.byte_offset(), 4); + assert_eq!(layout.image_count(), 2); + assert_eq!(layout.image_stride_bytes(), 9); + assert_eq!(layout.image_offset_bytes(0), Some(0)); + assert_eq!(layout.image_offset_bytes(1), Some(9)); + assert_eq!(layout.image_offset_bytes(2), None); + assert_eq!(layout.byte_len(), 18); + assert!(matches!( + MetalImageLayout::new_batch(4, (3, 3), 3, PixelFormat::Gray8, 0, 9), + Err(MetalSupportError::MetalImageLayout { .. }) + )); + assert!(matches!( + MetalImageLayout::new_batch(4, (3, 3), 3, PixelFormat::Gray8, 2, 8), + Err(MetalSupportError::MetalImageLayout { .. }) + )); +} + #[test] fn resident_metal_image_validates_bounds_and_device_identity() { if !j2k_test_support::metal_runtime_gate(module_path!()) { @@ -212,3 +234,67 @@ fn metal_image_device_identity_reports_mismatched_registry_ids() { }) )); } + +#[test] +fn metal_image_destination_rejects_unaligned_and_out_of_bounds_subranges() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let Ok(device) = system_default_device() else { + j2k_test_support::metal_device_unavailable_is_skip(module_path!()); + return; + }; + let buffer = checked_shared_buffer_for_len::(&device, 16).expect("bounded buffer"); + let unaligned = + MetalImageLayout::new(1, (4, 1), 4, PixelFormat::Gray8).expect("standalone layout"); + + // SAFETY: No CPU or GPU operation accesses this fresh allocation while + // destination construction validates the proposed exclusive range. + let error = unsafe { MetalImageDestination::from_exclusive_buffer(buffer.clone(), unaligned) } + .expect_err("unaligned destination"); + assert_eq!( + error, + MetalSupportError::BufferAlignment { + offset_bytes: 1, + align: 4, + } + ); + + let out_of_bounds = + MetalImageLayout::new(12, (8, 1), 8, PixelFormat::Gray8).expect("standalone layout"); + // SAFETY: As above, construction is the only access to the fresh buffer. + let error = unsafe { MetalImageDestination::from_exclusive_buffer(buffer, out_of_bounds) } + .expect_err("out-of-bounds destination"); + assert_eq!( + error, + MetalSupportError::BufferBounds { + offset_bytes: 12, + byte_len: 8, + buffer_len: 16, + } + ); +} + +#[test] +fn metal_image_destination_validates_device_and_preserves_subrange_layout() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let Ok(device) = system_default_device() else { + j2k_test_support::metal_device_unavailable_is_skip(module_path!()); + return; + }; + let buffer = checked_shared_buffer_for_len::(&device, 32).expect("bounded buffer"); + let layout = + MetalImageLayout::new(8, (4, 2), 4, PixelFormat::Gray8).expect("valid subrange layout"); + + // SAFETY: The test keeps no concurrent CPU/GPU access to this destination. + let destination = unsafe { MetalImageDestination::from_exclusive_buffer(buffer, layout) } + .expect("valid destination"); + + assert_eq!(destination.layout(), layout); + assert_eq!(destination.device_registry_id(), device.registry_id()); + destination + .validate_device(&device) + .expect("matching destination device"); +} diff --git a/crates/j2k-metal/Cargo.toml b/crates/j2k-metal/Cargo.toml index b46daf52..ba92bef5 100644 --- a/crates/j2k-metal/Cargo.toml +++ b/crates/j2k-metal/Cargo.toml @@ -39,7 +39,7 @@ rayon = { workspace = true } [dev-dependencies] rayon = { workspace = true } -j2k-test-support = { path = "../j2k-test-support" } +j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [build-dependencies] cc = { workspace = true } diff --git a/crates/j2k-metal/src/batch.rs b/crates/j2k-metal/src/batch.rs index d2a9b6be..4f3c3390 100644 --- a/crates/j2k-metal/src/batch.rs +++ b/crates/j2k-metal/src/batch.rs @@ -1,773 +1,27 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::{ - cell::OnceCell, - collections::hash_map::DefaultHasher, - hash::{Hash, Hasher}, - sync::{Arc, Mutex, MutexGuard}, -}; - -use j2k_core::{BackendRequest, DeviceSubmission, Downscale, PixelFormat, Rect}; - -use crate::{profile, Error, J2kDecoder, MetalSession, Surface}; - mod cpu; mod execute; mod heuristics; -use self::cpu::decode_cpu_host_batch; -use self::execute::process_batch; -#[cfg(all(test, target_os = "macos"))] -use self::heuristics::GroupedRequests; +mod request; +mod routes; +mod session; #[cfg(test)] -use self::heuristics::{ - auto_region_scaled_direct_metal_min_dim, can_decode_requests_as_repeated_region_scaled_batch, - profile_route_label, same_input_bytes, BatchRoute, AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT, - AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM, AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT, - AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, +mod tests; + +use self::cpu::decode_cpu_host_batch; +pub(crate) use self::request::BatchOp; +use self::request::{batch_scheduler_invariant, QueuedRequest}; +pub use self::request::{benchmark_group_region_scaled_requests, BenchmarkGroupedRequests}; +use self::routes::{ + decode_distinct_full_color_batch, decode_distinct_full_grayscale_batch, + decode_distinct_region_scaled_direct_batch, + decode_distinct_region_scaled_direct_batch_prechecked, decode_individual, + decode_repeated_full_color, decode_repeated_full_grayscale, + decode_repeated_region_scaled_direct_batch_prechecked, }; -use self::heuristics::{ - group_metal_requests, is_distinct_full_color_metal_candidate, - is_distinct_full_grayscale_metal_candidate, is_region_scaled_direct_batch_candidate, - is_repeated_full_color_candidate, is_repeated_full_grayscale_candidate, - should_auto_use_metal_for_region_scaled_direct_batch, +pub use self::session::MetalSubmission; +use self::session::SessionState; +pub(crate) use self::session::{ + queue_tile_request, queue_tile_request_shared_with_retained, SharedSession, }; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum BatchOp { - Full, - Region(Rect), - Scaled(Downscale), - RegionScaled { roi: Rect, scale: Downscale }, -} - -#[derive(Clone)] -struct QueuedRequest { - input: Arc<[u8]>, - fmt: PixelFormat, - backend: BackendRequest, - op: BatchOp, - output_slot: usize, - max_image_dim: OnceCell>, - input_fingerprint: OnceCell, -} - -impl QueuedRequest { - fn max_image_dim(&self) -> Option { - *self.max_image_dim.get_or_init(|| { - let decoder = J2kDecoder::new(self.input.as_ref()).ok()?; - let dims = decoder.inner.info().dimensions; - Some(dims.0.max(dims.1)) - }) - } - - fn input_fingerprint(&self) -> u64 { - *self.input_fingerprint.get_or_init(|| { - let mut hasher = DefaultHasher::new(); - self.input.len().hash(&mut hasher); - if !self.input.is_empty() { - let len = self.input.len(); - for offset in [0, len / 4, len / 2, len.saturating_sub(8)] { - let end = offset.saturating_add(8).min(len); - self.input[offset..end].hash(&mut hasher); - } - } - hasher.finish() - }) - } - - #[cfg(test)] - fn max_image_dim_cache_filled_for_test(&self) -> bool { - self.max_image_dim.get().is_some() - } - - #[cfg(test)] - fn input_fingerprint_cache_filled_for_test(&self) -> bool { - self.input_fingerprint.get().is_some() - } -} - -fn batch_scheduler_invariant(message: &'static str) -> Error { - Error::MetalKernel { - message: format!("internal J2K Metal batch scheduler error: {message}"), - } -} - -#[doc(hidden)] -pub struct BenchmarkGroupedRequests { - pub batch_count: usize, - pub max_batch_len: usize, -} - -#[doc(hidden)] -pub fn benchmark_group_region_scaled_requests( - inputs: &[Arc<[u8]>], - fmt: PixelFormat, - backend: BackendRequest, - roi: Rect, - scale: Downscale, -) -> Result { - let mut budget = - crate::batch_allocation::BatchMetadataBudget::new("J2K Metal benchmark grouping requests"); - let mut queued = budget.try_vec(inputs.len(), "J2K Metal benchmark queued requests")?; - queued.extend( - inputs - .iter() - .enumerate() - .map(|(output_slot, input)| QueuedRequest { - input: input.clone(), - fmt, - backend, - op: BatchOp::RegionScaled { roi, scale }, - output_slot, - max_image_dim: OnceCell::new(), - input_fingerprint: OnceCell::new(), - }), - ); - let batches = group_metal_requests(queued)?; - Ok(BenchmarkGroupedRequests { - batch_count: batches.len(), - max_batch_len: batches - .iter() - .map(|batch| batch.requests.len()) - .max() - .unwrap_or(0), - }) -} - -#[derive(Default)] -pub(crate) struct SessionState { - pub(crate) submissions: u64, - queued: Vec, - completed: Vec>>, -} - -#[derive(Clone, Default)] -pub(crate) struct SharedSession(pub(crate) Arc>); - -impl SharedSession { - pub(crate) fn lock(&self) -> Result, Error> { - self.0.lock().map_err(|_| Error::MetalStatePoisoned { - state: "J2K Metal session", - }) - } -} - -pub struct MetalSubmission { - session: SharedSession, - slot: usize, -} - -#[doc(hidden)] -impl DeviceSubmission for MetalSubmission { - type Output = Surface; - type Error = Error; - - fn wait(self) -> Result { - let mut session = self.session.lock()?; - flush_if_needed(&mut session); - take_surface(&mut session, self.slot) - } -} - -pub(crate) fn queue_tile_request( - session: &mut MetalSession, - input: &[u8], - fmt: PixelFormat, - backend: BackendRequest, - op: BatchOp, -) -> Result { - queue_tile_request_shared(session, Arc::<[u8]>::from(input), fmt, backend, op) -} - -pub(crate) fn queue_tile_request_shared( - session: &mut MetalSession, - input: Arc<[u8]>, - fmt: PixelFormat, - backend: BackendRequest, - op: BatchOp, -) -> Result { - queue_tile_request_shared_with_retained(session, input, fmt, backend, op, 0) -} - -pub(crate) fn queue_tile_request_shared_with_retained( - session: &mut MetalSession, - input: Arc<[u8]>, - fmt: PixelFormat, - backend: BackendRequest, - op: BatchOp, - retained_submission_capacity: usize, -) -> Result { - let mut state = session.shared.lock()?; - crate::batch_allocation::try_reserve_for_push( - &mut state.completed, - "J2K Metal queued completion slots", - )?; - crate::batch_allocation::try_reserve_for_push(&mut state.queued, "J2K Metal queued requests")?; - let aggregate = - crate::batch_allocation::BatchMetadataBudget::new("J2K Metal queued request state"); - aggregate.preflight(&[ - crate::batch_allocation::BatchMetadataRequest::of::( - retained_submission_capacity, - ), - crate::batch_allocation::BatchMetadataRequest::of::(state.queued.capacity()), - crate::batch_allocation::BatchMetadataRequest::of::>>( - state.completed.capacity(), - ), - ])?; - let slot = state.completed.len(); - state.completed.push(None); - state.queued.push(QueuedRequest { - input, - fmt, - backend, - op, - output_slot: slot, - max_image_dim: OnceCell::new(), - input_fingerprint: OnceCell::new(), - }); - Ok(MetalSubmission { - session: session.shared.clone(), - slot, - }) -} - -fn flush_if_needed(session: &mut SessionState) { - if session.queued.is_empty() { - return; - } - - let profile_enabled = profile::metal_profile_stages_enabled(); - let queued = std::mem::take(&mut session.queued); - let request_count = queued.len(); - let mut slot_budget = - crate::batch_allocation::BatchMetadataBudget::new("J2K Metal grouping recovery slots"); - let mut output_slots = - match slot_budget.try_vec(queued.len(), "J2K Metal grouping recovery output slots") { - Ok(slots) => slots, - Err(error) => { - for request in queued { - session.completed[request.output_slot] = Some(Err(error.into())); - } - return; - } - }; - output_slots.extend(queued.iter().map(|request| request.output_slot)); - let group_started = profile::profile_now(profile_enabled); - let batches = match group_metal_requests(queued) { - Ok(batches) => batches, - Err(error) => { - for output_slot in output_slots { - session.completed[output_slot] = Some(Err(error.into())); - } - return; - } - }; - drop(output_slots); - if profile_enabled { - profile::emit_metal_batch_profile_row( - "decode", - &profile::MetalBatchProfileRow { - slice: "decode_batch", - stage: "group", - pipeline: "metal_cpu_hybrid", - processor: "scheduler", - route: "all", - backend: profile::MetalBatchProfileValue::Mixed, - fmt: profile::MetalBatchProfileValue::Mixed, - request_count, - output_count: batches.len(), - elapsed_us: profile::elapsed_us(group_started), - outcome: "grouped", - }, - ); - } - - for batch in batches { - process_batch(session, batch); - } -} - -fn decode_repeated_full_grayscale( - request: &QueuedRequest, - count: usize, -) -> Option, Error>> { - if !is_repeated_full_grayscale_candidate(request) || count <= 1 { - return None; - } - - #[cfg(target_os = "macos")] - { - let result = - J2kDecoder::new(request.input.as_ref()).and_then(|mut decoder| match request.backend { - BackendRequest::Auto => { - decoder.decode_repeated_grayscale_auto_to_device(request.fmt, count) - } - BackendRequest::Metal => { - decoder.decode_repeated_grayscale_direct_to_device(request.fmt, count) - } - _ => Err(batch_scheduler_invariant( - "repeated grayscale batch contains an unsupported backend", - )), - }); - Some(result) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_repeated_full_color( - request: &QueuedRequest, - count: usize, -) -> Option, Error>> { - if !is_repeated_full_color_candidate(request) || count <= 1 { - return None; - } - - #[cfg(target_os = "macos")] - { - let result = J2kDecoder::new(request.input.as_ref()).and_then(|mut decoder| { - decoder.decode_repeated_color_direct_to_device(request.fmt, count) - }); - Some(result) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_distinct_full_grayscale_batch( - requests: &[QueuedRequest], -) -> Option, Error>> { - let first = requests.first()?; - if requests.len() <= 1 - || !requests.iter().all(|request| { - is_distinct_full_grayscale_metal_candidate(request) && request.fmt == first.fmt - }) - { - return None; - } - - #[cfg(target_os = "macos")] - { - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "J2K Metal distinct grayscale batch inputs", - ); - let mut inputs = - match budget.try_vec(requests.len(), "J2K Metal distinct grayscale input handles") { - Ok(inputs) => inputs, - Err(error) => return Some(Err(error.into())), - }; - inputs.extend(requests.iter().map(|request| request.input.clone())); - Some(crate::decoder::decode_full_grayscale_batch_direct_to_device(&inputs, first.fmt)) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_distinct_full_color_batch( - requests: &[QueuedRequest], -) -> Option, Error>> { - let first = requests.first()?; - if requests.len() <= 1 - || !requests.iter().all(|request| { - is_distinct_full_color_metal_candidate(request) && request.fmt == first.fmt - }) - { - return None; - } - - #[cfg(target_os = "macos")] - { - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "J2K Metal distinct color batch inputs", - ); - let mut inputs = - match budget.try_vec(requests.len(), "J2K Metal distinct color input handles") { - Ok(inputs) => inputs, - Err(error) => return Some(Err(error.into())), - }; - inputs.extend(requests.iter().map(|request| request.input.clone())); - Some(crate::decoder::decode_full_color_batch_direct_to_device( - &inputs, first.fmt, - )) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_distinct_region_scaled_direct_batch( - requests: &[QueuedRequest], -) -> Option, Error>> { - decode_distinct_region_scaled_direct_batch_inner(requests, false) -} - -fn decode_repeated_region_scaled_direct_batch_prechecked( - requests: &[QueuedRequest], -) -> Option, Error>> { - let first = requests.first()?; - if requests.len() <= 1 { - return None; - } - if !matches!(first.op, BatchOp::RegionScaled { .. }) { - return None; - } - - #[cfg(target_os = "macos")] - { - let BatchOp::RegionScaled { roi, scale } = first.op else { - return Some(Err(batch_scheduler_invariant( - "repeated direct batch is missing its region-scaled operation", - ))); - }; - let result = match first.fmt { - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => { - crate::hybrid::decode_repeated_region_scaled_color_batch_direct_to_device( - first.input.as_ref(), - roi, - scale, - first.fmt, - requests.len(), - ) - } - _ => return None, - }; - Some(result) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_distinct_region_scaled_direct_batch_prechecked( - requests: &[QueuedRequest], -) -> Option, Error>> { - decode_distinct_region_scaled_direct_batch_inner(requests, true) -} - -fn decode_distinct_region_scaled_direct_batch_inner( - requests: &[QueuedRequest], - auto_metal_prechecked: bool, -) -> Option, Error>> { - let first = requests.first()?; - if requests.len() <= 1 - || !requests.iter().all(|request| { - is_region_scaled_direct_batch_candidate(request) - && request.fmt == first.fmt - && request.backend == first.backend - }) - { - return None; - } - if first.backend == BackendRequest::Auto - && !auto_metal_prechecked - && !should_auto_use_metal_for_region_scaled_direct_batch(requests) - { - return None; - } - - #[cfg(target_os = "macos")] - { - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "J2K Metal direct batch request specifications", - ); - let mut request_specs = match budget.try_vec( - requests.len(), - "J2K Metal direct batch request specifications", - ) { - Ok(specs) => specs, - Err(error) => return Some(Err(error.into())), - }; - for request in requests { - match request.op { - BatchOp::RegionScaled { roi, scale } => { - request_specs.push((request.input.clone(), roi, scale)); - } - _ => { - return Some(Err(batch_scheduler_invariant( - "direct region-scaled batch contains a non-region-scaled request", - ))); - } - } - } - let result = match first.fmt { - PixelFormat::Gray8 | PixelFormat::Gray16 => { - crate::hybrid::decode_region_scaled_grayscale_batch_direct_to_device( - &request_specs, - first.fmt, - ) - } - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => { - crate::hybrid::decode_region_scaled_color_batch_direct_to_device( - &request_specs, - first.fmt, - ) - } - _ => Err(batch_scheduler_invariant( - "direct region-scaled batch contains an unsupported pixel format", - )), - }; - Some(result) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_individual(request: &QueuedRequest) -> Result { - let mut decoder = J2kDecoder::new(request.input.as_ref())?; - match request.op { - BatchOp::Full => decoder.decode_to_surface_impl(request.fmt, request.backend), - BatchOp::Region(roi) => { - decoder.decode_region_to_surface_impl(request.fmt, roi, request.backend) - } - BatchOp::Scaled(scale) => { - decoder.decode_scaled_to_surface_impl(request.fmt, scale, request.backend) - } - BatchOp::RegionScaled { roi, scale } => { - decoder.decode_region_scaled_to_surface_impl(request.fmt, roi, scale, request.backend) - } - } -} - -fn take_surface(session: &mut SessionState, slot: usize) -> Result { - session - .completed - .get_mut(slot) - .and_then(Option::take) - .ok_or_else(|| Error::MetalKernel { - message: format!("missing queued J2K Metal surface for slot {slot}"), - })? -} - -#[cfg(test)] -mod tests { - use super::*; - - fn auto_rgb_region_scaled_request(input: Arc<[u8]>) -> QueuedRequest { - QueuedRequest { - input, - fmt: PixelFormat::Rgb8, - backend: BackendRequest::Auto, - op: BatchOp::RegionScaled { - roi: Rect { - x: 128, - y: 128, - w: 512, - h: 256, - }, - scale: Downscale::Quarter, - }, - output_slot: 0, - max_image_dim: OnceCell::new(), - input_fingerprint: OnceCell::new(), - } - } - - fn auto_rgb_region_scaled_request_with_max_dim( - input: Arc<[u8]>, - max_image_dim: u32, - ) -> QueuedRequest { - let request = auto_rgb_region_scaled_request(input); - request.max_image_dim.set(Some(max_image_dim)).ok(); - request - } - - #[test] - #[expect( - clippy::cast_possible_truncation, - reason = "bounded test fixture index fits in u8" - )] - fn auto_region_scaled_rgb_threshold_requires_repeated_inputs() { - let requests = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) - .map(|idx| auto_rgb_region_scaled_request(Arc::from([idx as u8]))) - .collect::>(); - - assert!(!can_decode_requests_as_repeated_region_scaled_batch( - &requests - )); - assert_eq!( - auto_region_scaled_direct_metal_min_dim(&requests), - None, - "distinct RGB ROI+scaled Auto batches must stay CPU until hybrid wins for distinct inputs" - ); - - let shared = Arc::<[u8]>::from([1_u8]); - let repeated = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) - .map(|_| auto_rgb_region_scaled_request(shared.clone())) - .collect::>(); - assert!(can_decode_requests_as_repeated_region_scaled_batch( - &repeated - )); - } - - #[test] - fn auto_region_scaled_repeated_rgb_uses_measured_batch_two_metal_threshold() { - let shared = Arc::<[u8]>::from([1_u8]); - let repeated = (0..2) - .map(|_| auto_rgb_region_scaled_request_with_max_dim(shared.clone(), 512)) - .collect::>(); - - assert_eq!( - auto_region_scaled_direct_metal_min_dim(&repeated), - Some(512), - "measured repeated RGB ROI+scaled batches should route to Metal from batch 2 at 512px" - ); - - let single = vec![auto_rgb_region_scaled_request_with_max_dim(shared, 512)]; - assert_eq!(auto_region_scaled_direct_metal_min_dim(&single), None); - } - - #[test] - fn queued_request_caches_image_dimension_probe() { - let request = auto_rgb_region_scaled_request(Arc::from([0_u8])); - - assert!(!request.max_image_dim_cache_filled_for_test()); - assert_eq!(request.max_image_dim(), None); - assert!(request.max_image_dim_cache_filled_for_test()); - assert_eq!(request.max_image_dim(), None); - } - - #[test] - fn repeated_input_check_uses_pointer_identity_before_fingerprint() { - let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); - let first = auto_rgb_region_scaled_request(shared.clone()); - let next = auto_rgb_region_scaled_request(shared); - - assert!(same_input_bytes(&first, &next)); - assert!(!first.input_fingerprint_cache_filled_for_test()); - assert!(!next.input_fingerprint_cache_filled_for_test()); - } - - #[test] - fn auto_region_scaled_grouping_preserves_repeated_rgb_metal_decision() { - let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); - let requests = (0..AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT) - .map(|_| { - auto_rgb_region_scaled_request_with_max_dim( - shared.clone(), - AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, - ) - }) - .collect::>(); - - let grouped = group_metal_requests(requests).expect("group requests"); - - assert_eq!(grouped.len(), 1); - assert_eq!( - grouped[0].route, - BatchRoute::AutoRepeatedRegionScaledDirectMetal - ); - assert_eq!( - grouped[0].requests.len(), - AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT - ); - assert!( - grouped[0] - .requests - .iter() - .all(|request| !request.input_fingerprint_cache_filled_for_test()), - "shared repeated inputs should be classified by Arc identity without fingerprinting" - ); - } - - #[test] - #[expect( - clippy::cast_possible_truncation, - reason = "bounded test fixture index fits in u8" - )] - fn auto_region_scaled_distinct_rgb_grouping_preserves_cpu_decision() { - let requests = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) - .map(|idx| { - auto_rgb_region_scaled_request_with_max_dim( - Arc::from([idx as u8]), - AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM, - ) - }) - .collect::>(); - - let grouped = group_metal_requests(requests).expect("group requests"); - - assert_eq!(grouped.len(), 1); - assert_eq!(grouped[0].route, BatchRoute::AutoRegionScaledDirectCpu); - assert_eq!( - grouped[0].requests.len(), - AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT - ); - } - - #[test] - fn profile_route_labels_are_stable_for_decode_batch_slices() { - assert_eq!(profile_route_label(BatchRoute::Generic), "generic"); - assert_eq!( - profile_route_label(BatchRoute::AutoRegionScaledDirectCpu), - "auto_region_scaled_direct_cpu" - ); - assert_eq!( - profile_route_label(BatchRoute::AutoRegionScaledDirectMetal), - "auto_region_scaled_direct_metal" - ); - assert_eq!( - profile_route_label(BatchRoute::AutoRepeatedRegionScaledDirectMetal), - "auto_repeated_region_scaled_direct_metal" - ); - } - - #[cfg(target_os = "macos")] - #[test] - fn auto_region_scaled_prechecked_error_does_not_retry_generic_direct_path() { - let _guard = crate::hybrid::region_scaled_color_plan_test_lock_for_test(); - crate::hybrid::reset_region_scaled_color_plan_builds_for_test(); - let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); - let requests = (0..AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT) - .map(|slot| { - let mut request = auto_rgb_region_scaled_request_with_max_dim( - shared.clone(), - AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, - ); - request.output_slot = slot; - request - }) - .collect::>(); - let mut session = SessionState { - submissions: 0, - queued: Vec::new(), - completed: (0..requests.len()).map(|_| None).collect(), - }; - - process_batch( - &mut session, - GroupedRequests { - route: BatchRoute::AutoRepeatedRegionScaledDirectMetal, - requests, - }, - ); - - assert_eq!( - crate::hybrid::region_scaled_color_plan_builds_for_test(), - 1, - "failed prechecked Auto Metal routing should fall back to CPU without retrying generic direct Metal" - ); - assert!( - session - .completed - .iter() - .all(|result| matches!(result, Some(Err(_)))), - "invalid inputs should be surfaced on every fallback request" - ); - } -} diff --git a/crates/j2k-metal/src/batch/execute.rs b/crates/j2k-metal/src/batch/execute.rs index 5e8933e6..7065c079 100644 --- a/crates/j2k-metal/src/batch/execute.rs +++ b/crates/j2k-metal/src/batch/execute.rs @@ -2,7 +2,7 @@ use j2k_core::{BackendKind, BackendRequest, PixelFormat}; -use crate::{profile, Surface}; +use crate::{profile, MetalBackendSession, Surface}; use super::heuristics::{ can_decode_requests_as_repeated_full_color_batch, @@ -54,7 +54,7 @@ fn complete_cpu_host_fallback(session: &mut SessionState, requests: Vec, +) { let GroupedRequests { route, requests } = grouped; let profile_enabled = profile::metal_profile_stages_enabled(); let started = profile::profile_now(profile_enabled); @@ -84,7 +88,7 @@ pub(super) fn process_batch(session: &mut SessionState, grouped: GroupedRequests None }; - process_batch_inner(session, route, requests); + process_batch_inner(session, route, requests, backend); if let Some(pending) = pending_profile { profile::emit_metal_batch_profile_row( @@ -110,6 +114,7 @@ fn process_batch_inner( session: &mut SessionState, route: BatchRoute, requests: Vec, + backend: Option<&MetalBackendSession>, ) { if route == BatchRoute::AutoRegionScaledDirectCpu { complete_cpu_host_fallback(session, requests); @@ -122,9 +127,9 @@ fn process_batch_inner( ) && requests.len() > 1 { let decoded = if route == BatchRoute::AutoRepeatedRegionScaledDirectMetal { - decode_repeated_region_scaled_direct_batch_prechecked(&requests) + decode_repeated_region_scaled_direct_batch_prechecked(&requests, backend) } else { - decode_distinct_region_scaled_direct_batch_prechecked(&requests) + decode_distinct_region_scaled_direct_batch_prechecked(&requests, backend) }; if let Some(Ok(surfaces)) = decoded { if complete_batch_surfaces(session, &requests, surfaces) { @@ -136,7 +141,9 @@ fn process_batch_inner( } if can_decode_requests_as_repeated_full_grayscale_batch(&requests) { - if let Some(Ok(surfaces)) = decode_repeated_full_grayscale(&requests[0], requests.len()) { + if let Some(Ok(surfaces)) = + decode_repeated_full_grayscale(&requests[0], requests.len(), backend) + { if complete_batch_surfaces(session, &requests, surfaces) { return; } @@ -144,7 +151,9 @@ fn process_batch_inner( } if can_decode_requests_as_repeated_full_color_batch(&requests) { - if let Some(Ok(surfaces)) = decode_repeated_full_color(&requests[0], requests.len()) { + if let Some(Ok(surfaces)) = + decode_repeated_full_color(&requests[0], requests.len(), backend) + { if complete_batch_surfaces(session, &requests, surfaces) { return; } @@ -152,7 +161,7 @@ fn process_batch_inner( } if requests.len() > 1 { - if let Some(Ok(surfaces)) = decode_distinct_full_grayscale_batch(&requests) { + if let Some(Ok(surfaces)) = decode_distinct_full_grayscale_batch(&requests, backend) { if complete_batch_surfaces(session, &requests, surfaces) { return; } @@ -160,7 +169,7 @@ fn process_batch_inner( } if requests.len() > 1 { - if let Some(Ok(surfaces)) = decode_distinct_full_color_batch(&requests) { + if let Some(Ok(surfaces)) = decode_distinct_full_color_batch(&requests, backend) { if complete_batch_surfaces(session, &requests, surfaces) { return; } @@ -168,7 +177,7 @@ fn process_batch_inner( } if requests.len() > 1 { - if let Some(Ok(surfaces)) = decode_distinct_region_scaled_direct_batch(&requests) { + if let Some(Ok(surfaces)) = decode_distinct_region_scaled_direct_batch(&requests, backend) { if complete_batch_surfaces(session, &requests, surfaces) { return; } @@ -185,7 +194,7 @@ fn process_batch_inner( for request in requests { session.submissions = session.submissions.saturating_add(1); - session.completed[request.output_slot] = Some(decode_individual(&request)); + session.completed[request.output_slot] = Some(decode_individual(&request, backend)); } } diff --git a/crates/j2k-metal/src/batch/request.rs b/crates/j2k-metal/src/batch/request.rs new file mode 100644 index 00000000..01a9ce6a --- /dev/null +++ b/crates/j2k-metal/src/batch/request.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{ + cell::OnceCell, + collections::hash_map::DefaultHasher, + hash::{Hash, Hasher}, + sync::Arc, +}; + +use j2k_core::{BackendRequest, Downscale, PixelFormat, Rect}; + +use crate::{Error, J2kDecoder}; + +use super::heuristics::group_metal_requests; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BatchOp { + Full, + Region(Rect), + Scaled(Downscale), + RegionScaled { roi: Rect, scale: Downscale }, +} + +#[derive(Clone)] +pub(super) struct QueuedRequest { + pub(super) input: Arc<[u8]>, + pub(super) fmt: PixelFormat, + pub(super) backend: BackendRequest, + pub(super) op: BatchOp, + pub(super) output_slot: usize, + pub(super) max_image_dim: OnceCell>, + pub(super) input_fingerprint: OnceCell, +} + +impl QueuedRequest { + pub(super) fn new( + input: Arc<[u8]>, + fmt: PixelFormat, + backend: BackendRequest, + op: BatchOp, + output_slot: usize, + ) -> Self { + Self { + input, + fmt, + backend, + op, + output_slot, + max_image_dim: OnceCell::new(), + input_fingerprint: OnceCell::new(), + } + } + + pub(super) fn max_image_dim(&self) -> Option { + *self.max_image_dim.get_or_init(|| { + let decoder = J2kDecoder::new(self.input.as_ref()).ok()?; + let dims = decoder.inner.info().dimensions; + Some(dims.0.max(dims.1)) + }) + } + + pub(super) fn input_fingerprint(&self) -> u64 { + *self.input_fingerprint.get_or_init(|| { + let mut hasher = DefaultHasher::new(); + self.input.len().hash(&mut hasher); + if !self.input.is_empty() { + let len = self.input.len(); + for offset in [0, len / 4, len / 2, len.saturating_sub(8)] { + let end = offset.saturating_add(8).min(len); + self.input[offset..end].hash(&mut hasher); + } + } + hasher.finish() + }) + } + + #[cfg(test)] + pub(super) fn max_image_dim_cache_filled_for_test(&self) -> bool { + self.max_image_dim.get().is_some() + } + + #[cfg(test)] + pub(super) fn input_fingerprint_cache_filled_for_test(&self) -> bool { + self.input_fingerprint.get().is_some() + } +} + +pub(super) fn batch_scheduler_invariant(message: &'static str) -> Error { + Error::MetalKernel { + message: format!("internal J2K Metal batch scheduler error: {message}"), + } +} + +#[doc(hidden)] +pub struct BenchmarkGroupedRequests { + pub batch_count: usize, + pub max_batch_len: usize, +} + +#[doc(hidden)] +pub fn benchmark_group_region_scaled_requests( + inputs: &[Arc<[u8]>], + fmt: PixelFormat, + backend: BackendRequest, + roi: Rect, + scale: Downscale, +) -> Result { + let mut budget = + crate::batch_allocation::BatchMetadataBudget::new("J2K Metal benchmark grouping requests"); + let mut queued = budget.try_vec(inputs.len(), "J2K Metal benchmark queued requests")?; + queued.extend(inputs.iter().enumerate().map(|(output_slot, input)| { + QueuedRequest::new( + input.clone(), + fmt, + backend, + BatchOp::RegionScaled { roi, scale }, + output_slot, + ) + })); + let batches = group_metal_requests(queued)?; + Ok(BenchmarkGroupedRequests { + batch_count: batches.len(), + max_batch_len: batches + .iter() + .map(|batch| batch.requests.len()) + .max() + .unwrap_or(0), + }) +} diff --git a/crates/j2k-metal/src/batch/routes.rs b/crates/j2k-metal/src/batch/routes.rs new file mode 100644 index 00000000..c5461942 --- /dev/null +++ b/crates/j2k-metal/src/batch/routes.rs @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k_core::{BackendRequest, PixelFormat}; + +use crate::{Error, J2kDecoder, MetalBackendSession, MetalDecodeRequest, Surface}; + +use super::heuristics::{ + is_distinct_full_color_metal_candidate, is_distinct_full_grayscale_metal_candidate, + is_region_scaled_direct_batch_candidate, is_repeated_full_color_candidate, + is_repeated_full_grayscale_candidate, should_auto_use_metal_for_region_scaled_direct_batch, +}; +use super::request::{batch_scheduler_invariant, BatchOp, QueuedRequest}; + +pub(super) fn decode_repeated_full_grayscale( + request: &QueuedRequest, + count: usize, + backend: Option<&MetalBackendSession>, +) -> Option, Error>> { + if !is_repeated_full_grayscale_candidate(request) || count <= 1 { + return None; + } + + #[cfg(target_os = "macos")] + { + let result = + J2kDecoder::new(request.input.as_ref()).and_then(|mut decoder| match request.backend { + BackendRequest::Auto => { + decoder.decode_repeated_grayscale_auto_to_device(request.fmt, count) + } + BackendRequest::Metal => decoder.decode_repeated_grayscale_direct_to_device_routed( + request.fmt, + count, + backend, + ), + _ => Err(batch_scheduler_invariant( + "repeated grayscale batch contains an unsupported backend", + )), + }); + Some(result) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = backend; + None + } +} + +pub(super) fn decode_repeated_full_color( + request: &QueuedRequest, + count: usize, + backend: Option<&MetalBackendSession>, +) -> Option, Error>> { + if !is_repeated_full_color_candidate(request) || count <= 1 { + return None; + } + + #[cfg(target_os = "macos")] + { + Some( + J2kDecoder::new(request.input.as_ref()).and_then(|mut decoder| { + decoder.decode_repeated_color_direct_to_device_routed(request.fmt, count, backend) + }), + ) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = backend; + None + } +} + +pub(super) fn decode_distinct_full_grayscale_batch( + requests: &[QueuedRequest], + backend: Option<&MetalBackendSession>, +) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 + || !requests.iter().all(|request| { + is_distinct_full_grayscale_metal_candidate(request) && request.fmt == first.fmt + }) + { + return None; + } + + #[cfg(target_os = "macos")] + { + let inputs = match collect_inputs(requests, "J2K Metal distinct grayscale input handles") { + Ok(inputs) => inputs, + Err(error) => return Some(Err(error)), + }; + Some( + crate::decoder::decode_full_grayscale_batch_direct_to_device_routed( + &inputs, first.fmt, backend, + ), + ) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = backend; + None + } +} + +pub(super) fn decode_distinct_full_color_batch( + requests: &[QueuedRequest], + backend: Option<&MetalBackendSession>, +) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 + || !requests.iter().all(|request| { + is_distinct_full_color_metal_candidate(request) && request.fmt == first.fmt + }) + { + return None; + } + + #[cfg(target_os = "macos")] + { + let inputs = match collect_inputs(requests, "J2K Metal distinct color input handles") { + Ok(inputs) => inputs, + Err(error) => return Some(Err(error)), + }; + Some( + crate::decoder::decode_full_color_batch_direct_to_device_routed( + &inputs, first.fmt, backend, + ), + ) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = backend; + None + } +} + +#[cfg(target_os = "macos")] +fn collect_inputs(requests: &[QueuedRequest], what: &'static str) -> Result>, Error> { + let mut budget = crate::batch_allocation::BatchMetadataBudget::new(what); + let mut inputs = budget.try_vec(requests.len(), what)?; + inputs.extend(requests.iter().map(|request| request.input.clone())); + Ok(inputs) +} + +pub(super) fn decode_distinct_region_scaled_direct_batch( + requests: &[QueuedRequest], + backend: Option<&MetalBackendSession>, +) -> Option, Error>> { + decode_distinct_region_scaled_direct_batch_inner(requests, false, backend) +} + +pub(super) fn decode_repeated_region_scaled_direct_batch_prechecked( + requests: &[QueuedRequest], + backend: Option<&MetalBackendSession>, +) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 || !matches!(first.op, BatchOp::RegionScaled { .. }) { + return None; + } + + #[cfg(target_os = "macos")] + { + let BatchOp::RegionScaled { roi, scale } = first.op else { + return Some(Err(batch_scheduler_invariant( + "repeated direct batch is missing its region-scaled operation", + ))); + }; + match first.fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => Some( + crate::hybrid::decode_repeated_region_scaled_color_batch_direct_to_device_routed( + first.input.as_ref(), + roi, + scale, + first.fmt, + requests.len(), + backend, + ), + ), + _ => None, + } + } + + #[cfg(not(target_os = "macos"))] + { + let _ = backend; + None + } +} + +pub(super) fn decode_distinct_region_scaled_direct_batch_prechecked( + requests: &[QueuedRequest], + backend: Option<&MetalBackendSession>, +) -> Option, Error>> { + decode_distinct_region_scaled_direct_batch_inner(requests, true, backend) +} + +fn decode_distinct_region_scaled_direct_batch_inner( + requests: &[QueuedRequest], + auto_metal_prechecked: bool, + backend: Option<&MetalBackendSession>, +) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 + || !requests.iter().all(|request| { + is_region_scaled_direct_batch_candidate(request) + && request.fmt == first.fmt + && request.backend == first.backend + }) + { + return None; + } + if first.backend == BackendRequest::Auto + && !auto_metal_prechecked + && !should_auto_use_metal_for_region_scaled_direct_batch(requests) + { + return None; + } + + #[cfg(target_os = "macos")] + { + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal direct batch request specifications", + ); + let mut request_specs = match budget.try_vec( + requests.len(), + "J2K Metal direct batch request specifications", + ) { + Ok(specs) => specs, + Err(error) => return Some(Err(error.into())), + }; + for request in requests { + let BatchOp::RegionScaled { roi, scale } = request.op else { + return Some(Err(batch_scheduler_invariant( + "direct region-scaled batch contains a non-region-scaled request", + ))); + }; + request_specs.push((request.input.clone(), roi, scale)); + } + let result = match first.fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + crate::hybrid::decode_region_scaled_grayscale_batch_direct_to_device_routed( + &request_specs, + first.fmt, + backend, + ) + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => { + crate::hybrid::decode_region_scaled_color_batch_direct_to_device_routed( + &request_specs, + first.fmt, + backend, + ) + } + _ => Err(batch_scheduler_invariant( + "direct region-scaled batch contains an unsupported pixel format", + )), + }; + Some(result) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = backend; + None + } +} + +pub(super) fn decode_individual( + request: &QueuedRequest, + backend: Option<&MetalBackendSession>, +) -> Result { + let mut decoder = J2kDecoder::new(request.input.as_ref())?; + if let Some(backend) = backend { + return decoder.decode_request_to_device_with_session( + MetalDecodeRequest { + fmt: request.fmt, + op: request.op.into(), + backend: request.backend, + }, + backend, + ); + } + match request.op { + BatchOp::Full => decoder.decode_to_surface_impl(request.fmt, request.backend), + BatchOp::Region(roi) => { + decoder.decode_region_to_surface_impl(request.fmt, roi, request.backend) + } + BatchOp::Scaled(scale) => { + decoder.decode_scaled_to_surface_impl(request.fmt, scale, request.backend) + } + BatchOp::RegionScaled { roi, scale } => { + decoder.decode_region_scaled_to_surface_impl(request.fmt, roi, scale, request.backend) + } + } +} + +impl From for crate::MetalDecodeOp { + fn from(value: BatchOp) -> Self { + match value { + BatchOp::Full => Self::Full, + BatchOp::Region(roi) => Self::Region(roi), + BatchOp::Scaled(scale) => Self::Scaled(scale), + BatchOp::RegionScaled { roi, scale } => Self::RegionScaled { roi, scale }, + } + } +} diff --git a/crates/j2k-metal/src/batch/session.rs b/crates/j2k-metal/src/batch/session.rs new file mode 100644 index 00000000..d464315f --- /dev/null +++ b/crates/j2k-metal/src/batch/session.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::{Arc, Mutex, MutexGuard}; + +use j2k_core::{BackendRequest, DeviceSubmission, PixelFormat}; + +use crate::{profile, Error, MetalBackendSession, MetalSession, Surface}; + +use super::execute::process_batch; +use super::heuristics::group_metal_requests; +use super::request::{batch_scheduler_invariant, BatchOp, QueuedRequest}; + +#[derive(Default)] +pub(crate) struct SessionState { + pub(crate) submissions: u64, + pub(super) queued: Vec, + pub(super) completed: Vec>>, + pub(super) free_slots: Vec, +} + +#[derive(Clone)] +pub(crate) struct SharedSession { + state: Arc>, + backend: Option, +} + +impl Default for SharedSession { + fn default() -> Self { + Self { + state: Arc::new(Mutex::new(SessionState::default())), + backend: None, + } + } +} + +impl SharedSession { + #[cfg(target_os = "macos")] + pub(crate) fn with_backend_session(backend: MetalBackendSession) -> Self { + Self { + state: Arc::new(Mutex::new(SessionState::default())), + backend: Some(backend), + } + } + + pub(crate) fn lock(&self) -> Result, Error> { + self.state.lock().map_err(|_| Error::MetalStatePoisoned { + state: "J2K Metal session", + }) + } + + pub(crate) fn backend_session(&self) -> Option<&MetalBackendSession> { + self.backend.as_ref() + } +} + +pub struct MetalSubmission { + session: SharedSession, + pub(super) slot: Option, +} + +#[doc(hidden)] +impl DeviceSubmission for MetalSubmission { + type Output = Surface; + type Error = Error; + + fn wait(mut self) -> Result { + let slot = self.slot.take().ok_or(Error::MetalStateInvariant { + state: "J2K Metal submission", + reason: "submission slot was already consumed", + })?; + let mut session = self.session.lock()?; + flush_if_needed(&mut session, self.session.backend_session()); + take_surface(&mut session, slot) + } +} + +impl Drop for MetalSubmission { + fn drop(&mut self) { + let Some(slot) = self.slot.take() else { + return; + }; + let Ok(mut session) = self.session.lock() else { + return; + }; + if let Some(position) = session + .queued + .iter() + .position(|request| request.output_slot == slot) + { + session.queued.remove(position); + } + if let Some(completed) = session.completed.get_mut(slot) { + *completed = None; + } + let _release_result = release_surface_slot(&mut session, slot); + } +} + +pub(crate) fn queue_tile_request( + session: &mut MetalSession, + input: &[u8], + fmt: PixelFormat, + backend: BackendRequest, + op: BatchOp, +) -> Result { + queue_tile_request_shared(session, Arc::<[u8]>::from(input), fmt, backend, op) +} + +pub(crate) fn queue_tile_request_shared( + session: &mut MetalSession, + input: Arc<[u8]>, + fmt: PixelFormat, + backend: BackendRequest, + op: BatchOp, +) -> Result { + queue_tile_request_shared_with_retained(session, input, fmt, backend, op, 0) +} + +pub(crate) fn queue_tile_request_shared_with_retained( + session: &mut MetalSession, + input: Arc<[u8]>, + fmt: PixelFormat, + backend: BackendRequest, + op: BatchOp, + retained_submission_capacity: usize, +) -> Result { + let mut state = session.shared.lock()?; + let reuses_slot = !state.free_slots.is_empty(); + if !reuses_slot { + let slot_capacity = state.completed.len().checked_add(1).ok_or_else(|| { + Error::BatchInfrastructure(j2k_core::BatchInfrastructureError::AllocationTooLarge { + what: "J2K Metal reusable completion slots", + requested: usize::MAX, + cap: j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES, + }) + })?; + crate::batch_allocation::try_reserve_for_push( + &mut state.completed, + "J2K Metal queued completion slots", + )?; + j2k_core::try_batch_reserve_to( + &mut state.free_slots, + slot_capacity, + "J2K Metal reusable completion slots", + )?; + } + crate::batch_allocation::try_reserve_for_push(&mut state.queued, "J2K Metal queued requests")?; + let aggregate = + crate::batch_allocation::BatchMetadataBudget::new("J2K Metal queued request state"); + aggregate.preflight(&[ + crate::batch_allocation::BatchMetadataRequest::of::( + retained_submission_capacity, + ), + crate::batch_allocation::BatchMetadataRequest::of::(state.queued.capacity()), + crate::batch_allocation::BatchMetadataRequest::of::>>( + state.completed.capacity(), + ), + crate::batch_allocation::BatchMetadataRequest::of::(state.free_slots.capacity()), + ])?; + let slot = if reuses_slot { + state + .free_slots + .pop() + .ok_or_else(|| batch_scheduler_invariant("reusable slot disappeared"))? + } else { + let slot = state.completed.len(); + state.completed.push(None); + slot + }; + state + .queued + .push(QueuedRequest::new(input, fmt, backend, op, slot)); + Ok(MetalSubmission { + session: session.shared.clone(), + slot: Some(slot), + }) +} + +fn flush_if_needed(session: &mut SessionState, backend: Option<&MetalBackendSession>) { + if session.queued.is_empty() { + return; + } + + let profile_enabled = profile::metal_profile_stages_enabled(); + let queued = std::mem::take(&mut session.queued); + let request_count = queued.len(); + let mut slot_budget = + crate::batch_allocation::BatchMetadataBudget::new("J2K Metal grouping recovery slots"); + let mut output_slots = + match slot_budget.try_vec(queued.len(), "J2K Metal grouping recovery output slots") { + Ok(slots) => slots, + Err(error) => { + for request in queued { + session.completed[request.output_slot] = Some(Err(error.into())); + } + return; + } + }; + output_slots.extend(queued.iter().map(|request| request.output_slot)); + let group_started = profile::profile_now(profile_enabled); + let batches = match group_metal_requests(queued) { + Ok(batches) => batches, + Err(error) => { + for output_slot in output_slots { + session.completed[output_slot] = Some(Err(error.into())); + } + return; + } + }; + drop(output_slots); + if profile_enabled { + profile::emit_metal_batch_profile_row( + "decode", + &profile::MetalBatchProfileRow { + slice: "decode_batch", + stage: "group", + pipeline: "metal_cpu_hybrid", + processor: "scheduler", + route: "all", + backend: profile::MetalBatchProfileValue::Mixed, + fmt: profile::MetalBatchProfileValue::Mixed, + request_count, + output_count: batches.len(), + elapsed_us: profile::elapsed_us(group_started), + outcome: "grouped", + }, + ); + } + + for batch in batches { + process_batch(session, batch, backend); + } +} + +fn take_surface(session: &mut SessionState, slot: usize) -> Result { + let result = session + .completed + .get_mut(slot) + .and_then(Option::take) + .ok_or_else(|| Error::MetalKernel { + message: format!("missing queued J2K Metal surface for slot {slot}"), + }); + release_surface_slot(session, slot)?; + result? +} + +pub(super) fn release_surface_slot(session: &mut SessionState, slot: usize) -> Result<(), Error> { + if session.free_slots.contains(&slot) { + return Ok(()); + } + if session.free_slots.len() >= session.free_slots.capacity() { + return Err(Error::MetalStateInvariant { + state: "J2K Metal batch free-slot ledger", + reason: "slot creation did not retain capacity for its eventual release", + }); + } + session.free_slots.push(slot); + Ok(()) +} diff --git a/crates/j2k-metal/src/batch/tests.rs b/crates/j2k-metal/src/batch/tests.rs new file mode 100644 index 00000000..07c31b0e --- /dev/null +++ b/crates/j2k-metal/src/batch/tests.rs @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k_core::{BackendRequest, Downscale, PixelFormat, Rect}; + +use crate::{Error, MetalSession}; + +use super::execute::process_batch; +use super::heuristics::{ + auto_region_scaled_direct_metal_min_dim, can_decode_requests_as_repeated_region_scaled_batch, + group_metal_requests, profile_route_label, same_input_bytes, BatchRoute, GroupedRequests, + AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT, AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM, + AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT, + AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, +}; +use super::request::{BatchOp, QueuedRequest}; +use super::session::{queue_tile_request_shared, release_surface_slot, SessionState}; + +fn auto_rgb_region_scaled_request(input: Arc<[u8]>) -> QueuedRequest { + QueuedRequest::new( + input, + PixelFormat::Rgb8, + BackendRequest::Auto, + BatchOp::RegionScaled { + roi: Rect { + x: 128, + y: 128, + w: 512, + h: 256, + }, + scale: Downscale::Quarter, + }, + 0, + ) +} + +fn auto_rgb_region_scaled_request_with_max_dim( + input: Arc<[u8]>, + max_image_dim: u32, +) -> QueuedRequest { + let request = auto_rgb_region_scaled_request(input); + request.max_image_dim.set(Some(max_image_dim)).ok(); + request +} + +#[test] +#[expect( + clippy::cast_possible_truncation, + reason = "bounded test fixture index fits in u8" +)] +fn auto_region_scaled_rgb_threshold_requires_repeated_inputs() { + let requests = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) + .map(|idx| auto_rgb_region_scaled_request(Arc::from([idx as u8]))) + .collect::>(); + + assert!(!can_decode_requests_as_repeated_region_scaled_batch( + &requests + )); + assert_eq!( + auto_region_scaled_direct_metal_min_dim(&requests), + None, + "distinct RGB ROI+scaled Auto batches must stay CPU until hybrid wins for distinct inputs" + ); + + let shared = Arc::<[u8]>::from([1_u8]); + let repeated = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) + .map(|_| auto_rgb_region_scaled_request(shared.clone())) + .collect::>(); + assert!(can_decode_requests_as_repeated_region_scaled_batch( + &repeated + )); +} + +#[test] +fn auto_region_scaled_repeated_rgb_uses_measured_batch_two_metal_threshold() { + let shared = Arc::<[u8]>::from([1_u8]); + let repeated = (0..2) + .map(|_| auto_rgb_region_scaled_request_with_max_dim(shared.clone(), 512)) + .collect::>(); + + assert_eq!( + auto_region_scaled_direct_metal_min_dim(&repeated), + Some(512), + "measured repeated RGB ROI+scaled batches should route to Metal from batch 2 at 512px" + ); + + let single = vec![auto_rgb_region_scaled_request_with_max_dim(shared, 512)]; + assert_eq!(auto_region_scaled_direct_metal_min_dim(&single), None); +} + +#[test] +fn queued_request_caches_image_dimension_probe() { + let request = auto_rgb_region_scaled_request(Arc::from([0_u8])); + + assert!(!request.max_image_dim_cache_filled_for_test()); + assert_eq!(request.max_image_dim(), None); + assert!(request.max_image_dim_cache_filled_for_test()); + assert_eq!(request.max_image_dim(), None); +} + +#[test] +fn repeated_input_check_uses_pointer_identity_before_fingerprint() { + let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); + let first = auto_rgb_region_scaled_request(shared.clone()); + let next = auto_rgb_region_scaled_request(shared); + + assert!(same_input_bytes(&first, &next)); + assert!(!first.input_fingerprint_cache_filled_for_test()); + assert!(!next.input_fingerprint_cache_filled_for_test()); +} + +#[test] +fn dropping_an_unwaited_submission_releases_and_reuses_its_session_slot() { + let mut session = MetalSession::default(); + let submission = queue_tile_request_shared( + &mut session, + Arc::<[u8]>::from([0xff_u8, 0x4f]), + PixelFormat::Gray8, + BackendRequest::Cpu, + BatchOp::Full, + ) + .expect("queue submission"); + let first_slot = submission.slot.expect("active submission slot"); + assert_eq!(session.shared.lock().expect("session").queued.len(), 1); + + drop(submission); + + let state = session.shared.lock().expect("session"); + assert!(state.queued.is_empty()); + assert_eq!(state.free_slots, [first_slot]); + drop(state); + + let next = queue_tile_request_shared( + &mut session, + Arc::<[u8]>::from([0xff_u8, 0x4f]), + PixelFormat::Gray8, + BackendRequest::Cpu, + BatchOp::Full, + ) + .expect("reuse submission slot"); + assert_eq!(next.slot, Some(first_slot)); +} + +#[test] +fn slot_release_reports_missing_reserved_capacity_without_panicking() { + let mut state = SessionState::default(); + + assert!(matches!( + release_surface_slot(&mut state, 0), + Err(Error::MetalStateInvariant { + state: "J2K Metal batch free-slot ledger", + .. + }) + )); + assert!(state.free_slots.is_empty()); +} + +#[test] +fn auto_region_scaled_grouping_preserves_repeated_rgb_metal_decision() { + let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); + let requests = (0..AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT) + .map(|_| { + auto_rgb_region_scaled_request_with_max_dim( + shared.clone(), + AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, + ) + }) + .collect::>(); + + let grouped = group_metal_requests(requests).expect("group requests"); + + assert_eq!(grouped.len(), 1); + assert_eq!( + grouped[0].route, + BatchRoute::AutoRepeatedRegionScaledDirectMetal + ); + assert_eq!( + grouped[0].requests.len(), + AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT + ); + assert!( + grouped[0] + .requests + .iter() + .all(|request| !request.input_fingerprint_cache_filled_for_test()), + "shared repeated inputs should be classified by Arc identity without fingerprinting" + ); +} + +#[test] +#[expect( + clippy::cast_possible_truncation, + reason = "bounded test fixture index fits in u8" +)] +fn auto_region_scaled_distinct_rgb_grouping_preserves_cpu_decision() { + let requests = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) + .map(|idx| { + auto_rgb_region_scaled_request_with_max_dim( + Arc::from([idx as u8]), + AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM, + ) + }) + .collect::>(); + + let grouped = group_metal_requests(requests).expect("group requests"); + + assert_eq!(grouped.len(), 1); + assert_eq!(grouped[0].route, BatchRoute::AutoRegionScaledDirectCpu); + assert_eq!( + grouped[0].requests.len(), + AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT + ); +} + +#[test] +fn profile_route_labels_are_stable_for_decode_batch_slices() { + assert_eq!(profile_route_label(BatchRoute::Generic), "generic"); + assert_eq!( + profile_route_label(BatchRoute::AutoRegionScaledDirectCpu), + "auto_region_scaled_direct_cpu" + ); + assert_eq!( + profile_route_label(BatchRoute::AutoRegionScaledDirectMetal), + "auto_region_scaled_direct_metal" + ); + assert_eq!( + profile_route_label(BatchRoute::AutoRepeatedRegionScaledDirectMetal), + "auto_repeated_region_scaled_direct_metal" + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_region_scaled_prechecked_error_does_not_retry_generic_direct_path() { + let _guard = crate::hybrid::region_scaled_color_plan_test_lock_for_test(); + crate::hybrid::reset_region_scaled_color_plan_builds_for_test(); + let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); + let requests = (0..AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT) + .map(|slot| { + let mut request = auto_rgb_region_scaled_request_with_max_dim( + shared.clone(), + AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, + ); + request.output_slot = slot; + request + }) + .collect::>(); + let mut session = SessionState { + submissions: 0, + queued: Vec::new(), + completed: (0..requests.len()).map(|_| None).collect(), + free_slots: Vec::new(), + }; + + process_batch( + &mut session, + GroupedRequests { + route: BatchRoute::AutoRepeatedRegionScaledDirectMetal, + requests, + }, + None, + ); + + assert_eq!( + crate::hybrid::region_scaled_color_plan_builds_for_test(), + 1, + "failed prechecked Auto Metal routing should fall back to CPU without retrying generic direct Metal" + ); + assert!( + session + .completed + .iter() + .all(|result| matches!(result, Some(Err(_)))), + "invalid inputs should be surfaced on every fallback request" + ); +} diff --git a/crates/j2k-metal/src/batch_decoder.rs b/crates/j2k-metal/src/batch_decoder.rs new file mode 100644 index 00000000..39270e87 --- /dev/null +++ b/crates/j2k-metal/src/batch_decoder.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{ + BatchColor, BatchDecodeOptions, BatchGroupInfo, BatchLayout, DecodeRequest, EncodedImage, + IndexedBatchError, J2kDecodeWarning, PreparedBatch, PreparedBatchGroup, PreparedImage, +}; +use j2k_core::{DeviceSubmission, PixelFormat, Rect}; + +#[cfg(target_os = "macos")] +use j2k_metal_support::{MetalImageDestination, MetalImageLayout, ResidentMetalImage}; + +#[cfg(target_os = "macos")] +use metal::{Buffer, DeviceRef}; + +use crate::{Error, MetalBackendSession, Surface}; + +mod contracts; +mod decoder; +#[cfg(all(test, target_os = "macos"))] +mod encoder_count_tests; +#[cfg(target_os = "macos")] +mod external; +#[cfg(target_os = "macos")] +mod plan_cache; +#[cfg(all(test, target_os = "macos"))] +mod queue_ordering_tests; +#[cfg(target_os = "macos")] +mod resident; +#[cfg(target_os = "macos")] +mod submission; + +#[cfg(test)] +mod decoder_ownership_tests { + #[test] + fn persistent_decoder_does_not_layer_the_legacy_metal_session() { + let decoder_source = include_str!("batch_decoder/decoder.rs"); + let legacy_session_type = ["Metal", "Session"].concat(); + + assert!( + !decoder_source.contains(&legacy_session_type), + "MetalBatchDecoder must own only MetalBackendSession and its direct counter" + ); + assert!(decoder_source.contains("pub(super) backend: MetalBackendSession")); + assert!(decoder_source.contains("submission_count: u64")); + } +} + +#[cfg(target_os = "macos")] +pub use self::contracts::MetalResidentBatch; +pub use self::contracts::{ + MetalBatchDecodeResult, MetalBatchGroup, MetalBatchGroupCompletion, MetalBatchGroupError, + MetalBatchGroupParts, +}; +pub use self::decoder::MetalBatchDecoder; +#[cfg(target_os = "macos")] +pub use self::submission::{SubmittedMetalGroupDecodeInto, SubmittedMetalPreparedBatch}; + +use self::contracts::validate_group_contract; +#[cfg(target_os = "macos")] +use self::plan_cache::{PreparedColorPlanCache, PreparedGrayPlanCache}; +#[cfg(target_os = "macos")] +use self::submission::{ + allocate_codec_owned_group_destination, validate_codec_owned_resident_group, + MetalResidentGroupMetadata, SubmittedMetalResidentGroup, +}; +#[cfg(test)] +mod batch_contract_tests { + use super::*; + use j2k::{BatchAlpha, NativeSampleType}; + + fn color_info( + color: BatchColor, + sample_type: NativeSampleType, + signed: bool, + ) -> BatchGroupInfo { + BatchGroupInfo { + dimensions: (2, 2), + color, + alpha: if color == BatchColor::Rgba { + BatchAlpha::Straight + } else { + BatchAlpha::None + }, + precision: if sample_type == NativeSampleType::U8 { + 8 + } else { + 12 + }, + signed, + sample_type, + layout: BatchLayout::Nchw, + colorspace: j2k_core::Colorspace::Rgb, + route: j2k::BatchCodecRoute::Htj2k, + transform: j2k::BatchWaveletTransform::Reversible53, + transfer_syntax: j2k_core::CompressedTransferSyntax::HtJpeg2000Lossless, + payload_kind: j2k_core::CompressedPayloadKind::Jpeg2000Codestream, + } + } + + #[test] + fn exact_color_contract_maps_all_native_rgb_and_rgba_formats() { + assert_eq!( + validate_group_contract(&color_info(BatchColor::Rgb, NativeSampleType::I16, true)) + .expect("signed RGB must have an exact native Metal format"), + PixelFormat::RgbI16 + ); + for (sample_type, signed, expected) in [ + (NativeSampleType::U8, false, PixelFormat::Rgba8), + (NativeSampleType::U16, false, PixelFormat::Rgba16), + (NativeSampleType::I16, true, PixelFormat::RgbaI16), + ] { + assert_eq!( + validate_group_contract(&color_info(BatchColor::Rgba, sample_type, signed)) + .expect("RGBA must have an exact native Metal format"), + expected + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn external_ht_gray_group_uses_one_stacked_component_graph() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let bytes = j2k_test_support::openhtj2k_refinement_fixture(); + let options = BatchDecodeOptions { + settings: j2k::DecodeSettings::lenient(), + ..BatchDecodeOptions::default() + }; + let mut decoder = MetalBatchDecoder::system_default_with_options(options) + .expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(Arc::<[u8]>::from(bytes)), + EncodedImage::full(Arc::<[u8]>::from(bytes)), + ]) + .expect("prepare independent HT grayscale group"); + let group = &prepared.groups()[0]; + let (width, height) = group.info().dimensions; + let image_bytes = usize::try_from(width) + .expect("width") + .checked_mul(usize::try_from(height).expect("height")) + .expect("image bytes"); + let output = j2k_metal_support::checked_shared_buffer( + decoder.backend_session().device(), + image_bytes.checked_mul(2).expect("group bytes"), + ) + .expect("external group output"); + let layout = MetalImageLayout::new_batch( + 0, + (width, height), + width as usize, + PixelFormat::Gray8, + 2, + image_bytes, + ) + .expect("dense grayscale group layout"); + // SAFETY: the fresh allocation remains exclusively owned by the + // submitted codec work until completion. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(output.clone(), layout) + .expect("external group destination") + }; + + crate::compute::reset_stacked_component_batches_for_test(); + let completion = decoder + .submit_prepared_group_into(group, destination) + .expect("submit external group") + .wait() + .expect("complete external group"); + + assert_eq!( + crate::compute::stacked_component_batches_for_test(), + 1, + "two homogeneous HT images must share one stacked component graph" + ); + assert_eq!( + completion.decoded_rects(), + group + .images() + .iter() + .map(|image| image.plan().output_rect()) + .collect::>() + ); + assert_eq!( + completion.warnings(), + [ + vec![J2kDecodeWarning::LenientDecodeMode], + vec![J2kDecodeWarning::LenientDecodeMode], + ] + ); + + // The synchronous convenience API must use the same coalesced graph, + // not the legacy per-image destination loop. + // SAFETY: the prior submission completed and released this allocation; + // the synchronous call retires its work before returning. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(output, layout) + .expect("reused external group destination") + }; + crate::compute::reset_stacked_component_batches_for_test(); + decoder + .decode_prepared_group_into(group, &destination) + .expect("synchronous external group decode"); + assert_eq!(crate::compute::stacked_component_batches_for_test(), 1); + } + + #[cfg(target_os = "macos")] + #[test] + fn external_ht_rgb_group_stacks_each_component_across_images() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-rgb-s12-53-single-raw") + .expect("independent signed RGB fixture"); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(Arc::<[u8]>::from(fixture.encoded)), + EncodedImage::full(Arc::<[u8]>::from(fixture.encoded)), + ]) + .expect("prepare independent HT RGB group"); + let group = &prepared.groups()[0]; + let (width, height) = group.info().dimensions; + let row_bytes = usize::try_from(width) + .expect("width") + .checked_mul(PixelFormat::RgbI16.bytes_per_pixel()) + .expect("row bytes"); + let image_bytes = row_bytes + .checked_mul(usize::try_from(height).expect("height")) + .expect("image bytes"); + let output = j2k_metal_support::checked_shared_buffer( + decoder.backend_session().device(), + image_bytes.checked_mul(2).expect("group bytes"), + ) + .expect("external RGB group output"); + let layout = MetalImageLayout::new_batch( + 0, + (width, height), + row_bytes, + PixelFormat::RgbI16, + 2, + image_bytes, + ) + .expect("dense RGB group layout"); + // SAFETY: the fresh allocation remains exclusively owned by the + // submitted codec work until completion. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(output.clone(), layout) + .expect("external RGB group destination") + }; + + crate::compute::reset_stacked_component_batches_for_test(); + decoder + .submit_prepared_group_into(group, destination) + .expect("submit external RGB group") + .wait() + .expect("complete external RGB group"); + + assert_eq!( + crate::compute::stacked_component_batches_for_test(), + 3, + "RGB components must each coalesce the two image plans" + ); + // SAFETY: codec completion released the exclusive destination before + // this host-only parity check. + let bytes = unsafe { + j2k_metal_support::checked_buffer_read_vec::( + &output, + layout.byte_offset(), + layout.byte_len(), + ) + .expect("completed RGB group bytes") + }; + assert_eq!( + &bytes[..image_bytes], + &bytes[image_bytes..], + "independently prepared identical inputs must remain identical after stacking" + ); + } +} diff --git a/crates/j2k-metal/src/batch_decoder/contracts.rs b/crates/j2k-metal/src/batch_decoder/contracts.rs new file mode 100644 index 00000000..9d681f15 --- /dev/null +++ b/crates/j2k-metal/src/batch_decoder/contracts.rs @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Shared Metal batch result and metadata contracts. + +use super::{ + BatchDecodeOptions, BatchGroupInfo, BatchLayout, Buffer, Error, IndexedBatchError, + J2kDecodeWarning, PixelFormat, PreparedBatchGroup, Rect, ResidentMetalImage, Surface, +}; + +pub(super) fn validate_group_contract(info: &BatchGroupInfo) -> Result { + if !matches!(info.layout, BatchLayout::Nchw | BatchLayout::Nhwc) { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal batch received an unknown output layout", + }); + } + info.native_pixel_format() + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal batch metadata contains an unsupported color/sample combination", + }) +} + +/// Codec metadata released only after a caller-owned Metal group destination +/// has completed and its device status has been validated. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MetalBatchGroupCompletion { + pub(super) decoded_rects: Vec, + pub(super) warnings: Vec>, +} + +impl MetalBatchGroupCompletion { + pub(super) fn from_prepared(group: &PreparedBatchGroup, options: BatchDecodeOptions) -> Self { + let decoded_rects = group + .images() + .iter() + .map(|image| image.plan().output_rect()) + .collect(); + let warnings = group + .images() + .iter() + .map(|_| { + if options.settings.lenient_tolerance_enabled() { + vec![J2kDecodeWarning::LenientDecodeMode] + } else { + Vec::new() + } + }) + .collect(); + Self { + decoded_rects, + warnings, + } + } + + /// Actual decoded source rectangle for every completed destination item. + #[must_use] + pub fn decoded_rects(&self) -> &[Rect] { + &self.decoded_rects + } + + /// Non-fatal codec warnings for every completed destination item. + #[must_use] + pub fn warnings(&self) -> &[Vec] { + &self.warnings + } + + /// Consume the completion into rectangles and warnings in batch order. + #[must_use] + pub fn into_parts(self) -> (Vec, Vec>) { + (self.decoded_rects, self.warnings) + } +} + +/// One completed dense codec-owned Metal batch allocation. +/// +/// The byte ordering is described by the owning [`MetalBatchGroup`]'s +/// [`BatchGroupInfo`], including its NCHW or NHWC layout. The allocation is +/// logically immutable after codec completion; raw Metal access therefore +/// remains an explicit unsafe interop boundary. +#[cfg(target_os = "macos")] +#[derive(Clone)] +pub struct MetalResidentBatch { + pub(super) storage: ResidentMetalImage, +} + +#[cfg(target_os = "macos")] +impl core::fmt::Debug for MetalResidentBatch { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MetalResidentBatch") + .field("device_registry_id", &self.device_registry_id()) + .field("byte_offset", &self.byte_offset()) + .field("byte_len", &self.byte_len()) + .field("image_count", &self.image_count()) + .field("image_stride_bytes", &self.image_stride_bytes()) + .finish_non_exhaustive() + } +} + +#[cfg(target_os = "macos")] +impl MetalResidentBatch { + /// Byte offset of the dense group inside the retained allocation. + #[must_use] + pub const fn byte_offset(&self) -> usize { + self.storage.layout().byte_offset() + } + + /// Number of bytes occupied by the complete dense group. + #[must_use] + pub const fn byte_len(&self) -> usize { + self.storage.layout().byte_len() + } + + /// Number of images in the dense batch dimension. + #[must_use] + pub const fn image_count(&self) -> usize { + self.storage.layout().image_count() + } + + /// Byte distance between consecutive images in the dense batch. + #[must_use] + pub const fn image_stride_bytes(&self) -> usize { + self.storage.layout().image_stride_bytes() + } + + /// Registry identifier of the Metal device that owns the allocation. + #[must_use] + pub fn device_registry_id(&self) -> u64 { + self.storage.device_registry_id() + } + + /// Borrow the completed Metal allocation for read-only GPU interop. + /// + /// # Safety + /// + /// The caller may bind the handle only for reads and must retain this + /// value (or a clone) until that GPU work completes. No CPU or GPU writer + /// may access the allocation while any resident-batch owner exists. + #[must_use] + pub unsafe fn metal_buffer(&self) -> &Buffer { + // SAFETY: the caller accepts the immutable raw-handle contract above. + unsafe { self.storage.raw_buffer() } + } +} + +/// One successfully decoded homogeneous Metal-resident output group. +pub struct MetalBatchGroup { + pub(super) info: BatchGroupInfo, + pub(super) source_indices: Vec, + pub(super) decoded_rects: Vec, + pub(super) warnings: Vec>, + pub(super) surfaces: Vec, + #[cfg(target_os = "macos")] + pub(super) resident_batch: MetalResidentBatch, +} + +/// Owned parts returned when consuming one homogeneous Metal batch group. +pub type MetalBatchGroupParts = ( + BatchGroupInfo, + Vec, + Vec, + Vec>, + Vec, +); + +/// Failure while executing one homogeneous Metal group. +/// +/// No partially written output from the affected group is exposed. Other +/// prepared groups may still succeed when the retained Metal session remains +/// usable. +#[derive(Debug, thiserror::Error)] +#[error("Metal batch group containing source indices {source_indices:?} failed: {source}")] +pub struct MetalBatchGroupError { + pub(super) source_indices: Vec, + #[source] + pub(super) source: Box, +} + +impl MetalBatchGroupError { + pub(super) fn new(group: &PreparedBatchGroup, source: Error) -> Self { + Self { + source_indices: group.source_indices().to_vec(), + source: Box::new(source), + } + } + + /// Original input indices whose dense group output was discarded. + #[must_use] + pub fn source_indices(&self) -> &[usize] { + &self.source_indices + } + + /// Strict Metal adapter or runtime failure for this group. + #[must_use] + pub fn source(&self) -> &Error { + &self.source + } + + /// Consume the group failure into affected indices and its source. + #[must_use] + pub fn into_parts(self) -> (Vec, Error) { + (self.source_indices, *self.source) + } +} + +impl core::fmt::Debug for MetalBatchGroup { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut debug = f.debug_struct("MetalBatchGroup"); + debug + .field("info", &self.info) + .field("source_indices", &self.source_indices) + .field("decoded_rects", &self.decoded_rects) + .field("warnings", &self.warnings) + .field("surface_count", &self.surfaces.len()); + #[cfg(target_os = "macos")] + debug.field("resident_batch", &self.resident_batch); + debug.finish() + } +} + +impl MetalBatchGroup { + /// Shared native-width output metadata. + pub fn info(&self) -> &BatchGroupInfo { + &self.info + } + + /// Original input indices in the resident batch dimension. + pub fn source_indices(&self) -> &[usize] { + &self.source_indices + } + + /// Actual decoded rectangle for every resident image. + pub fn decoded_rects(&self) -> &[Rect] { + &self.decoded_rects + } + + /// Non-fatal warnings for every resident image. + pub fn warnings(&self) -> &[Vec] { + &self.warnings + } + + /// Interleaved Metal-resident image views in group order. + /// + /// Gray groups and NHWC color groups expose one convenience [`Surface`] + /// per image. NCHW color groups return an empty slice because a [`Surface`] + /// promises interleaved pixel semantics; use [`Self::resident_batch`] for + /// their dense planar allocation. + pub fn surfaces(&self) -> &[Surface] { + &self.surfaces + } + + /// Completed dense codec-owned Metal storage for this group. + /// + /// Its byte ordering is exactly `self.info().layout`; no decoded host + /// transfer or final device copy is performed to construct this owner. + #[cfg(target_os = "macos")] + #[must_use] + pub fn resident_batch(&self) -> Option<&MetalResidentBatch> { + Some(&self.resident_batch) + } + + /// Consume this group and retain its dense codec-owned Metal storage. + #[cfg(target_os = "macos")] + #[must_use] + pub fn into_resident_batch(self) -> Option { + Some(self.resident_batch) + } + + /// Consume this group into metadata, indices, rectangles, warnings, and surfaces. + pub fn into_parts(self) -> MetalBatchGroupParts { + ( + self.info, + self.source_indices, + self.decoded_rects, + self.warnings, + self.surfaces, + ) + } +} + +/// Successful resident groups plus indexed input preparation failures. +#[derive(Debug)] +pub struct MetalBatchDecodeResult { + pub(super) groups: Vec, + pub(super) errors: Vec, + pub(super) group_errors: Vec, +} + +impl MetalBatchDecodeResult { + /// Successfully decoded homogeneous groups. + pub fn groups(&self) -> &[MetalBatchGroup] { + &self.groups + } + + /// Indexed parsing, planning, and representability failures. + pub fn errors(&self) -> &[IndexedBatchError] { + &self.errors + } + + /// Homogeneous groups that failed during Metal execution. + #[must_use] + pub fn group_errors(&self) -> &[MetalBatchGroupError] { + &self.group_errors + } + + /// Consume the result into resident groups, indexed preparation failures, + /// and homogeneous execution failures. + #[must_use] + pub fn into_parts( + self, + ) -> ( + Vec, + Vec, + Vec, + ) { + (self.groups, self.errors, self.group_errors) + } +} diff --git a/crates/j2k-metal/src/batch_decoder/decoder.rs b/crates/j2k-metal/src/batch_decoder/decoder.rs new file mode 100644 index 00000000..979bc61d --- /dev/null +++ b/crates/j2k-metal/src/batch_decoder/decoder.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Persistent shared-contract Metal batch decoder. + +use super::{ + BatchDecodeOptions, EncodedImage, Error, MetalBackendSession, MetalBatchDecodeResult, + MetalBatchGroup, MetalBatchGroupError, MetalImageDestination, PixelFormat, PreparedBatch, + PreparedBatchGroup, PreparedColorPlanCache, PreparedGrayPlanCache, PreparedImage, + SubmittedMetalPreparedBatch, +}; + +/// Persistent Metal batch decoder. +/// +/// The decoder retains one backend session across calls, including its Metal +/// runtime, command queue, pipelines, lookup tables, scratch pools, and prepared +/// direct-plan caches. Inputs queued in one call are grouped by the existing +/// distinct/repeated HTJ2K batch scheduler. +pub struct MetalBatchDecoder { + pub(super) backend: MetalBackendSession, + pub(super) options: BatchDecodeOptions, + submission_count: u64, + #[cfg(target_os = "macos")] + pub(super) prepared_gray_plans: PreparedGrayPlanCache, + #[cfg(target_os = "macos")] + pub(super) prepared_color_plans: PreparedColorPlanCache, +} + +impl core::fmt::Debug for MetalBatchDecoder { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut debug = f.debug_struct("MetalBatchDecoder"); + debug.field("backend", &self.backend); + debug.field("options", &self.options); + debug.field("submissions", &self.submission_count); + #[cfg(target_os = "macos")] + debug.field("prepared_gray_plans", &self.prepared_gray_plans.len()); + #[cfg(target_os = "macos")] + debug.field("prepared_color_plans", &self.prepared_color_plans.len()); + debug.finish_non_exhaustive() + } +} + +impl MetalBatchDecoder { + /// Create a persistent decoder using the system default Metal device. + pub fn system_default() -> Result { + Self::system_default_with_options(BatchDecodeOptions::default()) + } + + /// Create a persistent decoder with retained shared batch policy. + pub fn system_default_with_options(options: BatchDecodeOptions) -> Result { + let backend = MetalBackendSession::system_default()?; + #[cfg(target_os = "macos")] + { + Ok(Self::with_backend_session_and_options(backend, options)) + } + #[cfg(not(target_os = "macos"))] + { + let _ = backend; + let _ = options; + Err(Error::MetalUnavailable) + } + } + + /// Create a persistent decoder from an existing backend session. + #[cfg(target_os = "macos")] + pub fn with_backend_session(backend: MetalBackendSession) -> Self { + Self::with_backend_session_and_options(backend, BatchDecodeOptions::default()) + } + + /// Create a persistent decoder from an existing backend session and + /// retain its shared batch policy. + #[cfg(target_os = "macos")] + pub fn with_backend_session_and_options( + backend: MetalBackendSession, + options: BatchDecodeOptions, + ) -> Self { + Self { + backend, + options, + submission_count: 0, + prepared_gray_plans: PreparedGrayPlanCache::new( + super::plan_cache::PREPARED_BATCH_PLAN_CACHE_CAP, + ), + prepared_color_plans: PreparedColorPlanCache::new( + super::plan_cache::PREPARED_BATCH_PLAN_CACHE_CAP, + ), + } + } + + /// Shared preparation and output policy retained by this session. + #[must_use] + pub const fn options(&self) -> BatchDecodeOptions { + self.options + } + + /// Backend session retained by this decoder. + #[cfg(target_os = "macos")] + pub fn backend_session(&self) -> &MetalBackendSession { + &self.backend + } + + /// Number of grouped codec submissions completed by the retained session. + pub fn submissions(&self) -> Result { + Ok(self.submission_count) + } + + pub(super) fn record_submission(&mut self) { + self.submission_count = self.submission_count.saturating_add(1); + } + + /// Parse, validate, and group shared codec inputs for repeated Metal decode. + pub fn prepare(&self, inputs: Vec) -> Result { + j2k::prepare_batch(inputs, self.options).map_err(Error::from) + } + + /// Regroup caller-supplied prepared images under this session's retained + /// settings and output policy without reparsing their encoded bytes. + pub fn prepare_prepared_images( + &self, + images: Vec, + ) -> Result { + j2k::prepare_batch_from_images(images, self.options).map_err(Error::from) + } + + /// Prepare and decode owned codec inputs into Metal-resident groups. + pub fn decode_batch( + &mut self, + inputs: Vec, + ) -> Result { + let prepared = self.prepare(inputs)?; + self.decode_prepared(&prepared) + } + + /// Regroup and decode caller-supplied prepared images without reparsing + /// their encoded bytes. + pub fn decode_prepared_images( + &mut self, + images: Vec, + ) -> Result { + let prepared = self.prepare_prepared_images(images)?; + self.decode_prepared(&prepared) + } + + /// Decode a reusable shared codec batch without consuming its inputs or plans. + pub fn decode_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result { + #[cfg(target_os = "macos")] + { + self.submit_prepared(prepared)?.wait() + } + #[cfg(not(target_os = "macos"))] + { + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K persistent Metal prepared batch output", + ); + let mut groups = budget.try_vec( + prepared.groups().len(), + "J2K persistent Metal prepared output groups", + )?; + let mut group_errors = budget.try_vec( + prepared.groups().len(), + "J2K persistent Metal prepared group execution failures", + )?; + for group in prepared.groups() { + match self.decode_prepared_group_with_options(group, prepared.options()) { + Ok(decoded) => groups.push(decoded), + Err(source) if source.session_is_unusable() => return Err(source), + Err(source) => group_errors.push(MetalBatchGroupError::new(group, source)), + } + } + Ok(MetalBatchDecodeResult { + groups, + errors: prepared.errors().to_vec(), + group_errors, + }) + } + } + + /// Commit every representable shared prepared group to codec-owned Metal + /// output storage without waiting on the CPU. + /// + /// Indexed preflight failures are retained in the returned guard. A + /// non-fatal group submission failure is retained alongside other pending + /// groups; a session-fatal failure aborts submission after safely retiring + /// any groups already committed. + #[cfg(target_os = "macos")] + pub fn submit_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result { + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K submitted shared prepared Metal batch", + ); + let mut pending_groups = budget.try_vec( + prepared.groups().len(), + "J2K submitted shared prepared Metal groups", + )?; + let mut errors = budget.try_vec( + prepared.errors().len(), + "J2K submitted shared prepared indexed errors", + )?; + errors.extend_from_slice(prepared.errors()); + let mut group_errors = budget.try_vec( + prepared.groups().len(), + "J2K submitted shared prepared group errors", + )?; + + for group in prepared.groups() { + match self.submit_prepared_resident_group(group, prepared.options()) { + Ok(pending) => pending_groups.push(pending), + Err(source) if source.session_is_unusable() => return Err(source), + Err(source) => group_errors.push(MetalBatchGroupError::new(group, source)), + } + } + Ok(SubmittedMetalPreparedBatch { + pending_groups, + errors, + group_errors, + }) + } + + /// Prepare and commit one shared encoded batch to codec-owned Metal output + /// storage without waiting on the CPU. + #[cfg(target_os = "macos")] + pub fn submit_batch( + &mut self, + inputs: Vec, + ) -> Result { + let prepared = self.prepare(inputs)?; + self.submit_prepared(&prepared) + } + + /// Decode one homogeneous shared codec group using strict session defaults. + pub fn decode_prepared_group( + &mut self, + group: &PreparedBatchGroup, + ) -> Result { + self.decode_prepared_group_with_options(group, group.options()) + } + /// Validate a caller-owned destination before a direct final-store encode. + /// + /// This establishes the checked external-write handoff used by framework + /// adapters. Validation does not submit work or expose the raw buffer. + #[cfg(target_os = "macos")] + pub fn validate_destination( + &self, + destination: &MetalImageDestination, + dimensions: (u32, u32), + pixel_format: PixelFormat, + ) -> Result<(), Error> { + destination + .validate_device(self.backend_session().device()) + .and_then(|()| destination.validate_image(dimensions, pixel_format)) + .map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K Metal external decode destination validation failed", + source, + ) + }) + } +} + +impl j2k::BatchDecoder for MetalBatchDecoder { + type Output = MetalBatchDecodeResult; + type Error = Error; + + fn options(&self) -> BatchDecodeOptions { + self.options + } + + fn decode_prepared(&mut self, prepared: &PreparedBatch) -> Result { + MetalBatchDecoder::decode_prepared(self, prepared) + } +} diff --git a/crates/j2k-metal/src/batch_decoder/encoder_count_tests.rs b/crates/j2k-metal/src/batch_decoder/encoder_count_tests.rs new file mode 100644 index 00000000..596a95fe --- /dev/null +++ b/crates/j2k-metal/src/batch_decoder/encoder_count_tests.rs @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{ + wrap_j2k_codestream, BatchDecodeOptions, CpuBatchDecoder, CpuBatchSamples, EncodedImage, + J2kChannelAssociation, J2kChannelDefinition, J2kChannelType, J2kFileBoxMetadata, + J2kFileColorSpec, J2kFileWrapOptions, +}; +use j2k_core::{Colorspace, PixelFormat}; +use j2k_metal_support::{MetalImageDestination, MetalImageLayout}; +use j2k_native::{encode, encode_htj2k, EncodeOptions}; + +use super::MetalBatchDecoder; + +#[derive(Clone, Copy, Debug)] +enum FixtureColor { + Gray, + Rgb, + Rgba, +} + +#[derive(Clone, Copy, Debug)] +enum FixtureRoute { + Classic, + Ht, +} + +impl FixtureColor { + fn channels(self) -> usize { + match self { + Self::Gray => 1, + Self::Rgb => 3, + Self::Rgba => 4, + } + } + + fn format(self) -> PixelFormat { + match self { + Self::Gray => PixelFormat::Gray8, + Self::Rgb => PixelFormat::Rgb8, + Self::Rgba => PixelFormat::Rgba8, + } + } +} + +fn distinct_j2k(route: FixtureRoute, color: FixtureColor, seed: u8) -> Arc<[u8]> { + const WIDTH: u32 = 8; + const HEIGHT: u32 = 8; + + let channels = color.channels(); + let mut pixels = Vec::with_capacity(WIDTH as usize * HEIGHT as usize * channels); + for y in 0..HEIGHT { + let y = u8::try_from(y).expect("fixture height fits u8"); + for x in 0..WIDTH { + let x = u8::try_from(x).expect("fixture width fits u8"); + for channel in 0..channels { + let channel = u8::try_from(channel).expect("fixture channel count fits u8"); + pixels.push( + x.wrapping_mul(17) + .wrapping_add(y.wrapping_mul(29)) + .wrapping_add(channel.wrapping_mul(37)) + .wrapping_add(seed), + ); + } + } + } + let channels = u16::try_from(channels).expect("fixture channel count fits u16"); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + use_mct: matches!(color, FixtureColor::Rgb), + ..EncodeOptions::default() + }; + let codestream = match route { + FixtureRoute::Classic => encode(&pixels, WIDTH, HEIGHT, channels, 8, false, &options), + FixtureRoute::Ht => encode_htj2k(&pixels, WIDTH, HEIGHT, channels, 8, false, &options), + } + .expect("encode structural J2K fixture"); + if !matches!(color, FixtureColor::Rgba) { + return Arc::from(codestream); + } + + let channel_definitions = [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 3, + channel_type: J2kChannelType::Opacity, + association: J2kChannelAssociation::WholeImage, + }, + ]; + Arc::from( + wrap_j2k_codestream( + &codestream, + match route { + FixtureRoute::Classic => J2kFileWrapOptions::jp2(), + FixtureRoute::Ht => J2kFileWrapOptions::jph(), + } + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &channel_definitions, + }), + ) + .expect("wrap structural HTJ2K RGBA fixture"), + ) +} + +fn assert_external_group_uses_one_command_buffer_and_encoder( + route: FixtureRoute, + color: FixtureColor, + count: usize, +) -> Option<(usize, usize)> { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return None; + } + + let options = BatchDecodeOptions::default(); + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let encoded = (0..count) + .map(|index| { + distinct_j2k( + route, + color, + u8::try_from(index).expect("structural batch index fits u8"), + ) + }) + .collect::>(); + let inputs = encoded + .iter() + .cloned() + .map(EncodedImage::full) + .collect::>(); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(inputs.clone()) + .expect("CPU structural group oracle"); + assert!(expected.errors().is_empty(), "{:?}", expected.errors()); + assert_eq!(expected.groups().len(), 1); + let prepared = decoder.prepare(inputs).expect("prepare structural group"); + assert!(prepared.errors().is_empty(), "{:?}", prepared.errors()); + assert_eq!(prepared.groups().len(), 1); + let group = &prepared.groups()[0]; + let (width, height) = group.info().dimensions; + let format = color.format(); + let row_bytes = width as usize * format.bytes_per_pixel(); + let image_bytes = row_bytes * height as usize; + let output = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + image_bytes * count, + ) + .expect("structural destination allocation"); + let layout = + MetalImageLayout::new_batch(0, (width, height), row_bytes, format, count, image_bytes) + .expect("structural destination layout"); + // SAFETY: the fresh allocation has one logical writer; the retained raw + // handle is not read until completion releases the destination guard. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(output.clone(), layout) + .expect("structural destination") + }; + + crate::compute::reset_metal_command_buffers_for_test(); + crate::compute::reset_metal_compute_encoders_for_test(); + let completion = decoder + .submit_prepared_group_into(group, destination) + .expect("submit structural group") + .wait() + .expect("complete structural group"); + assert_eq!(completion.decoded_rects().len(), count); + + // SAFETY: group completion released the destination's exclusive write + // guard, so the retained shared allocation is now host-readable. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&output, 0, image_bytes * count) + .expect("completed structural destination bytes") + }; + let expected_group = &expected.groups()[0]; + assert_eq!(expected_group.source_indices(), group.source_indices()); + let CpuBatchSamples::U8(expected) = expected_group.samples() else { + panic!("8-bit structural fixtures must have a U8 CPU oracle") + }; + assert_eq!( + actual, + expected.as_slice(), + "{route:?} {color:?} batch {count} must retain exact source-order CPU parity" + ); + + Some(( + crate::compute::metal_command_buffers_for_test(), + crate::compute::metal_compute_encoders_for_test(), + )) +} + +#[test] +fn external_groups_use_one_producer_command_buffer_and_compute_encoder() { + let mut counts = Vec::new(); + for route in [FixtureRoute::Classic, FixtureRoute::Ht] { + for count in [1, 8] { + for color in [FixtureColor::Gray, FixtureColor::Rgb, FixtureColor::Rgba] { + if let Some(actual) = + assert_external_group_uses_one_command_buffer_and_encoder(route, color, count) + { + counts.push(actual); + } + } + } + } + if !counts.is_empty() { + assert_eq!( + counts, + vec![(1, 1); 12], + "classic and HT Gray/RGB/RGBA batch 1 and 8 must each use one producer command buffer and one compute encoder" + ); + } +} diff --git a/crates/j2k-metal/src/batch_decoder/external.rs b/crates/j2k-metal/src/batch_decoder/external.rs new file mode 100644 index 00000000..0f753210 --- /dev/null +++ b/crates/j2k-metal/src/batch_decoder/external.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Direct decode into caller-owned Metal group storage. + +use super::{ + validate_group_contract, BatchColor, Error, MetalBatchDecoder, MetalBatchGroupCompletion, + MetalImageDestination, PreparedBatchGroup, SubmittedMetalGroupDecodeInto, +}; + +pub(super) fn validate_consumer_registry_ids( + producer_registry_id: u64, + consumer_registry_id: u64, +) -> Result<(), Error> { + if producer_registry_id == consumer_registry_id { + return Ok(()); + } + Err(crate::error::metal_kernel_support_error( + "J2K Metal consumer queue belongs to a different device", + j2k_metal_support::MetalSupportError::MetalImageDeviceMismatch { + image_registry_id: producer_registry_id, + requested_registry_id: consumer_registry_id, + }, + )) +} + +impl MetalBatchDecoder { + /// Decode one prepared homogeneous Gray or unsigned RGB group directly + /// into one caller-owned Metal allocation. + /// + /// The group allocation is bound once at its validated base. Per-image + /// offsets are applied by the final-store kernel, so tightly packed Gray8 + /// images do not need independently aligned byte offsets. + #[cfg(target_os = "macos")] + pub fn decode_prepared_group_into( + &mut self, + group: &PreparedBatchGroup, + destination: &MetalImageDestination, + ) -> Result<(), Error> { + let fmt = validate_group_contract(group.info())?; + destination + .validate_device(self.backend_session().device()) + .and_then(|()| { + destination.validate_batch(group.info().dimensions, fmt, group.images().len()) + }) + .map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K Metal prepared group destination validation failed", + source, + ) + })?; + match group.info().color { + BatchColor::Gray => { + let plans = self.prepared_gray_group_plans(group, fmt, false)?; + let runtime = self.backend_session().runtime()?; + crate::compute::submit_prepared_direct_grayscale_plan_batch_into_group( + runtime, + &plans, + fmt, + destination, + Some(group.source_indices()), + crate::compute::DirectDestinationConsumerOrdering::HostCompletionOnly, + )? + .wait()?; + } + BatchColor::Rgb | BatchColor::Rgba => { + let plans = self.prepared_color_group_plans(group, fmt)?; + let runtime = self.backend_session().runtime()?; + crate::compute::submit_prepared_direct_color_plan_batch_into_group( + runtime, + &plans, + fmt, + group.info().layout, + destination, + Some(group.source_indices()), + crate::compute::DirectDestinationConsumerOrdering::HostCompletionOnly, + )? + .wait()?; + } + _ => { + return Err(Error::UnsupportedMetalRequest { + reason: + "J2K Metal exact external final-store received an unknown color contract", + }) + } + } + self.record_submission(); + Ok(()) + } + + /// Submit one prepared homogeneous Gray, RGB, or RGBA group directly + /// into one caller-owned Metal allocation without waiting on the CPU. + /// + /// The returned guard retains exclusive destination access, the committed + /// command buffer, status buffers, and scratch resources. Call + /// [`SubmittedMetalGroupDecodeInto::wait`] to surface execution failures. + /// Dropping the guard also retires the work safely. + #[cfg(target_os = "macos")] + pub fn submit_prepared_group_into( + &mut self, + group: &PreparedBatchGroup, + destination: MetalImageDestination, + ) -> Result { + self.submit_prepared_group_into_with_ordering( + group, + destination, + crate::compute::DirectDestinationConsumerOrdering::Deferred, + ) + } + + /// Submit one prepared group into caller-owned storage while registering + /// its dependency on a consumer queue known before producer commit. + /// + /// The exact producer queue needs no event bridge. A different queue on + /// the same device receives a GPU-side `MTLEvent` wait before this method + /// returns. Queues from another device are rejected before codec work is + /// committed. + #[cfg(target_os = "macos")] + pub fn submit_prepared_group_into_for_consumer_queue( + &mut self, + group: &PreparedBatchGroup, + destination: MetalImageDestination, + consumer_queue: &metal::CommandQueueRef, + ) -> Result { + let producer_registry_id = self.backend_session().device().registry_id(); + let consumer_registry_id = consumer_queue.device().registry_id(); + validate_consumer_registry_ids(producer_registry_id, consumer_registry_id)?; + self.submit_prepared_group_into_with_ordering( + group, + destination, + crate::compute::DirectDestinationConsumerOrdering::Known { + consumer_queue: consumer_queue.to_owned(), + timeline: self.backend_session().consumer_event_timeline(), + }, + ) + } + + fn submit_prepared_group_into_with_ordering( + &mut self, + group: &PreparedBatchGroup, + destination: MetalImageDestination, + consumer_ordering: crate::compute::DirectDestinationConsumerOrdering, + ) -> Result { + let fmt = validate_group_contract(group.info())?; + destination + .validate_device(self.backend_session().device()) + .and_then(|()| { + destination.validate_batch(group.info().dimensions, fmt, group.images().len()) + }) + .map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K Metal submitted prepared group destination validation failed", + source, + ) + })?; + let runtime = self.backend_session().runtime()?; + let submission = + match group.info().color { + BatchColor::Gray => { + let plans = self.prepared_gray_group_plans(group, fmt, true)?; + crate::compute::submit_prepared_direct_grayscale_plan_batch_into_group( + runtime, + &plans, + fmt, + &destination, + Some(group.source_indices()), + consumer_ordering, + )? + } + BatchColor::Rgb | BatchColor::Rgba => { + let plans = self.prepared_color_group_plans(group, fmt)?; + crate::compute::submit_prepared_direct_color_plan_batch_into_group( + runtime, + &plans, + fmt, + group.info().layout, + &destination, + Some(group.source_indices()), + consumer_ordering, + )? + } + _ => return Err(Error::UnsupportedMetalRequest { + reason: + "J2K Metal exact external final-store received an unknown color contract", + }), + }; + self.record_submission(); + Ok(SubmittedMetalGroupDecodeInto { + submission, + destination, + completion: MetalBatchGroupCompletion::from_prepared(group, group.options()), + }) + } +} diff --git a/crates/j2k-metal/src/batch_decoder/plan_cache.rs b/crates/j2k-metal/src/batch_decoder/plan_cache.rs new file mode 100644 index 00000000..079e599e --- /dev/null +++ b/crates/j2k-metal/src/batch_decoder/plan_cache.rs @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Persistent prepared-plan caches for exact Metal batch output. + +use super::{ + Arc, DecodeRequest, Error, MetalBackendSession, MetalBatchDecoder, PixelFormat, + PreparedBatchGroup, PreparedImage, +}; + +fn prepare_referenced_gray_plan( + session: &MetalBackendSession, + image: &PreparedImage, +) -> Result, Error> { + if let Some(prepared) = image.htj2k_plan() { + if !prepared.is_grayscale() { + return Ok(None); + } + let referenced = prepared + .adapter_view() + .downcast_ref::() + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal does not recognize the prepared HTJ2K grayscale plan adapter", + })?; + return crate::compute::with_runtime_for_session(session, |_| { + crate::compute::prepare_referenced_htj2k_grayscale_plan(referenced, image.bytes()) + .map(Some) + }); + } + let Some(prepared) = image.classic_plan() else { + return Ok(None); + }; + if !prepared.is_grayscale() { + return Ok(None); + } + let referenced = prepared + .adapter_view() + .downcast_ref::() + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal does not recognize the prepared classic grayscale plan adapter", + })?; + crate::compute::with_runtime_for_session(session, |_| { + crate::compute::prepare_referenced_classic_grayscale_plan(referenced, image.bytes()) + .map(Some) + }) +} + +#[cfg(target_os = "macos")] +fn prepare_referenced_color_plan( + session: &MetalBackendSession, + image: &PreparedImage, + fmt: PixelFormat, +) -> Result, Error> { + let rgba = matches!( + fmt, + PixelFormat::Rgba8 | PixelFormat::Rgba16 | PixelFormat::RgbaI16 + ); + let signed = matches!(fmt, PixelFormat::RgbI16 | PixelFormat::RgbaI16); + let plan = if let Some(prepared) = image.htj2k_plan() { + if (rgba && !prepared.is_rgba()) || (!rgba && !prepared.is_color()) { + return Ok(None); + } + let referenced = prepared + .adapter_view() + .downcast_ref::() + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal does not recognize the prepared HTJ2K color plan adapter", + })?; + crate::compute::with_runtime_for_session(session, |_| { + if rgba { + crate::compute::prepare_referenced_htj2k_rgba_plan( + referenced, + image.bytes(), + signed, + ) + } else { + crate::compute::prepare_referenced_htj2k_color_plan( + referenced, + image.bytes(), + signed, + ) + } + })? + } else if let Some(prepared) = image.classic_plan() { + if (rgba && !prepared.is_rgba()) || (!rgba && !prepared.is_color()) { + return Ok(None); + } + let referenced = prepared + .adapter_view() + .downcast_ref::() + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal does not recognize the prepared classic color plan adapter", + })?; + crate::compute::with_runtime_for_session(session, |_| { + if rgba { + crate::compute::prepare_referenced_classic_rgba_plan( + referenced, + image.bytes(), + signed, + ) + } else { + crate::compute::prepare_referenced_classic_color_plan( + referenced, + image.bytes(), + signed, + ) + } + })? + } else { + return Ok(None); + }; + Ok(Some(plan)) +} + +#[cfg(target_os = "macos")] +pub(super) type PreparedGrayPlanCache = + crate::session::PreparedPlanCache>; + +#[cfg(target_os = "macos")] +pub(super) type PreparedColorPlanCache = + crate::session::PreparedPlanCache>; + +#[cfg(target_os = "macos")] +pub(super) const PREPARED_BATCH_PLAN_CACHE_CAP: usize = 128; +impl MetalBatchDecoder { + #[cfg(target_os = "macos")] + pub(super) fn prepared_gray_group_plans( + &mut self, + group: &PreparedBatchGroup, + fmt: PixelFormat, + asynchronous: bool, + ) -> Result>, Error> { + let mut plans = Vec::new(); + plans + .try_reserve_exact(group.images().len()) + .map_err(|source| Error::PreparedPlanCacheAllocation { + context: "J2K Metal prepared grayscale group destination", + source, + })?; + for image in group.images() { + if let Some(plan) = self.prepared_gray_plan_for_image(image, fmt)? { + plans.push(plan); + continue; + } + if image.request() != DecodeRequest::Full { + return Err(Error::UnsupportedMetalRequest { + reason: if asynchronous { + "J2K Metal asynchronous external ROI/reduction requires a prepared HTJ2K offset plan" + } else { + "J2K Metal external ROI/reduction requires a prepared HTJ2K offset plan" + }, + }); + } + plans.push( + crate::decoder::prepare_full_grayscale_direct_plan_with_session( + image.bytes(), + fmt, + self.backend_session(), + )?, + ); + } + Ok(plans) + } + + #[cfg(target_os = "macos")] + pub(super) fn prepared_color_group_plans( + &mut self, + group: &PreparedBatchGroup, + fmt: PixelFormat, + ) -> Result>, Error> { + let mut plans = Vec::new(); + plans + .try_reserve_exact(group.images().len()) + .map_err(|source| Error::PreparedPlanCacheAllocation { + context: "J2K Metal prepared exact color group destination", + source, + })?; + for image in group.images() { + let Some(plan) = self.prepared_color_plan_for_image(image, fmt)? else { + return Err(Error::UnsupportedMetalRequest { + reason: if matches!( + fmt, + PixelFormat::Rgba8 | PixelFormat::Rgba16 | PixelFormat::RgbaI16 + ) { + "J2K Metal exact RGBA final-store requires a prepared classic or HTJ2K four-component offset plan" + } else { + "J2K Metal exact RGB final-store requires a prepared classic or HTJ2K three-component offset plan" + }, + }); + }; + plans.push(plan); + } + Ok(plans) + } + + #[cfg(target_os = "macos")] + fn prepared_gray_plan_for_image( + &mut self, + image: &PreparedImage, + fmt: PixelFormat, + ) -> Result>, Error> { + let key = crate::session::PreparedPlanCacheKey::prepared_gray( + image.bytes(), + image.request(), + fmt, + ); + if let Some(plan) = self.prepared_gray_plans.get(key) { + return Ok(Some(plan.clone())); + } + let Some(plan) = prepare_referenced_gray_plan(self.backend_session(), image)? else { + return Ok(None); + }; + let plan = Arc::new(plan); + self.prepared_gray_plans + .insert(key, plan.clone()) + .map_err(|error| { + crate::session::direct_plan_cache::prepared_plan_cache_error( + "J2K Metal prepared codec grayscale plan cache", + error, + ) + })?; + Ok(Some(plan)) + } + + #[cfg(target_os = "macos")] + fn prepared_color_plan_for_image( + &mut self, + image: &PreparedImage, + fmt: PixelFormat, + ) -> Result>, Error> { + let key = crate::session::PreparedPlanCacheKey::prepared_color( + image.bytes(), + image.request(), + fmt, + ); + if let Some(plan) = self.prepared_color_plans.get(key) { + return Ok(Some(plan.clone())); + } + let Some(plan) = prepare_referenced_color_plan(self.backend_session(), image, fmt)? else { + return Ok(None); + }; + let plan = Arc::new(plan); + self.prepared_color_plans + .insert(key, plan.clone()) + .map_err(|error| { + crate::session::direct_plan_cache::prepared_plan_cache_error( + "J2K Metal prepared codec color plan cache", + error, + ) + })?; + Ok(Some(plan)) + } +} diff --git a/crates/j2k-metal/src/batch_decoder/queue_ordering_tests.rs b/crates/j2k-metal/src/batch_decoder/queue_ordering_tests.rs new file mode 100644 index 00000000..25938abc --- /dev/null +++ b/crates/j2k-metal/src/batch_decoder/queue_ordering_tests.rs @@ -0,0 +1,545 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use super::*; + +fn prepared_gray_group( + decoder: &MetalBatchDecoder, +) -> Result> { + let bytes = Arc::<[u8]>::from(j2k_test_support::htj2k_gray8_fixture(4, 4)); + Ok(decoder.prepare(vec![EncodedImage::full(bytes)])?) +} + +fn gray8_cpu_oracle(encoded: &[u8]) -> [u8; 16] { + let mut expected = [0_u8; 16]; + j2k::J2kDecoder::new(encoded) + .expect("CPU oracle decoder") + .decode_into(&mut expected, 4, PixelFormat::Gray8) + .expect("CPU oracle decode"); + expected +} + +fn distinct_gray8_fixture(seed: u8) -> Arc<[u8]> { + let pixels = (0_u8..16) + .map(|value| value.wrapping_mul(13).wrapping_add(seed)) + .collect::>(); + Arc::from( + j2k_native::encode_htj2k( + &pixels, + 4, + 4, + 1, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode distinct Gray8 fixture"), + ) +} + +fn gray8_destination( + device: &metal::DeviceRef, +) -> Result> { + let buffer = j2k_metal_support::checked_shared_buffer_for_len::(device, 16)?; + let layout = MetalImageLayout::new_batch(0, (4, 4), 4, PixelFormat::Gray8, 1, 16)?; + // SAFETY: The fresh allocation has one exclusive owner and the returned + // submission retains that owner until completion or drop. + Ok(unsafe { MetalImageDestination::from_exclusive_buffer(buffer, layout)? }) +} + +fn wrong_size_gray8_destination( + device: &metal::DeviceRef, +) -> Result> { + let buffer = j2k_metal_support::checked_shared_buffer_for_len::(device, 4)?; + let layout = MetalImageLayout::new_batch(0, (2, 2), 2, PixelFormat::Gray8, 1, 4)?; + // SAFETY: The fresh allocation has one exclusive owner. Destination + // preflight rejects its dimensions before any codec work can retain it. + Ok(unsafe { MetalImageDestination::from_exclusive_buffer(buffer, layout)? }) +} + +#[test] +fn known_exact_consumer_queue_submits_without_an_event_bridge() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let device = metal::Device::system_default().expect("system Metal device"); + let queue = + j2k_metal_support::checked_command_queue(&device).expect("exact consumer command queue"); + let backend = MetalBackendSession::with_command_queue(device.clone(), queue.clone()) + .expect("backend using exact consumer queue"); + let mut decoder = MetalBatchDecoder::with_backend_session(backend); + let encoded = Arc::<[u8]>::from(j2k_test_support::htj2k_gray8_fixture(4, 4)); + let prepared = decoder + .prepare(vec![EncodedImage::full(encoded.clone())]) + .expect("prepare Gray8 group"); + let destination_buffer = j2k_metal_support::checked_shared_buffer_for_len::(&device, 16) + .expect("Gray8 destination buffer"); + let layout = MetalImageLayout::new_batch(0, (4, 4), 4, PixelFormat::Gray8, 1, 16) + .expect("Gray8 destination layout"); + // SAFETY: the fresh allocation is retained exclusively by the pending + // submission until completion. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(destination_buffer.clone(), layout) + .expect("Gray8 destination") + }; + + let pending = decoder + .submit_prepared_group_into_for_consumer_queue(&prepared.groups()[0], destination, &queue) + .expect("submit on exact producer/consumer queue"); + + assert_eq!( + pending.submission.ordering_diagnostics_for_test(), + (false, false, 0), + "exact-queue submission must not allocate or signal an event or enqueue a wait command" + ); + pending.wait().expect("exact-queue completion"); + // SAFETY: completion released the destination's exclusive write guard. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&destination_buffer, 0, 16) + .expect("completed exact-queue pixels") + }; + assert_eq!(actual, gray8_cpu_oracle(&encoded)); +} + +#[test] +fn synchronous_external_decode_has_no_consumer_event_bridge() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = prepared_gray_group(&decoder).expect("prepare Gray8 group"); + let destination = gray8_destination(decoder.backend_session().device()) + .expect("synchronous Gray8 destination"); + crate::compute::reset_direct_destination_event_bridge_for_test(); + + decoder + .decode_prepared_group_into(&prepared.groups()[0], &destination) + .expect("synchronous external decode"); + + assert_eq!( + crate::compute::direct_destination_event_bridge_for_test(), + (0, 0, 0), + "host-completed external decode must not allocate, signal, or wait on an event" + ); +} + +#[test] +fn codec_owned_resident_decode_has_no_consumer_event_bridge() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = prepared_gray_group(&decoder).expect("prepare Gray8 group"); + crate::compute::reset_direct_destination_event_bridge_for_test(); + + let result = decoder + .decode_prepared(&prepared) + .expect("codec-owned resident decode"); + assert_eq!(result.groups().len(), 1); + assert_eq!( + crate::compute::direct_destination_event_bridge_for_test(), + (0, 0, 0), + "codec-owned output hidden behind host completion needs no consumer event bridge" + ); +} + +#[test] +fn deferred_async_submission_preserves_the_compatibility_event_bridge() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = prepared_gray_group(&decoder).expect("prepare Gray8 group"); + let destination = + gray8_destination(decoder.backend_session().device()).expect("async Gray8 destination"); + let consumer_queue = + j2k_metal_support::checked_command_queue(decoder.backend_session().device()) + .expect("compatibility consumer command queue"); + crate::compute::reset_direct_destination_event_bridge_for_test(); + + let mut pending = decoder + .submit_prepared_group_into(&prepared.groups()[0], destination) + .expect("deferred async submission"); + pending + .enqueue_consumer_wait(&consumer_queue) + .expect("compatibility consumer wait"); + + assert_eq!( + crate::compute::direct_destination_event_bridge_for_test(), + (1, 1, 1), + "deferred compatibility submission must retain its event bridge" + ); + pending.wait().expect("deferred async completion"); +} + +#[test] +fn known_cross_queue_submissions_reuse_one_monotonic_event_timeline() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let device = metal::Device::system_default().expect("system Metal device"); + let producer_queue = + j2k_metal_support::checked_command_queue(&device).expect("producer command queue"); + let consumer_queue = + j2k_metal_support::checked_command_queue(&device).expect("consumer command queue"); + let backend = MetalBackendSession::with_command_queue(device.clone(), producer_queue) + .expect("backend using producer queue"); + let mut decoder = MetalBatchDecoder::with_backend_session(backend); + let prepared = prepared_gray_group(&decoder).expect("prepare Gray8 group"); + + let first = decoder + .submit_prepared_group_into_for_consumer_queue( + &prepared.groups()[0], + gray8_destination(&device).expect("first Gray8 destination"), + &consumer_queue, + ) + .expect("first cross-queue submission"); + let second = decoder + .submit_prepared_group_into_for_consumer_queue( + &prepared.groups()[0], + gray8_destination(&device).expect("second Gray8 destination"), + &consumer_queue, + ) + .expect("second cross-queue submission"); + + let (first_event, first_value) = first + .submission + .known_consumer_timeline_for_test() + .expect("first cross-queue event dependency"); + let (second_event, second_value) = second + .submission + .known_consumer_timeline_for_test() + .expect("second cross-queue event dependency"); + assert_eq!( + first_event, second_event, + "session must reuse one MTL event" + ); + assert_eq!((first_value, second_value), (1, 2)); + assert_eq!(first.submission.ordering_diagnostics_for_test().2, 1); + assert_eq!(second.submission.ordering_diagnostics_for_test().2, 1); + + first.wait().expect("first cross-queue completion"); + second.wait().expect("second cross-queue completion"); +} + +#[test] +fn overlapping_prepared_submissions_keep_distinct_status_and_scratch_owners() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let encoded = Arc::<[u8]>::from(j2k_test_support::openhtj2k_refinement_fixture()); + let device = metal::Device::system_default().expect("system Metal device"); + let queue = j2k_metal_support::checked_command_queue(&device).expect("overlap command queue"); + let backend = MetalBackendSession::with_command_queue(device.clone(), queue.clone()) + .expect("isolated exact-queue backend"); + let mut decoder = MetalBatchDecoder::with_backend_session(backend); + let prepared = decoder + .prepare(vec![EncodedImage::full(encoded.clone())]) + .expect("prepare overlapping HT group"); + let info = prepared.groups()[0].info(); + let pixel_format = info.native_pixel_format().expect("native HT pixel format"); + let row_bytes = info.dimensions.0 as usize * pixel_format.bytes_per_pixel(); + let image_bytes = row_bytes * info.dimensions.1 as usize; + + let allocate_destination = || { + let buffer = j2k_metal_support::checked_shared_buffer_for_len::(&device, image_bytes) + .expect("overlapping destination buffer"); + let layout = MetalImageLayout::new_batch( + 0, + info.dimensions, + row_bytes, + pixel_format, + 1, + image_bytes, + ) + .expect("overlapping destination layout"); + // SAFETY: each fresh allocation has one exclusive owner retained by + // its pending submission until completion. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("overlapping destination") + }; + (destination, buffer) + }; + let (first_destination, first_buffer) = allocate_destination(); + let (second_destination, second_buffer) = allocate_destination(); + + let first = decoder + .submit_prepared_group_into_for_consumer_queue( + &prepared.groups()[0], + first_destination, + &queue, + ) + .expect("first overlapping submission"); + let second = decoder + .submit_prepared_group_into_for_consumer_queue( + &prepared.groups()[0], + second_destination, + &queue, + ) + .expect("second overlapping submission"); + + let (first_status, first_scratch) = first.submission.in_flight_owner_ptrs_for_test(); + let (second_status, second_scratch) = second.submission.in_flight_owner_ptrs_for_test(); + assert!(!first_status.is_empty() && !first_scratch.is_empty()); + assert!(!second_status.is_empty() && !second_scratch.is_empty()); + assert!(first_status + .iter() + .all(|owner| !second_status.contains(owner))); + assert!(first_scratch + .iter() + .all(|owner| !second_scratch.contains(owner))); + + first.wait().expect("first overlapping completion"); + second.wait().expect("second overlapping completion"); + let mut expected = vec![0_u8; image_bytes]; + j2k::J2kDecoder::new(&encoded) + .expect("overlapping CPU oracle decoder") + .decode_into(&mut expected, row_bytes, pixel_format) + .expect("overlapping CPU oracle decode"); + // SAFETY: both exclusive codec writes have completed and released their + // destination guards. + let first_pixels = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&first_buffer, 0, image_bytes) + .expect("first overlapping pixels") + }; + // SAFETY: as above for the independent second destination. + let second_pixels = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&second_buffer, 0, image_bytes) + .expect("second overlapping pixels") + }; + assert_eq!(first_pixels, expected); + assert_eq!(second_pixels, expected); +} + +#[test] +fn known_cross_queue_wait_orders_subsequent_consumer_work() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let encoded = Arc::<[u8]>::from(j2k_test_support::htj2k_gray8_fixture(4, 4)); + let device = metal::Device::system_default().expect("system Metal device"); + let producer_queue = + j2k_metal_support::checked_command_queue(&device).expect("producer command queue"); + let consumer_queue = + j2k_metal_support::checked_command_queue(&device).expect("consumer command queue"); + let backend = MetalBackendSession::with_command_queue(device.clone(), producer_queue) + .expect("backend using producer queue"); + let mut decoder = MetalBatchDecoder::with_backend_session(backend); + let prepared = decoder + .prepare(vec![EncodedImage::full(encoded.clone())]) + .expect("prepare Gray8 group"); + let destination_buffer = j2k_metal_support::checked_shared_buffer_for_len::(&device, 16) + .expect("destination buffer"); + let copy_buffer = j2k_metal_support::checked_shared_buffer_for_len::(&device, 16) + .expect("consumer copy buffer"); + let layout = MetalImageLayout::new_batch(0, (4, 4), 4, PixelFormat::Gray8, 1, 16) + .expect("destination layout"); + // SAFETY: The fresh destination is retained exclusively by the pending + // decode until producer completion. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(destination_buffer.clone(), layout) + .expect("exclusive destination") + }; + let pending = decoder + .submit_prepared_group_into_for_consumer_queue( + &prepared.groups()[0], + destination, + &consumer_queue, + ) + .expect("cross-queue submission"); + + let consumer_command = + j2k_metal_support::checked_command_buffer(&consumer_queue).expect("consumer command"); + let blit = + j2k_metal_support::checked_blit_command_encoder(&consumer_command).expect("consumer blit"); + blit.copy_from_buffer(&destination_buffer, 0, ©_buffer, 0, 16); + blit.end_encoding(); + consumer_command.commit(); + + pending.wait().expect("producer completion"); + j2k_metal_support::wait_for_completion(&consumer_command).expect("consumer completion"); + // SAFETY: Both the producer and event-ordered consumer copy have completed. + let copied = unsafe { + j2k_metal_support::checked_buffer_read_vec::(©_buffer, 0, 16) + .expect("copied pixels") + }; + assert_eq!(copied, gray8_cpu_oracle(&encoded)); +} + +#[test] +fn consumer_queue_registry_mismatch_is_rejected() { + let error = super::external::validate_consumer_registry_ids(17, 23) + .expect_err("different Metal devices must be rejected"); + assert!(matches!( + error, + Error::MetalSupport { + source: j2k_metal_support::MetalSupportError::MetalImageDeviceMismatch { + image_registry_id: 17, + requested_registry_id: 23, + }, + .. + } + )); +} + +#[test] +fn dropping_exact_queue_work_leaves_the_session_reusable() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let device = metal::Device::system_default().expect("system Metal device"); + let queue = + j2k_metal_support::checked_command_queue(&device).expect("exact consumer command queue"); + let backend = MetalBackendSession::with_command_queue(device.clone(), queue.clone()) + .expect("backend using exact consumer queue"); + let mut decoder = MetalBatchDecoder::with_backend_session(backend); + let prepared = prepared_gray_group(&decoder).expect("prepare Gray8 group"); + + let rejected = decoder.submit_prepared_group_into_for_consumer_queue( + &prepared.groups()[0], + wrong_size_gray8_destination(&device).expect("wrong-size Gray8 destination"), + &queue, + ); + assert!( + rejected.is_err(), + "wrong-size destination must fail preflight" + ); + assert_eq!( + decoder + .submissions() + .expect("submission count after rejection"), + 0, + "rejected preflight must not increment the submission counter" + ); + + let pending = decoder + .submit_prepared_group_into_for_consumer_queue( + &prepared.groups()[0], + gray8_destination(&device).expect("first Gray8 destination"), + &queue, + ) + .expect("first exact-queue submission"); + drop(pending); + + decoder + .submit_prepared_group_into_for_consumer_queue( + &prepared.groups()[0], + gray8_destination(&device).expect("second Gray8 destination"), + &queue, + ) + .expect("second exact-queue submission") + .wait() + .expect("reused session completion"); + assert_eq!(decoder.submissions().expect("submission count"), 2); +} + +#[test] +fn cloned_sessions_assign_cross_queue_values_in_commit_order() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let device = metal::Device::system_default().expect("system Metal device"); + let producer_queue = + j2k_metal_support::checked_command_queue(&device).expect("producer command queue"); + let consumer_queue = + j2k_metal_support::checked_command_queue(&device).expect("consumer command queue"); + let backend = MetalBackendSession::with_command_queue(device, producer_queue) + .expect("backend using producer queue"); + let start = Arc::new(std::sync::Barrier::new(2)); + + let observations = std::thread::scope(|scope| { + let mut workers = Vec::new(); + for seed in [0_u8, 31] { + let backend = backend.clone(); + let consumer_queue = consumer_queue.clone(); + let start = start.clone(); + workers.push(scope.spawn(move || { + let mut decoder = MetalBatchDecoder::with_backend_session(backend); + let encoded = distinct_gray8_fixture(seed); + let prepared = decoder + .prepare(vec![EncodedImage::full(encoded.clone())]) + .expect("prepare worker group"); + let destination_buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + 16, + ) + .expect("worker destination buffer"); + let copy_buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + 16, + ) + .expect("worker consumer copy buffer"); + let layout = MetalImageLayout::new_batch(0, (4, 4), 4, PixelFormat::Gray8, 1, 16) + .expect("worker destination layout"); + // SAFETY: the fresh allocation is retained exclusively by + // the pending decode until producer completion. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(destination_buffer.clone(), layout) + .expect("worker destination") + }; + start.wait(); + let pending = decoder + .submit_prepared_group_into_for_consumer_queue( + &prepared.groups()[0], + destination, + &consumer_queue, + ) + .expect("worker cross-queue submission"); + let timeline = pending + .submission + .known_consumer_timeline_for_test() + .expect("worker timeline"); + let consumer_command = j2k_metal_support::checked_command_buffer(&consumer_queue) + .expect("worker consumer command"); + let blit = j2k_metal_support::checked_blit_command_encoder(&consumer_command) + .expect("worker consumer blit"); + blit.copy_from_buffer(&destination_buffer, 0, ©_buffer, 0, 16); + blit.end_encoding(); + consumer_command.commit(); + pending.wait().expect("worker completion"); + j2k_metal_support::wait_for_completion(&consumer_command) + .expect("worker consumer completion"); + // SAFETY: both the producer and event-ordered consumer copy + // completed before this read. + let copied = unsafe { + j2k_metal_support::checked_buffer_read_vec::(©_buffer, 0, 16) + .expect("worker copied pixels") + }; + (timeline, copied, gray8_cpu_oracle(&encoded)) + })); + } + workers + .into_iter() + .map(|worker| worker.join().expect("worker thread")) + .collect::>() + }); + + let first_timeline = observations[0].0; + let second_timeline = observations[1].0; + assert_eq!(first_timeline.0, second_timeline.0); + let mut values = [first_timeline.1, second_timeline.1]; + values.sort_unstable(); + assert_eq!(values, [1, 2]); + for (_, copied, expected) in &observations { + assert_eq!(copied, expected); + } + assert_ne!( + observations[0].1, observations[1].1, + "concurrent consumer observations must come from distinct producer outputs" + ); +} diff --git a/crates/j2k-metal/src/batch_decoder/resident.rs b/crates/j2k-metal/src/batch_decoder/resident.rs new file mode 100644 index 00000000..4e265cd7 --- /dev/null +++ b/crates/j2k-metal/src/batch_decoder/resident.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Codec-owned resident Metal group submission. + +use super::{ + allocate_codec_owned_group_destination, validate_codec_owned_resident_group, BatchColor, + BatchDecodeOptions, Error, MetalBatchDecoder, MetalBatchGroup, MetalResidentGroupMetadata, + PreparedBatchGroup, SubmittedMetalResidentGroup, +}; + +impl MetalBatchDecoder { + pub(super) fn decode_prepared_group_with_options( + &mut self, + group: &PreparedBatchGroup, + options: BatchDecodeOptions, + ) -> Result { + #[cfg(target_os = "macos")] + { + let pending = self.submit_prepared_resident_group(group, options)?; + pending.wait().map_err(|(_, source)| *source) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = group; + let _ = options; + Err(Error::MetalUnavailable) + } + } + + #[cfg(target_os = "macos")] + pub(super) fn submit_prepared_resident_group( + &mut self, + group: &PreparedBatchGroup, + options: BatchDecodeOptions, + ) -> Result { + let fmt = validate_codec_owned_resident_group(group)?; + let allocation = + allocate_codec_owned_group_destination(self.backend_session().device(), group, fmt)?; + let runtime = self.backend_session().runtime()?; + let submission = + match group.info().color { + BatchColor::Gray => { + let plans = self.prepared_gray_group_plans(group, fmt, true)?; + crate::compute::submit_prepared_direct_grayscale_plan_batch_into_group( + runtime, + &plans, + fmt, + &allocation.destination, + Some(group.source_indices()), + crate::compute::DirectDestinationConsumerOrdering::HostCompletionOnly, + )? + } + BatchColor::Rgb | BatchColor::Rgba => { + let plans = self.prepared_color_group_plans(group, fmt)?; + crate::compute::submit_prepared_direct_color_plan_batch_into_group( + runtime, + &plans, + fmt, + group.info().layout, + &allocation.destination, + Some(group.source_indices()), + crate::compute::DirectDestinationConsumerOrdering::HostCompletionOnly, + )? + } + _ => return Err(Error::UnsupportedMetalRequest { + reason: + "J2K Metal codec-owned resident output received an unknown color contract", + }), + }; + self.record_submission(); + Ok(SubmittedMetalResidentGroup { + metadata: MetalResidentGroupMetadata::from_prepared(group, options), + submission, + destination: allocation.destination, + output: allocation.output, + layout: allocation.layout, + }) + } +} diff --git a/crates/j2k-metal/src/batch_decoder/submission.rs b/crates/j2k-metal/src/batch_decoder/submission.rs new file mode 100644 index 00000000..638567bc --- /dev/null +++ b/crates/j2k-metal/src/batch_decoder/submission.rs @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Metal submission ownership, allocation, and completion. + +use super::{ + validate_group_contract, BatchColor, BatchDecodeOptions, BatchGroupInfo, BatchLayout, Buffer, + DeviceRef, DeviceSubmission, Error, IndexedBatchError, J2kDecodeWarning, + MetalBatchDecodeResult, MetalBatchGroup, MetalBatchGroupCompletion, MetalBatchGroupError, + MetalImageDestination, MetalImageLayout, MetalResidentBatch, PixelFormat, PreparedBatchGroup, + Rect, ResidentMetalImage, Surface, +}; + +/// Pending direct decode of one prepared group into caller-owned Metal storage. +/// +/// This guard retains the exclusive destination and all codec scratch owners +/// until completion. [`Self::wait`] reports command or codec-status failures. +/// Dropping it safely retires the committed work before releasing the +/// destination, so the decoder session remains reusable. +#[cfg(target_os = "macos")] +pub struct SubmittedMetalGroupDecodeInto { + pub(super) submission: crate::compute::SubmittedDirectDestination, + pub(super) destination: MetalImageDestination, + pub(super) completion: MetalBatchGroupCompletion, +} + +#[cfg(target_os = "macos")] +impl core::fmt::Debug for SubmittedMetalGroupDecodeInto { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("SubmittedMetalGroupDecodeInto") + .field("pending", &true) + .finish_non_exhaustive() + } +} + +#[cfg(target_os = "macos")] +impl SubmittedMetalGroupDecodeInto { + /// Enqueue a GPU-side wait on a same-device consumer command queue. + /// + /// The wait command is committed before this method returns, so commands + /// subsequently submitted to `consumer_queue` cannot observe the decoded + /// destination before the codec producer signals completion. This method + /// performs no CPU wait. + pub fn enqueue_consumer_wait( + &mut self, + consumer_queue: &metal::CommandQueueRef, + ) -> Result<(), Error> { + self.submission.enqueue_consumer_wait(consumer_queue) + } + + /// Wait for GPU completion, validate group status, and release exclusive + /// access to the destination. + pub fn wait(self) -> Result { + let Self { + submission, + destination, + completion, + } = self; + let result = submission.wait(); + drop(destination); + result?; + Ok(completion) + } +} + +#[cfg(target_os = "macos")] +impl DeviceSubmission for SubmittedMetalGroupDecodeInto { + type Output = MetalBatchGroupCompletion; + type Error = Error; + + fn wait(self) -> Result { + Self::wait(self) + } +} + +/// Pending shared-contract decode into codec-owned Metal-resident groups. +/// +/// Every group is committed before this guard is returned. [`Self::wait`] +/// retires all groups, preserves indexed preparation failures and non-fatal +/// group failures, then exposes completed resident surfaces. Dropping the +/// guard also retires every committed command buffer before releasing its +/// output allocation. +#[cfg(target_os = "macos")] +pub struct SubmittedMetalPreparedBatch { + pub(super) pending_groups: Vec, + pub(super) errors: Vec, + pub(super) group_errors: Vec, +} + +#[cfg(target_os = "macos")] +impl core::fmt::Debug for SubmittedMetalPreparedBatch { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("SubmittedMetalPreparedBatch") + .field("pending_groups", &self.pending_groups.len()) + .field("errors", &self.errors.len()) + .field("group_errors", &self.group_errors.len()) + .finish_non_exhaustive() + } +} + +#[cfg(target_os = "macos")] +impl SubmittedMetalPreparedBatch { + /// Number of successfully committed resident groups awaiting completion. + #[must_use] + pub fn len(&self) -> usize { + self.pending_groups.len() + } + + /// Whether no resident group was committed. + #[must_use] + pub fn is_empty(&self) -> bool { + self.pending_groups.is_empty() + } + + /// Retire every group and collect the shared codec batch result. + pub fn wait(mut self) -> Result { + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K submitted prepared Metal batch completion", + ); + let mut groups = budget.try_vec( + self.pending_groups.len(), + "J2K submitted prepared Metal output groups", + )?; + let mut first_fatal = None; + for pending in self.pending_groups.drain(..) { + match pending.wait() { + Ok(group) => groups.push(group), + Err((_, source)) if source.session_is_unusable() => { + if first_fatal.is_none() { + first_fatal = Some(source); + } + } + Err((source_indices, source)) => self.group_errors.push(MetalBatchGroupError { + source_indices, + source, + }), + } + } + if let Some(source) = first_fatal { + return Err(*source); + } + Ok(MetalBatchDecodeResult { + groups, + errors: core::mem::take(&mut self.errors), + group_errors: core::mem::take(&mut self.group_errors), + }) + } +} + +#[cfg(target_os = "macos")] +impl DeviceSubmission for SubmittedMetalPreparedBatch { + type Output = MetalBatchDecodeResult; + type Error = Error; + + fn wait(self) -> Result { + Self::wait(self) + } +} + +#[cfg(target_os = "macos")] +pub(super) struct SubmittedMetalResidentGroup { + pub(super) metadata: MetalResidentGroupMetadata, + pub(super) submission: crate::compute::SubmittedDirectDestination, + pub(super) destination: MetalImageDestination, + pub(super) output: Buffer, + pub(super) layout: MetalImageLayout, +} + +#[cfg(target_os = "macos")] +pub(super) struct MetalResidentGroupMetadata { + info: BatchGroupInfo, + source_indices: Vec, + decoded_rects: Vec, + warnings: Vec>, +} + +#[cfg(target_os = "macos")] +impl MetalResidentGroupMetadata { + pub(super) fn from_prepared(group: &PreparedBatchGroup, options: BatchDecodeOptions) -> Self { + let MetalBatchGroupCompletion { + decoded_rects, + warnings, + } = MetalBatchGroupCompletion::from_prepared(group, options); + Self { + info: group.info().clone(), + source_indices: group.source_indices().to_vec(), + decoded_rects, + warnings, + } + } +} + +#[cfg(target_os = "macos")] +pub(super) struct CodecOwnedMetalGroupDestination { + pub(super) destination: MetalImageDestination, + pub(super) output: Buffer, + pub(super) layout: MetalImageLayout, +} + +#[cfg(target_os = "macos")] +pub(super) fn validate_codec_owned_resident_group( + group: &PreparedBatchGroup, +) -> Result { + let format = validate_group_contract(group.info())?; + if group.images().is_empty() { + return Err(Error::MetalStateInvariant { + state: "J2K submitted codec-owned Metal group", + reason: "prepared homogeneous group contains no images", + }); + } + Ok(format) +} + +#[cfg(target_os = "macos")] +pub(super) fn allocate_codec_owned_group_destination( + device: &DeviceRef, + group: &PreparedBatchGroup, + fmt: PixelFormat, +) -> Result { + let dimensions = group.info().dimensions; + let row_bytes = usize::try_from(dimensions.0) + .ok() + .and_then(|width| width.checked_mul(fmt.bytes_per_pixel())) + .ok_or_else(|| Error::MetalKernel { + message: "J2K submitted codec-owned Metal row size overflow".to_string(), + })?; + let image_bytes = row_bytes + .checked_mul(dimensions.1 as usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K submitted codec-owned Metal image size overflow".to_string(), + })?; + let total_bytes = image_bytes + .checked_mul(group.images().len()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K submitted codec-owned Metal group size overflow".to_string(), + })?; + let output = + j2k_metal_support::checked_shared_buffer(device, total_bytes).map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K submitted codec-owned Metal group allocation failed", + source, + ) + })?; + let layout = MetalImageLayout::new_batch( + 0, + dimensions, + row_bytes, + fmt, + group.images().len(), + image_bytes, + ) + .map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K submitted codec-owned Metal group layout failed", + source, + ) + })?; + // SAFETY: `output` is a fresh codec-owned allocation and remains + // exclusively retained by the pending group until GPU completion. + let destination = + unsafe { MetalImageDestination::from_exclusive_buffer(output.clone(), layout) }.map_err( + |source| { + crate::error::metal_kernel_support_error( + "J2K submitted codec-owned Metal destination failed", + source, + ) + }, + )?; + Ok(CodecOwnedMetalGroupDestination { + destination, + output, + layout, + }) +} + +#[cfg(target_os = "macos")] +fn completed_codec_owned_resident_batch( + output: Buffer, + layout: MetalImageLayout, + expose_surface_views: bool, +) -> Result<(MetalResidentBatch, Vec), Error> { + // SAFETY: the producer submission completed successfully, its exclusive + // destination was dropped, and `output` is the last raw writable owner. + let storage = + unsafe { ResidentMetalImage::from_completed_buffer(output, layout) }.map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K submitted codec-owned resident batch", + source, + ) + })?; + let mut surfaces = Vec::new(); + if expose_surface_views { + surfaces + .try_reserve_exact(layout.image_count()) + .map_err(|source| Error::PreparedPlanCacheAllocation { + context: "J2K submitted codec-owned Metal surface views", + source, + })?; + for index in 0..layout.image_count() { + let relative_offset = + layout + .image_offset_bytes(index) + .ok_or_else(|| Error::MetalKernel { + message: "J2K submitted codec-owned Metal surface offset overflow" + .to_string(), + })?; + let offset = layout + .byte_offset() + .checked_add(relative_offset) + .ok_or_else(|| Error::MetalKernel { + message: "J2K submitted codec-owned Metal surface offset overflow".to_string(), + })?; + let view_layout = MetalImageLayout::new( + offset, + layout.dimensions(), + layout.pitch_bytes(), + layout.pixel_format(), + ) + .map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K submitted codec-owned Metal surface layout", + source, + ) + })?; + let view = storage.view(view_layout).map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K submitted codec-owned Metal surface view", + source, + ) + })?; + surfaces.push(Surface::from_resident_metal_image(view)); + } + } + Ok((MetalResidentBatch { storage }, surfaces)) +} + +#[cfg(target_os = "macos")] +impl SubmittedMetalResidentGroup { + pub(super) fn wait(self) -> Result, Box)> { + let Self { + mut metadata, + submission, + destination, + output, + layout, + } = self; + if let Err(source) = submission.wait() { + return Err((metadata.source_indices, Box::new(source))); + } + drop(destination); + let expose_surface_views = + metadata.info.color == BatchColor::Gray || metadata.info.layout == BatchLayout::Nhwc; + let (resident_batch, surfaces) = + completed_codec_owned_resident_batch(output, layout, expose_surface_views).map_err( + |source| { + ( + core::mem::take(&mut metadata.source_indices), + Box::new(source), + ) + }, + )?; + Ok(MetalBatchGroup { + info: metadata.info, + source_indices: metadata.source_indices, + decoded_rects: metadata.decoded_rects, + warnings: metadata.warnings, + surfaces, + resident_batch, + }) + } +} diff --git a/crates/j2k-metal/src/compute.rs b/crates/j2k-metal/src/compute.rs index c218ce7d..a477fae1 100644 --- a/crates/j2k-metal/src/compute.rs +++ b/crates/j2k-metal/src/compute.rs @@ -1,21 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -#[cfg(target_os = "macos")] -use std::{ - mem::{size_of, size_of_val}, - sync::Arc, - time::{Duration, Instant}, -}; - -#[cfg(target_os = "macos")] -use j2k_core::checked_surface_len; use j2k_core::{PixelFormat, Rect}; #[cfg(target_os = "macos")] -use j2k_metal_support::{ - checked_blit_command_encoder, checked_command_buffer, checked_compute_command_encoder, - dispatch_1d_pipeline, dispatch_2d_pipeline, dispatch_3d_pipeline, dispatch_single_thread, -}; -#[cfg(target_os = "macos")] use j2k_native::{ idwt_required_input_windows, idwt_required_output_margin, pack_j2k_code_block_scalar_from_tier1_tokens, ColorSpace as NativeColorSpace, @@ -31,72 +17,13 @@ use j2k_native::{ }; #[cfg(target_os = "macos")] use metal::{ - foreign_types::ForeignType, BlitCommandEncoder, Buffer, CommandBuffer, CommandBufferRef, - CommandQueueRef, ComputeCommandEncoder, ComputeCommandEncoderRef, ComputePipelineState, Device, - MTLSize, + foreign_types::ForeignType, Buffer, CommandBuffer, CommandBufferRef, ComputeCommandEncoderRef, + ComputePipelineState, Device, MTLSize, }; #[cfg(target_os = "macos")] use rayon::prelude::*; -#[cfg(target_os = "macos")] -use crate::profile_env::{ - classic_tier1_gpu_token_pack_requested, classic_tier1_split_gpu_token_pack_requested, - classic_tier1_split_mq_byte_gpu_token_pack_disabled, - classic_tier1_split_mq_byte_gpu_token_pack_requested, hybrid_stage_signpost, - label_command_buffer, label_compute_encoder, - metal_profile_classic_tier1_arithmetic_pack_enabled, - metal_profile_classic_tier1_density_enabled, metal_profile_classic_tier1_pass_plan_enabled, - metal_profile_classic_tier1_raw_pack_enabled, - metal_profile_classic_tier1_split_token_emit_enabled, - metal_profile_classic_tier1_symbol_plan_enabled, - metal_profile_classic_tier1_token_emit_enabled, metal_profile_classic_tier1_token_pack_enabled, - metal_profile_coefficient_prep_split_commands_enabled, - metal_profile_decode_split_commands_enabled, HybridSignpostName, - SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD, SIGNPOST_DECODE_HYBRID_COMMAND_WAIT, - SIGNPOST_DECODE_HYBRID_CPU_TIER1, SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE, - SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE, SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE, - SIGNPOST_ENCODE_HYBRID_CLASSIC_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, - SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKETIZATION_COMMAND_ENCODE, - SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_BUFFER_SETUP, SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_PLAN, - SIGNPOST_ENCODE_HYBRID_CLASSIC_PAYLOAD_COPY_COMMAND_ENCODE, - SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_COMMAND_ENCODE, - SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_SETUP, SIGNPOST_ENCODE_HYBRID_COMMAND_WAIT, - SIGNPOST_ENCODE_HYBRID_HT_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, - SIGNPOST_ENCODE_HYBRID_HT_PACKETIZATION_COMMAND_ENCODE, - SIGNPOST_ENCODE_HYBRID_HT_PACKET_BLOCK_PREP_COMMAND_ENCODE, - SIGNPOST_ENCODE_HYBRID_HT_PACKET_BUFFER_SETUP, SIGNPOST_ENCODE_HYBRID_HT_PACKET_PLAN, - SIGNPOST_ENCODE_HYBRID_HT_PAYLOAD_COPY_COMMAND_ENCODE, - SIGNPOST_ENCODE_HYBRID_HT_TIER1_COMMAND_ENCODE, SIGNPOST_ENCODE_HYBRID_HT_TIER1_SETUP, - SIGNPOST_ENCODE_HYBRID_RESULT_HARVEST, -}; -use crate::{error::metal_kernel_support_error, Error, Surface}; - -#[cfg(target_os = "macos")] -pub(in crate::compute) fn new_command_buffer( - queue: &CommandQueueRef, -) -> Result { - checked_command_buffer(queue).map_err(|source| { - metal_kernel_support_error("J2K Metal command buffer creation failed", source) - }) -} - -#[cfg(target_os = "macos")] -pub(in crate::compute) fn new_compute_command_encoder( - command_buffer: &CommandBufferRef, -) -> Result { - checked_compute_command_encoder(command_buffer).map_err(|source| { - metal_kernel_support_error("J2K Metal compute encoder creation failed", source) - }) -} - -#[cfg(target_os = "macos")] -pub(in crate::compute) fn new_blit_command_encoder( - command_buffer: &CommandBufferRef, -) -> Result { - checked_blit_command_encoder(command_buffer).map_err(|source| { - metal_kernel_support_error("J2K Metal blit encoder creation failed", source) - }) -} +use crate::{Error, Surface}; mod abi; #[cfg(target_os = "macos")] @@ -142,22 +69,23 @@ use self::direct_buffers::{ mod direct_commands; #[cfg(target_os = "macos")] use self::direct_commands::{ + new_blit_command_encoder, new_command_buffer, new_compute_command_encoder, DecodeHybridSplitCommandBuffers, DirectColorBatchCommandBuffers, DirectIdwtCommandBuffers, }; #[cfg(target_os = "macos")] mod direct_cpu; -#[cfg(all(target_os = "macos", test))] -use self::direct_cpu::decode_prepared_classic_sub_band_on_cpu; #[cfg(target_os = "macos")] use self::direct_cpu::{ decode_classic_inputs_on_cpu_with_plan_cache, decode_ht_inputs_on_cpu_with_plan_cache, decode_prepared_classic_jobs_on_cpu_with_scratch, decode_prepared_classic_jobs_on_cpu_with_scratch_profiled, - decode_prepared_classic_sub_band_group_on_cpu_profile, - decode_prepared_classic_sub_band_on_cpu_profile, decode_prepared_ht_jobs_on_cpu_with_workspace, - decode_prepared_ht_jobs_on_cpu_with_workspace_profiled, - decode_prepared_ht_sub_band_group_on_cpu_profile, decode_prepared_ht_sub_band_on_cpu_profile, - ClassicCpuDecodeInput, ClassicCpuDecodeScratch, HtCpuDecodeInput, + decode_prepared_ht_jobs_on_cpu_with_workspace, + decode_prepared_ht_jobs_on_cpu_with_workspace_profiled, ClassicCpuDecodeInput, + ClassicCpuDecodeScratch, HtCpuDecodeInput, +}; +#[cfg(all(target_os = "macos", test))] +use self::direct_cpu::{ + decode_prepared_classic_sub_band_on_cpu, decode_prepared_ht_sub_band_group_on_cpu_profile, }; #[cfg(target_os = "macos")] mod direct_flattened; @@ -175,10 +103,8 @@ use self::direct_profile::{ elapsed_us, emit_direct_hybrid_stage_timings, CpuTier1DecodeSubstageCounters, DirectHybridStageTimings, }; -mod direct_plan_support; -#[cfg(test)] -use self::direct_plan_support::prepared_direct_color_tier1_input_count; -use self::direct_plan_support::{ +mod direct_plan_validation; +use self::direct_plan_validation::{ classic_group_shapes_match, classic_sub_band_shapes_match, direct_preflight_invariant, ht_group_shapes_match, ht_sub_band_shapes_match, idwt_shapes_match, prepared_direct_color_plan_supports_runtime, store_shapes_match, @@ -190,10 +116,13 @@ use self::direct_scratch::{ }; #[cfg(target_os = "macos")] mod direct_status; +#[cfg(all(target_os = "macos", test))] +use self::direct_status::validate_direct_status; #[cfg(target_os = "macos")] use self::direct_status::{ decode_classic_status_error, decode_ht_status_error, decode_idwt_status_error, - decode_mct_status_error, validate_direct_status, DirectStatusCheck, + decode_mct_status_error, retire_direct_status_checks, DirectStatusCheck, + DirectStatusRetirementMode, }; #[cfg(target_os = "macos")] mod direct_tier1; @@ -202,8 +131,9 @@ use self::direct_tier1::{ flattened_hybrid_cpu_tier1_enabled, prepare_direct_tier1_input_buffer, record_flattened_hybrid_cpu_decode_batch, record_hybrid_cpu_decode_inputs, record_hybrid_cpu_decode_worker_init, record_hybrid_repeated_output_blit, - record_hybrid_stacked_component_batch, should_flatten_hybrid_cpu_tier1_color_batch, - DirectTier1Mode, HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK, + record_hybrid_stacked_component_batch, record_stacked_component_batch, + should_flatten_hybrid_cpu_tier1_color_batch, DirectTier1Mode, + HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK, }; mod pack_params; use self::pack_params::{j2k_pack_scale_arrays, j2k_scalar_pack_params, j2k_u32_param}; @@ -253,22 +183,27 @@ mod test_counters; pub(crate) use self::test_counters::{ classic_gpu_token_pack_dispatches_for_test, classic_split_mq_byte_gpu_token_pack_dispatches_for_test, - direct_tier1_input_buffer_prepares_for_test, flattened_hybrid_cpu_decode_batches_for_test, - ht_batch_coefficient_copy_blits_for_test, hybrid_cpu_decode_inputs_for_test, - hybrid_cpu_decode_worker_inits_for_test, hybrid_repeated_output_blits_for_test, - hybrid_stacked_component_batches_for_test, lossless_deinterleave_rct_fused_dispatches_for_test, - reset_classic_gpu_token_pack_dispatches_for_test, + direct_destination_event_bridge_for_test, direct_tier1_input_buffer_prepares_for_test, + direct_tier1_input_buffer_runtime_for_test, + flattened_hybrid_cpu_decode_batches_for_test, ht_batch_coefficient_copy_blits_for_test, + hybrid_cpu_decode_inputs_for_test, hybrid_cpu_decode_worker_inits_for_test, + hybrid_repeated_output_blits_for_test, hybrid_stacked_component_batches_for_test, + lossless_deinterleave_rct_fused_dispatches_for_test, metal_command_buffers_for_test, + metal_compute_encoders_for_test, reset_classic_gpu_token_pack_dispatches_for_test, reset_classic_split_mq_byte_gpu_token_pack_dispatches_for_test, + reset_direct_destination_event_bridge_for_test, reset_direct_tier1_input_buffer_prepares_for_test, reset_flattened_hybrid_cpu_decode_batches_for_test, reset_ht_batch_coefficient_copy_blits_for_test, reset_hybrid_cpu_decode_inputs_for_test, reset_hybrid_cpu_decode_worker_inits_for_test, reset_hybrid_repeated_output_blits_for_test, reset_hybrid_stacked_component_batches_for_test, reset_lossless_deinterleave_rct_fused_dispatches_for_test, + reset_metal_command_buffers_for_test, reset_metal_compute_encoders_for_test, reset_resident_codestream_command_buffer_waits_for_test, - reset_resident_gpu_timestamp_queries_for_test, reset_thread_hybrid_cpu_decode_inputs_for_test, + reset_resident_gpu_timestamp_queries_for_test, reset_stacked_component_batches_for_test, + reset_thread_hybrid_cpu_decode_inputs_for_test, resident_codestream_command_buffer_waits_for_test, resident_gpu_timestamp_queries_for_test, - thread_hybrid_cpu_decode_inputs_for_test, + stacked_component_batches_for_test, thread_hybrid_cpu_decode_inputs_for_test, }; #[cfg(target_os = "macos")] @@ -301,26 +236,18 @@ pub(crate) use self::runtime::{ #[cfg(all(target_os = "macos", test))] pub(crate) use j2k_metal_support::MetalSupportError; -#[cfg(target_os = "macos")] -fn checked_metal_surface_len( - dims: (u32, u32), - bytes_per_pixel: usize, - context: &'static str, -) -> Result<(usize, usize), Error> { - checked_surface_len(dims, bytes_per_pixel, usize::MAX, context).map_err(|error| { - Error::MetalKernel { - message: format!("{context}: {error}"), - } - }) -} - #[cfg(target_os = "macos")] mod direct_plan_types; +#[cfg(all(target_os = "macos", test))] +mod ht_forward_reader_tests; +#[cfg(all(target_os = "macos", test))] +mod ht_sigprop_context_tests; #[cfg(target_os = "macos")] use self::direct_plan_types::{ - HtCodedArena, PreparedClassicSubBand, PreparedClassicSubBandGroup, - PreparedClassicSubBandGroupMember, PreparedDirectGrayscaleStep, PreparedDirectIdwt, - PreparedHtSubBand, PreparedHtSubBandGroup, PreparedHtSubBandGroupMember, + PreparedClassicSubBand, PreparedClassicSubBandGroup, PreparedClassicSubBandGroupMember, + PreparedDirectGrayscaleStep, PreparedDirectIdwt, PreparedHtExecutionOwner, + PreparedHtPayloadSource, PreparedHtSubBand, PreparedHtSubBandGroup, + PreparedHtSubBandGroupMember, }; #[cfg(target_os = "macos")] pub(crate) use self::direct_plan_types::{PreparedDirectColorPlan, PreparedDirectGrayscalePlan}; @@ -336,10 +263,31 @@ use self::direct_plane_pack::{ #[cfg(target_os = "macos")] mod direct_grayscale_execute; #[cfg(target_os = "macos")] +pub(crate) use self::direct_grayscale_execute::{ + execute_hybrid_cpu_tier1_direct_color_plan, execute_hybrid_cpu_tier1_direct_color_plan_batch, + execute_hybrid_cpu_tier1_direct_color_plan_with_device, execute_prepared_direct_color_plan, + execute_prepared_direct_color_plan_batch, execute_prepared_direct_color_plan_with_device, + execute_prepared_direct_grayscale_plan, execute_prepared_direct_grayscale_plan_batch, + execute_prepared_direct_grayscale_plan_with_device, + execute_repeated_prepared_direct_grayscale_plan, + submit_prepared_direct_color_plan_batch_into_group, + submit_prepared_direct_grayscale_plan_batch_into_group, DirectDestinationConsumerOrdering, + SubmittedDirectDestination, +}; +#[cfg(target_os = "macos")] mod direct_prepare; #[cfg(target_os = "macos")] +pub(crate) use self::direct_prepare::{ + prepare_direct_grayscale_plan, prepare_referenced_classic_color_plan, + prepare_referenced_classic_grayscale_plan, prepare_referenced_classic_rgba_plan, + prepare_referenced_htj2k_color_plan, prepare_referenced_htj2k_grayscale_plan, + prepare_referenced_htj2k_rgba_plan, +}; +#[cfg(target_os = "macos")] mod direct_roi; #[cfg(target_os = "macos")] +pub(crate) use self::direct_roi::crop_prepared_direct_grayscale_plan_to_output_region; +#[cfg(target_os = "macos")] mod direct_stacked_batch; #[cfg(target_os = "macos")] use self::direct_stacked_batch::{ @@ -354,6 +302,8 @@ use self::direct_stacked_batch::{ #[cfg(target_os = "macos")] mod direct_surface_pack; #[cfg(all(target_os = "macos", test))] +use self::direct_surface_pack::checked_metal_surface_len; +#[cfg(all(target_os = "macos", test))] use self::direct_surface_pack::j2k_pack_kernel_name_for; #[cfg(target_os = "macos")] use self::direct_surface_pack::{ @@ -376,20 +326,71 @@ use self::direct_execute::{ #[cfg(target_os = "macos")] mod decode_cleanup; #[cfg(target_os = "macos")] +pub(crate) use self::decode_cleanup::{ + decode_classic_cleanup_code_block, decode_classic_cleanup_sub_band, + decode_ht_cleanup_code_block, decode_ht_cleanup_sub_band, +}; +#[cfg(target_os = "macos")] mod decode_dispatch; +#[cfg(all(target_os = "macos", test))] +pub(crate) use self::decode_dispatch::idwt::decode_irreversible97_staged_single_decomposition_idwt; +#[cfg(target_os = "macos")] +pub(crate) use self::decode_dispatch::idwt::{ + decode_irreversible97_single_decomposition_idwt, decode_reversible53_single_decomposition_idwt, +}; +#[cfg(target_os = "macos")] +pub(crate) use self::decode_dispatch::mct::decode_inverse_mct; +#[cfg(target_os = "macos")] +pub(crate) use self::decode_dispatch::store::decode_store_component_and_capture; #[cfg(target_os = "macos")] mod forward_transform; #[cfg(target_os = "macos")] +pub(crate) use self::forward_transform::{ + encode_deinterleave_to_f32, encode_forward_dwt53, encode_forward_dwt97, +}; +#[cfg(target_os = "macos")] mod lossless_prepare; #[cfg(target_os = "macos")] +pub(crate) use self::lossless_prepare::{ + encode_forward_ict, encode_forward_rct, encode_quantize_subband, + prepare_lossless_device_code_blocks, prepare_lossless_device_code_blocks_batch, +}; +#[cfg(target_os = "macos")] mod resident_codestream; #[cfg(target_os = "macos")] +pub(crate) use self::resident_codestream::{ + encode_lossless_codestream_buffer_from_resident_tier1, encode_tier2_packetization, + submit_lossless_codestream_buffers_from_prepared_classic_batch, + submit_lossless_codestream_buffers_from_prepared_ht_batch, +}; +#[cfg(target_os = "macos")] mod resident_tier1; -#[path = "compute/symbol_inventory.rs"] -mod symbol_inventory; +#[cfg(target_os = "macos")] +pub(crate) use self::resident_tier1::{ + wait_resident_lossless_codestream_batch, J2kLosslessCodestreamAssemblyJob, + J2kLosslessCodestreamBlockCodingMode, J2kLosslessDeviceBatchPrepareItem, + J2kLosslessDeviceCodeBlock, J2kLosslessDevicePrepareJob, + J2kPendingResidentLosslessCodestreamBatch, J2kPreparedLosslessDeviceCodeBlocks, + J2kResidentPacketizationEncodeJob, J2kResidentPacketizationResolution, + J2kResidentPacketizationSubband, +}; #[cfg(target_os = "macos")] mod tier1_encode; -symbol_inventory::wire_compute_symbols!(); +#[cfg(target_os = "macos")] +pub(crate) use self::tier1_encode::{ + encode_classic_tier1_code_block, encode_classic_tier1_code_blocks, + encode_classic_tier1_prepared_device_code_blocks_resident, encode_ht_cleanup_code_block, + encode_ht_cleanup_code_blocks, encode_ht_prepared_device_code_blocks_resident, + read_resident_ht_tier1_code_blocks_for_cpu_packetization, +}; +#[cfg(all(target_os = "macos", test))] +pub(crate) use self::tier1_encode::{ + encode_classic_tier1_code_blocks_via_gpu_token_pack_for_test, + encode_classic_tier1_code_blocks_via_ordered_tokens_cpu_pack_for_test, + encode_classic_tier1_code_blocks_via_split_mq_byte_raw_tokens_gpu_pack_for_test, + encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_cpu_pack_for_test, + encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_gpu_pack_for_test, +}; #[cfg(target_os = "macos")] fn required_classic_output_len(job: J2kCodeBlockDecodeJob<'_>) -> Result { @@ -409,19 +410,19 @@ fn required_classic_output_len(job: J2kCodeBlockDecodeJob<'_>) -> Result u32 { let mut flags = 0u32; if style.reset_context_probabilities { - flags |= J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES; + flags |= abi::J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES; } if style.termination_on_each_pass { - flags |= J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS; + flags |= abi::J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS; } if style.vertically_causal_context { - flags |= J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT; + flags |= abi::J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT; } if style.segmentation_symbols { - flags |= J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS; + flags |= abi::J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS; } if style.selective_arithmetic_coding_bypass { - flags |= J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS; + flags |= abi::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS; } flags } diff --git a/crates/j2k-metal/src/compute/abi.rs b/crates/j2k-metal/src/compute/abi.rs index 50798757..d5a908ac 100644 --- a/crates/j2k-metal/src/compute/abi.rs +++ b/crates/j2k-metal/src/compute/abi.rs @@ -421,6 +421,8 @@ pub(crate) struct J2kGrayStoreParams { pub(crate) copy_width: u32, pub(crate) copy_height: u32, pub(crate) output_width: u32, + pub(crate) output_stride: u32, + pub(crate) output_item_offset: u32, pub(crate) output_x: u32, pub(crate) output_y: u32, pub(crate) addend: f32, @@ -429,6 +431,23 @@ pub(crate) struct J2kGrayStoreParams { pub(crate) u16_scale: f32, } +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kNativeColorBatchStoreParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) plane_stride: u32, + pub(crate) output_row_stride: u32, + pub(crate) output_item_stride: u32, + pub(crate) batch_count: u32, + pub(crate) layout: u32, + pub(crate) mct: u32, + pub(crate) transform: u32, + pub(crate) signed: u32, + pub(crate) bit_depths: [u32; 4], +} + pub(crate) const J2K_HT_STATUS_OK: u32 = 0; #[cfg(target_os = "macos")] pub(crate) const J2K_HT_STATUS_FAIL: u32 = 1; @@ -1010,6 +1029,7 @@ impl_gpu_readback_abi!( J2kRepeatedStoreParams, J2kRepeatedGrayStoreParams, J2kGrayStoreParams, + J2kNativeColorBatchStoreParams, J2kHtCleanupParams, J2kHtCleanupBatchJob, J2kHtRepeatedBatchParams, diff --git a/crates/j2k-metal/src/compute/buffer_validation.rs b/crates/j2k-metal/src/compute/buffer_validation.rs index 180784de..ca0a5ed2 100644 --- a/crates/j2k-metal/src/compute/buffer_validation.rs +++ b/crates/j2k-metal/src/compute/buffer_validation.rs @@ -7,12 +7,12 @@ use metal::Buffer; use crate::{profile_env::label_command_buffer, Error}; +use super::abi::{J2kCopyInterleavedParams, J2kValidateBytesParams, J2kValidateBytesStatus}; use super::{ commit_and_wait_metal, direct_buffers::{checked_buffer_read, zeroed_shared_buffer}, new_command_buffer, new_compute_command_encoder, new_shared_buffer, - new_shared_buffer_with_slice, with_runtime_for_session, J2kCopyInterleavedParams, - J2kValidateBytesParams, J2kValidateBytesStatus, + new_shared_buffer_with_slice, with_runtime_for_session, }; pub(crate) fn validate_metal_buffer_matches_bytes( diff --git a/crates/j2k-metal/src/compute/classic_encode_pipeline.rs b/crates/j2k-metal/src/compute/classic_encode_pipeline.rs index 33b67853..de4fe98d 100644 --- a/crates/j2k-metal/src/compute/classic_encode_pipeline.rs +++ b/crates/j2k-metal/src/compute/classic_encode_pipeline.rs @@ -4,12 +4,13 @@ use metal::ComputePipelineState; use crate::profile_env::classic_selective_bypass_disabled; -use super::{ - J2kClassicEncodeBatchJob, MetalRuntime, J2K_CLASSIC_ENCODE_32_MAX_HEIGHT, - J2K_CLASSIC_ENCODE_32_MAX_WIDTH, J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES, - J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, +use super::abi::{ + J2kClassicEncodeBatchJob, J2K_CLASSIC_ENCODE_32_MAX_HEIGHT, J2K_CLASSIC_ENCODE_32_MAX_WIDTH, + J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES, J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS, + J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT, }; +use super::MetalRuntime; pub(super) fn classic_resident_style_flags_from_env() -> u32 { if classic_selective_bypass_disabled() { diff --git a/crates/j2k-metal/src/compute/classic_tier1_stats.rs b/crates/j2k-metal/src/compute/classic_tier1_stats.rs index 6020cf26..99497c14 100644 --- a/crates/j2k-metal/src/compute/classic_tier1_stats.rs +++ b/crates/j2k-metal/src/compute/classic_tier1_stats.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::{J2kResidentEncodeStageStats, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS}; +use super::abi::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS; +use super::J2kResidentEncodeStageStats; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(super) struct J2kClassicTier1PassClassCounts { diff --git a/crates/j2k-metal/src/compute/decode_cleanup.rs b/crates/j2k-metal/src/compute/decode_cleanup.rs index ab6a909f..997a4c23 100644 --- a/crates/j2k-metal/src/compute/decode_cleanup.rs +++ b/crates/j2k-metal/src/compute/decode_cleanup.rs @@ -1,11 +1,14 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use super::abi::{ + J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob, J2kHtCleanupParams, +}; +use super::decode_dispatch::{dispatch_classic_cleanup_batched, required_ht_output_len}; +use super::resident_codestream::{dispatch_ht_cleanup, dispatch_ht_cleanup_batched}; use super::{ - checked_buffer_slice, classic_style_flags, copied_slice_buffer, - dispatch_classic_cleanup_batched, dispatch_ht_cleanup, dispatch_ht_cleanup_batched, - required_classic_output_len, required_ht_output_len, with_runtime, Error, HtCodeBlockDecodeJob, - HtSubBandDecodeJob, J2kClassicCleanupBatchJob, J2kClassicSegment, J2kCodeBlockDecodeJob, - J2kHtCleanupBatchJob, J2kHtCleanupParams, J2kSubBandDecodeJob, + checked_buffer_slice, classic_style_flags, copied_slice_buffer, required_classic_output_len, + with_runtime, Error, HtCodeBlockDecodeJob, HtSubBandDecodeJob, J2kCodeBlockDecodeJob, + J2kSubBandDecodeJob, }; #[cfg(target_os = "macos")] diff --git a/crates/j2k-metal/src/compute/decode_dispatch.rs b/crates/j2k-metal/src/compute/decode_dispatch.rs index abd6aaf9..d93a64d6 100644 --- a/crates/j2k-metal/src/compute/decode_dispatch.rs +++ b/crates/j2k-metal/src/compute/decode_dispatch.rs @@ -1,35 +1,42 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::{ - checked_buffer_read, checked_buffer_slice, checked_metal_surface_len, commit_and_wait_metal, - copied_slice_buffer, decode_classic_status_error, decode_idwt_status_error, - decode_mct_status_error, dispatch_1d_pipeline, dispatch_2d_pipeline, dispatch_3d_pipeline, - dispatch_ht_cleanup_batched_in_command_buffer, dispatch_ht_cleanup_batched_in_encoder, - dispatch_ht_cleanup_repeated_batched_in_command_buffer, hybrid_stage_signpost, j2k_u32_param, - label_compute_encoder, new_command_buffer, new_compute_command_encoder, new_shared_buffer, - prepared_ht_buffer, size_of, take_classic_coefficients_scratch_buffer, - take_classic_states_scratch_buffer, with_runtime, zeroed_shared_buffer, Buffer, - CommandBufferRef, ComputeCommandEncoderRef, DirectIdwtCommandBuffers, DirectScratchBuffer, - DirectStatusCheck, Error, HtCodeBlockDecodeJob, HtRepeatedCleanupDispatch, +use std::mem::size_of; + +use j2k_metal_support::{dispatch_1d_pipeline, dispatch_2d_pipeline, dispatch_3d_pipeline}; + +use crate::profile_env::{ + hybrid_stage_signpost, label_compute_encoder, SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE, + SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE, SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE, +}; + +use super::abi::{ J2kClassicCleanupBatchJob, J2kClassicRepeatedBatchParams, J2kClassicSegment, J2kClassicStatus, - J2kGrayStoreParams, J2kHtCleanupBatchJob, J2kIdwt97StepParams, - J2kIdwtSingleDecompositionParams, J2kIdwtStatus, J2kInverseMctJob, J2kInverseMctParams, + J2kGrayStoreParams, J2kHtCleanupBatchJob, J2kHtRepeatedBatchParams, J2kHtStatus, + J2kIdwt97StepParams, J2kIdwtSingleDecompositionParams, J2kIdwtStatus, J2kInverseMctParams, J2kMctStatus, J2kRepeatedGrayStoreParams, J2kRepeatedIdwtSingleDecompositionParams, - J2kRepeatedStoreParams, J2kSingleDecompositionIdwtJob, J2kStoreComponentJob, J2kStoreParams, - J2kWaveletTransform, MTLSize, MetalRuntime, PixelFormat, PreparedClassicSubBand, - PreparedClassicSubBandGroup, PreparedHtSubBand, PreparedHtSubBandGroup, Surface, - J2K_CLASSIC_MAX_HEIGHT, J2K_CLASSIC_MAX_WIDTH, J2K_CLASSIC_STATUS_OK, J2K_IDWT_STATUS_OK, - J2K_MCT_STATUS_OK, SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE, - SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE, SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE, + J2kRepeatedStoreParams, J2kStoreParams, J2K_CLASSIC_MAX_HEIGHT, J2K_CLASSIC_MAX_WIDTH, + J2K_CLASSIC_STATUS_OK, J2K_IDWT_STATUS_OK, J2K_MCT_STATUS_OK, +}; +use super::{ + checked_buffer_read, checked_buffer_slice, commit_and_wait_metal, copied_slice_buffer, + decode_classic_status_error, decode_idwt_status_error, decode_mct_status_error, j2k_u32_param, + new_command_buffer, new_compute_command_encoder, new_shared_buffer, + take_classic_coefficients_scratch_buffer, take_classic_states_scratch_buffer, with_runtime, + zeroed_shared_buffer, Buffer, CommandBufferRef, ComputeCommandEncoderRef, + DirectIdwtCommandBuffers, DirectScratchBuffer, DirectStatusCheck, Error, HtCodeBlockDecodeJob, + J2kInverseMctJob, J2kSingleDecompositionIdwtJob, J2kStoreComponentJob, J2kWaveletTransform, + MTLSize, MetalRuntime, PixelFormat, PreparedClassicSubBand, PreparedClassicSubBandGroup, + PreparedHtSubBand, PreparedHtSubBandGroup, Surface, }; mod classic_cleanup; mod classic_subband; +mod ht_chunks; mod ht_distinct; mod ht_subband; -mod idwt; -mod mct; -mod store; +pub(in crate::compute) mod idwt; +pub(in crate::compute) mod mct; +pub(in crate::compute) mod store; pub(in crate::compute) use self::classic_cleanup::{ classic_batch_is_plain_arithmetic, classic_batch_uses_plain_fast_path, @@ -39,7 +46,9 @@ pub(in crate::compute) use self::classic_cleanup::{ dispatch_classic_cleanup_repeated_batched_in_command_buffer, dispatch_classic_store_repeated_batched_in_command_buffer, encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer, - encode_distinct_classic_sub_bands_to_buffer_in_command_buffer, ClassicCleanupBatchDispatch, + encode_distinct_classic_sub_band_groups_to_buffer_in_encoder, + encode_distinct_classic_sub_bands_to_buffer_in_command_buffer, + encode_distinct_classic_sub_bands_to_buffer_in_encoder, ClassicCleanupBatchDispatch, ClassicPlainDevRepeatedCleanupDispatch, ClassicRepeatedCleanupDispatch, ClassicRepeatedStoreDispatch, }; @@ -49,9 +58,16 @@ pub(in crate::compute) use self::classic_subband::{ encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer, encode_repeated_classic_sub_band_to_buffer_in_command_buffer, }; +pub(in crate::compute) use self::ht_chunks::{ + default_metal_ht_chunk_limits, encode_metal_ht_batches_in_encoder, + encode_repeated_metal_ht_batch_in_command_buffer, HtBatchInput, HtPayloadSource, + MetalHtPipelineKind, PreparedMetalHtExecutionCache, +}; pub(in crate::compute) use self::ht_distinct::{ encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer, + encode_distinct_ht_sub_band_groups_to_buffer_in_encoder, encode_distinct_ht_sub_bands_to_buffer_in_command_buffer, + encode_distinct_ht_sub_bands_to_buffer_in_encoder, }; pub(in crate::compute) use self::ht_subband::{ dispatch_zero_u32_buffer_in_encoder, encode_prepared_ht_sub_band_group_to_buffer_in_encoder, @@ -60,27 +76,20 @@ pub(in crate::compute) use self::ht_subband::{ encode_repeated_ht_sub_band_to_buffer_in_command_buffer, ht_batch_output_word_count, ht_output_word_count, required_ht_output_len, }; -#[cfg(test)] -pub(crate) use self::idwt::decode_irreversible97_staged_single_decomposition_idwt; -pub(crate) use self::idwt::{ - decode_irreversible97_single_decomposition_idwt, decode_reversible53_single_decomposition_idwt, -}; pub(in crate::compute) use self::idwt::{ dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets, dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets, dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets, + dispatch_reversible53_repeated_buffers_in_encoder_with_offsets, dispatch_reversible53_single_decomposition_buffers_in_command_buffer_with_offsets, dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets, IdwtSubBandBuffers, RepeatedIdwtDispatch, SingleIdwtDispatch, }; -pub(crate) use self::mct::decode_inverse_mct; -pub(in crate::compute) use self::mct::dispatch_inverse_mct_buffers_in_command_buffer; -pub(crate) use self::store::decode_store_component_and_capture; -#[cfg(test)] -pub(in crate::compute) use self::store::repeated_gray_store_is_contiguous_full_surface; pub(in crate::compute) use self::store::{ dispatch_store_component_buffer_in_command_buffer_with_offsets, dispatch_store_component_buffer_in_encoder_with_offsets, - dispatch_store_component_repeated_in_command_buffer, encode_gray_store_to_surface_in_encoder, - encode_repeated_gray_store_to_surfaces_in_command_buffer, + dispatch_store_component_repeated_in_command_buffer, + dispatch_store_component_repeated_in_encoder, encode_gray_store_to_destination_in_encoder, + encode_gray_store_to_surface_in_encoder, + encode_repeated_gray_store_to_surfaces_in_command_buffer, GrayStoreDestinationRequest, }; diff --git a/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup.rs b/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup.rs index 0a2d0c39..fd479ace 100644 --- a/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup.rs +++ b/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup.rs @@ -2,19 +2,25 @@ use super::{ checked_buffer_slice, commit_and_wait_metal, copied_slice_buffer, decode_classic_status_error, - j2k_u32_param, new_command_buffer, new_compute_command_encoder, new_shared_buffer, size_of, + j2k_u32_param, new_command_buffer, new_compute_command_encoder, size_of, take_classic_coefficients_scratch_buffer, zeroed_shared_buffer, Buffer, CommandBufferRef, - ComputeCommandEncoderRef, DirectScratchBuffer, DirectStatusCheck, Error, - J2kClassicCleanupBatchJob, J2kClassicRepeatedBatchParams, J2kClassicSegment, J2kClassicStatus, - MTLSize, MetalRuntime, PreparedClassicSubBand, PreparedClassicSubBandGroup, + ComputeCommandEncoderRef, DirectStatusCheck, Error, J2kClassicCleanupBatchJob, + J2kClassicRepeatedBatchParams, J2kClassicSegment, J2kClassicStatus, MTLSize, MetalRuntime, J2K_CLASSIC_MAX_HEIGHT, J2K_CLASSIC_MAX_WIDTH, J2K_CLASSIC_STATUS_OK, }; #[cfg(target_os = "macos")] mod distinct_allocation; +#[cfg(target_os = "macos")] +mod distinct_batch; #[cfg(target_os = "macos")] -use self::distinct_allocation::{allocate_distinct_classic_metadata, DistinctClassicMetadata}; +pub(in crate::compute) use self::distinct_batch::{ + encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer, + encode_distinct_classic_sub_band_groups_to_buffer_in_encoder, + encode_distinct_classic_sub_bands_to_buffer_in_command_buffer, + encode_distinct_classic_sub_bands_to_buffer_in_encoder, +}; #[cfg(target_os = "macos")] pub(in crate::compute) fn classic_batch_uses_plain_fast_path( @@ -207,29 +213,6 @@ pub(in crate::compute) struct ClassicRepeatedStoreDispatch<'a> { pub(in crate::compute) coefficients_scratch: &'a Buffer, } -#[cfg(target_os = "macos")] -pub(in crate::compute) fn dispatch_classic_cleanup_batched_in_command_buffer( - command_buffer: &CommandBufferRef, - dispatch: ClassicCleanupBatchDispatch<'_>, -) -> Result<(DirectStatusCheck, Option), Error> { - let status_buffer = zeroed_shared_buffer( - &dispatch.runtime.device, - dispatch.job_count.max(1) * size_of::(), - )?; - - let encoder = new_compute_command_encoder(command_buffer)?; - dispatch_classic_cleanup_batched_in_encoder_with_status(&encoder, dispatch, &status_buffer); - encoder.end_encoding(); - - Ok(( - DirectStatusCheck::Classic { - buffer: status_buffer, - len: dispatch.job_count, - }, - None, - )) -} - #[cfg(target_os = "macos")] pub(in crate::compute) fn dispatch_classic_cleanup_batched_in_encoder( encoder: &ComputeCommandEncoderRef, @@ -512,221 +495,3 @@ pub(in crate::compute) fn dispatch_classic_store_repeated_batched_in_command_buf encoder.end_encoding(); Ok(()) } - -#[cfg(target_os = "macos")] -pub(in crate::compute) fn encode_distinct_classic_sub_bands_to_buffer_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - sub_bands: &[&PreparedClassicSubBand], - output: &Buffer, - scratch_buffers: &mut Vec, -) -> Result<(Vec, DirectStatusCheck), Error> { - let Some(first) = sub_bands.first() else { - let empty = new_shared_buffer(&runtime.device, 1)?; - return Ok(( - vec![empty.clone()], - DirectStatusCheck::Classic { - buffer: empty, - len: 0, - }, - )); - }; - let per_instance_len = first.width as usize * first.height as usize; - encode_distinct_classic_batches_to_buffer_in_command_buffer( - runtime, - command_buffer, - sub_bands - .iter() - .enumerate() - .map(|(index, sub_band)| DistinctClassicBatch { - coded_data: &sub_band.coded_data, - jobs: &sub_band.jobs, - segments: &sub_band.segments, - output_base: index * per_instance_len, - }), - output, - scratch_buffers, - ) -} - -#[cfg(target_os = "macos")] -pub(in crate::compute) fn encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - groups: &[&PreparedClassicSubBandGroup], - output: &Buffer, - scratch_buffers: &mut Vec, -) -> Result<(Vec, DirectStatusCheck), Error> { - let Some(first) = groups.first() else { - let empty = new_shared_buffer(&runtime.device, 1)?; - return Ok(( - vec![empty.clone()], - DirectStatusCheck::Classic { - buffer: empty, - len: 0, - }, - )); - }; - let per_instance_len = first.total_coefficients; - encode_distinct_classic_batches_to_buffer_in_command_buffer( - runtime, - command_buffer, - groups - .iter() - .enumerate() - .map(|(index, group)| DistinctClassicBatch { - coded_data: &group.coded_data, - jobs: &group.jobs, - segments: &group.segments, - output_base: index * per_instance_len, - }), - output, - scratch_buffers, - ) -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -pub(in crate::compute) struct DistinctClassicBatch<'a> { - pub(in crate::compute) coded_data: &'a [u8], - pub(in crate::compute) jobs: &'a [J2kClassicCleanupBatchJob], - pub(in crate::compute) segments: &'a [J2kClassicSegment], - pub(in crate::compute) output_base: usize, -} - -#[cfg(target_os = "macos")] -fn append_distinct_classic_batch( - metadata: &mut DistinctClassicMetadata, - batch: DistinctClassicBatch<'_>, -) -> Result<(), Error> { - let coded_base = u32::try_from(metadata.coded_data.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect distinct color coded payload exceeds u32".to_string(), - })?; - let segment_base = u32::try_from(metadata.segments.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect distinct color segment table exceeds u32".to_string(), - })?; - metadata.coded_data.extend_from_slice(batch.coded_data); - for segment in batch.segments { - let mut adjusted = *segment; - adjusted.data_offset = adjusted - .data_offset - .checked_add(coded_base) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect distinct color segment offset overflow" - .to_string(), - })?; - metadata.segments.push(adjusted); - } - let output_base = u32::try_from(batch.output_base).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect distinct color output offset exceeds u32".to_string(), - })?; - for job in batch.jobs { - let mut adjusted = *job; - adjusted.coded_offset = adjusted - .coded_offset - .checked_add(coded_base) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect distinct color job coded offset overflow" - .to_string(), - })?; - adjusted.segment_offset = adjusted - .segment_offset - .checked_add(segment_base) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect distinct color job segment offset overflow" - .to_string(), - })?; - adjusted.output_offset = - adjusted - .output_offset - .checked_add(output_base) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect distinct color job output offset overflow" - .to_string(), - })?; - metadata.jobs.push(adjusted); - } - Ok(()) -} - -#[cfg(target_os = "macos")] -pub(in crate::compute) fn encode_distinct_classic_batches_to_buffer_in_command_buffer<'a>( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - batches: impl Iterator> + Clone, - output: &Buffer, - scratch_buffers: &mut Vec, -) -> Result<(Vec, DirectStatusCheck), Error> { - let coded_len = crate::batch_allocation::checked_count_sum( - batches.clone().map(|batch| batch.coded_data.len()), - "classic J2K MetalDirect distinct color coded payload", - )?; - let job_count = crate::batch_allocation::checked_count_sum( - batches.clone().map(|batch| batch.jobs.len()), - "classic J2K MetalDirect distinct color jobs", - )?; - let segment_count = crate::batch_allocation::checked_count_sum( - batches.clone().map(|batch| batch.segments.len()), - "classic J2K MetalDirect distinct color segments", - )?; - let mut metadata = allocate_distinct_classic_metadata( - coded_len, - job_count, - segment_count, - crate::batch_allocation::BatchMetadataBudget::new( - "classic J2K MetalDirect distinct color submission", - ), - )?; - - for batch in batches { - append_distinct_classic_batch(&mut metadata, batch)?; - } - let DistinctClassicMetadata { - coded_data, - jobs, - segments, - } = metadata; - - if jobs.is_empty() { - let empty = new_shared_buffer(&runtime.device, 1)?; - return Ok(( - vec![empty.clone()], - DirectStatusCheck::Classic { - buffer: empty, - len: 0, - }, - )); - } - - let coded_buffer = copied_slice_buffer(&runtime.device, &coded_data)?; - let jobs_buffer = copied_slice_buffer(&runtime.device, &jobs)?; - let segments_buffer = copied_slice_buffer(&runtime.device, &segments)?; - let use_plain_fast_path = classic_batch_uses_plain_fast_path(&jobs, &segments) - && runtime - .classic_cleanup_plain_batched - .max_total_threads_per_threadgroup() - >= 32; - let coefficients_scratch = take_classic_coefficients_scratch_buffer(runtime, jobs.len())?; - let (status_check, states_scratch) = dispatch_classic_cleanup_batched_in_command_buffer( - command_buffer, - ClassicCleanupBatchDispatch { - runtime, - coded_data: &coded_buffer, - jobs: &jobs_buffer, - job_count: jobs.len(), - use_plain_fast_path, - segments: &segments_buffer, - decoded: output, - coefficients_scratch: &coefficients_scratch.buffer, - }, - )?; - let mut retained_buffers = vec![coded_buffer, jobs_buffer, segments_buffer]; - scratch_buffers.push(coefficients_scratch); - if let Some(states_scratch) = states_scratch { - retained_buffers.push(states_scratch); - } - Ok((retained_buffers, status_check)) -} - -#[cfg(all(test, target_os = "macos"))] -mod distinct_metadata_tests; diff --git a/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_batch.rs b/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_batch.rs new file mode 100644 index 00000000..bc3b98d9 --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_batch.rs @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::ht_subband::dispatch_zero_u32_buffer_in_encoder; +use super::{ + classic_batch_uses_plain_fast_path, dispatch_classic_cleanup_batched_in_encoder, + distinct_allocation::{allocate_distinct_classic_metadata, DistinctClassicMetadata}, + ClassicCleanupBatchDispatch, +}; +use crate::compute::abi::{J2kClassicCleanupBatchJob, J2kClassicSegment}; +use crate::compute::{ + copied_slice_buffer, new_compute_command_encoder, new_shared_buffer, + take_classic_coefficients_scratch_buffer, Buffer, CommandBufferRef, ComputeCommandEncoderRef, + DirectScratchBuffer, DirectStatusCheck, Error, MetalRuntime, PreparedClassicSubBand, + PreparedClassicSubBandGroup, +}; + +pub(in crate::compute) fn encode_distinct_classic_sub_bands_to_buffer_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + sub_bands: &[&PreparedClassicSubBand], + output: &Buffer, + scratch_buffers: &mut Vec, +) -> Result<(Vec, DirectStatusCheck), Error> { + let encoder = new_compute_command_encoder(command_buffer)?; + let result = encode_distinct_classic_sub_bands_to_buffer_in_encoder( + runtime, + &encoder, + sub_bands, + output, + scratch_buffers, + ); + encoder.end_encoding(); + result +} + +pub(in crate::compute) fn encode_distinct_classic_sub_bands_to_buffer_in_encoder( + runtime: &MetalRuntime, + encoder: &ComputeCommandEncoderRef, + sub_bands: &[&PreparedClassicSubBand], + output: &Buffer, + scratch_buffers: &mut Vec, +) -> Result<(Vec, DirectStatusCheck), Error> { + let Some(first) = sub_bands.first() else { + let empty = new_shared_buffer(&runtime.device, 1)?; + return Ok(( + vec![empty.clone()], + DirectStatusCheck::Classic { + buffer: empty, + len: 0, + }, + )); + }; + let per_instance_len = first.width as usize * first.height as usize; + encode_distinct_classic_batches_to_buffer_in_encoder( + runtime, + encoder, + sub_bands + .iter() + .enumerate() + .map(|(index, sub_band)| DistinctClassicBatch { + coded_data: &sub_band.coded_data, + jobs: &sub_band.jobs, + segments: &sub_band.segments, + output_base: index * per_instance_len, + output_len: per_instance_len, + zero_fill: sub_band.zero_fill, + }), + output, + scratch_buffers, + ) +} + +pub(in crate::compute) fn encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + groups: &[&PreparedClassicSubBandGroup], + output: &Buffer, + scratch_buffers: &mut Vec, +) -> Result<(Vec, DirectStatusCheck), Error> { + let encoder = new_compute_command_encoder(command_buffer)?; + let result = encode_distinct_classic_sub_band_groups_to_buffer_in_encoder( + runtime, + &encoder, + groups, + output, + scratch_buffers, + ); + encoder.end_encoding(); + result +} + +pub(in crate::compute) fn encode_distinct_classic_sub_band_groups_to_buffer_in_encoder( + runtime: &MetalRuntime, + encoder: &ComputeCommandEncoderRef, + groups: &[&PreparedClassicSubBandGroup], + output: &Buffer, + scratch_buffers: &mut Vec, +) -> Result<(Vec, DirectStatusCheck), Error> { + let Some(first) = groups.first() else { + let empty = new_shared_buffer(&runtime.device, 1)?; + return Ok(( + vec![empty.clone()], + DirectStatusCheck::Classic { + buffer: empty, + len: 0, + }, + )); + }; + let per_instance_len = first.total_coefficients; + encode_distinct_classic_batches_to_buffer_in_encoder( + runtime, + encoder, + groups + .iter() + .enumerate() + .map(|(index, group)| DistinctClassicBatch { + coded_data: &group.coded_data, + jobs: &group.jobs, + segments: &group.segments, + output_base: index * per_instance_len, + output_len: per_instance_len, + zero_fill: group.zero_fill, + }), + output, + scratch_buffers, + ) +} + +#[derive(Clone, Copy)] +struct DistinctClassicBatch<'a> { + coded_data: &'a [u8], + jobs: &'a [J2kClassicCleanupBatchJob], + segments: &'a [J2kClassicSegment], + output_base: usize, + output_len: usize, + zero_fill: bool, +} + +fn append_distinct_classic_batch( + metadata: &mut DistinctClassicMetadata, + batch: DistinctClassicBatch<'_>, +) -> Result<(), Error> { + let coded_base = u32::try_from(metadata.coded_data.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect distinct color coded payload exceeds u32".to_string(), + })?; + let segment_base = u32::try_from(metadata.segments.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect distinct color segment table exceeds u32".to_string(), + })?; + metadata.coded_data.extend_from_slice(batch.coded_data); + for segment in batch.segments { + let mut adjusted = *segment; + adjusted.data_offset = adjusted + .data_offset + .checked_add(coded_base) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect distinct color segment offset overflow" + .to_string(), + })?; + metadata.segments.push(adjusted); + } + let output_base = u32::try_from(batch.output_base).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect distinct color output offset exceeds u32".to_string(), + })?; + for job in batch.jobs { + let mut adjusted = *job; + adjusted.coded_offset = adjusted + .coded_offset + .checked_add(coded_base) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect distinct color job coded offset overflow" + .to_string(), + })?; + adjusted.segment_offset = adjusted + .segment_offset + .checked_add(segment_base) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect distinct color job segment offset overflow" + .to_string(), + })?; + adjusted.output_offset = + adjusted + .output_offset + .checked_add(output_base) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect distinct color job output offset overflow" + .to_string(), + })?; + metadata.jobs.push(adjusted); + } + Ok(()) +} + +fn encode_distinct_classic_batches_to_buffer_in_encoder<'a>( + runtime: &MetalRuntime, + encoder: &ComputeCommandEncoderRef, + batches: impl Iterator> + Clone, + output: &Buffer, + scratch_buffers: &mut Vec, +) -> Result<(Vec, DirectStatusCheck), Error> { + let zero_fill_word_count = batches.clone().try_fold(0usize, |word_count, batch| { + if !batch.zero_fill && !batch.jobs.is_empty() { + return Ok(word_count); + } + batch + .output_base + .checked_add(batch.output_len) + .map(|end| word_count.max(end)) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect distinct output span overflow".to_string(), + }) + })?; + let coded_len = crate::batch_allocation::checked_count_sum( + batches.clone().map(|batch| batch.coded_data.len()), + "classic J2K MetalDirect distinct color coded payload", + )?; + let job_count = crate::batch_allocation::checked_count_sum( + batches.clone().map(|batch| batch.jobs.len()), + "classic J2K MetalDirect distinct color jobs", + )?; + let segment_count = crate::batch_allocation::checked_count_sum( + batches.clone().map(|batch| batch.segments.len()), + "classic J2K MetalDirect distinct color segments", + )?; + let mut metadata = allocate_distinct_classic_metadata( + coded_len, + job_count, + segment_count, + crate::batch_allocation::BatchMetadataBudget::new( + "classic J2K MetalDirect distinct color submission", + ), + )?; + + for batch in batches { + append_distinct_classic_batch(&mut metadata, batch)?; + } + let DistinctClassicMetadata { + coded_data, + jobs, + segments, + } = metadata; + + dispatch_zero_u32_buffer_in_encoder(runtime, encoder, output, zero_fill_word_count)?; + + if jobs.is_empty() { + let empty = new_shared_buffer(&runtime.device, 1)?; + return Ok(( + vec![empty.clone()], + DirectStatusCheck::Classic { + buffer: empty, + len: 0, + }, + )); + } + if zero_fill_word_count != 0 { + encoder.memory_barrier_with_resources(&[output]); + } + + let coded_buffer = copied_slice_buffer(&runtime.device, &coded_data)?; + let jobs_buffer = copied_slice_buffer(&runtime.device, &jobs)?; + let segments_buffer = copied_slice_buffer(&runtime.device, &segments)?; + let use_plain_fast_path = classic_batch_uses_plain_fast_path(&jobs, &segments) + && runtime + .classic_cleanup_plain_batched + .max_total_threads_per_threadgroup() + >= 32; + let coefficients_scratch = take_classic_coefficients_scratch_buffer(runtime, jobs.len())?; + let (status_check, states_scratch) = dispatch_classic_cleanup_batched_in_encoder( + encoder, + ClassicCleanupBatchDispatch { + runtime, + coded_data: &coded_buffer, + jobs: &jobs_buffer, + job_count: jobs.len(), + use_plain_fast_path, + segments: &segments_buffer, + decoded: output, + coefficients_scratch: &coefficients_scratch.buffer, + }, + )?; + let mut retained_buffers = vec![coded_buffer, jobs_buffer, segments_buffer]; + scratch_buffers.push(coefficients_scratch); + if let Some(states_scratch) = states_scratch { + retained_buffers.push(states_scratch); + } + Ok((retained_buffers, status_check)) +} + +#[cfg(test)] +#[path = "distinct_metadata_tests.rs"] +mod tests; diff --git a/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_metadata_tests.rs b/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_metadata_tests.rs index 10950608..ad6b093a 100644 --- a/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_metadata_tests.rs +++ b/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_metadata_tests.rs @@ -5,9 +5,33 @@ use core::mem::size_of; use j2k_core::BatchInfrastructureError; use super::{ - allocate_distinct_classic_metadata, Error, J2kClassicCleanupBatchJob, J2kClassicSegment, + allocate_distinct_classic_metadata, encode_distinct_classic_batches_to_buffer_in_encoder, + DistinctClassicBatch, Error, J2kClassicCleanupBatchJob, J2kClassicSegment, }; use crate::batch_allocation::BatchMetadataBudget; +use crate::compute::{ + checked_buffer_slice, commit_and_wait_metal, new_command_buffer, new_compute_command_encoder, + new_shared_buffer_with_slice, validate_direct_status, with_runtime, +}; + +#[test] +fn distinct_classic_zero_fill_is_barriered_before_cleanup_dispatch() { + let source = include_str!("distinct_batch.rs"); + let body = source + .split_once("fn encode_distinct_classic_batches_to_buffer_in_encoder") + .expect("distinct classic batch encoder") + .1; + let zero_fill = body + .find("dispatch_zero_u32_buffer_in_encoder(runtime, encoder, output") + .expect("distinct classic zero-fill dispatch"); + let barrier = body + .find("encoder.memory_barrier_with_resources(&[output]);") + .expect("distinct classic zero-fill resource barrier"); + let cleanup = body + .find("dispatch_classic_cleanup_batched_in_encoder(") + .expect("distinct classic cleanup dispatch"); + assert!(zero_fill < barrier && barrier < cleanup); +} #[test] fn distinct_classic_metadata_honors_exact_cap_and_one_byte_over() { @@ -50,3 +74,100 @@ fn distinct_classic_metadata_honors_exact_cap_and_one_byte_over() { )) if requested == exact_cap && cap == exact_cap - 1 )); } + +#[test] +fn distinct_classic_batches_honor_empty_and_zero_fill_output_semantics() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + with_runtime(|runtime| { + let output = new_shared_buffer_with_slice(&runtime.device, &[7.0_f32; 8])?; + let command_buffer = new_command_buffer(&runtime.queue)?; + let encoder = new_compute_command_encoder(&command_buffer)?; + let batches = [ + DistinctClassicBatch { + coded_data: &[], + jobs: &[], + segments: &[], + output_base: 0, + output_len: 4, + zero_fill: false, + }, + DistinctClassicBatch { + coded_data: &[], + jobs: &[], + segments: &[], + output_base: 4, + output_len: 4, + zero_fill: false, + }, + ]; + let mut scratch_buffers = Vec::new(); + let (retained, status) = encode_distinct_classic_batches_to_buffer_in_encoder( + runtime, + &encoder, + batches.into_iter(), + &output, + &mut scratch_buffers, + )?; + encoder.end_encoding(); + commit_and_wait_metal(&command_buffer)?; + validate_direct_status(runtime, status)?; + assert_eq!( + checked_buffer_slice::(&output, 8, "distinct classic empty batch output")?, + vec![0.0; 8] + ); + drop(retained); + drop(scratch_buffers); + + let zero_pass_job = J2kClassicCleanupBatchJob { + coded_offset: 0, + coded_len: 0, + segment_offset: 0, + segment_count: 0, + width: 4, + height: 1, + output_stride: 4, + output_offset: 0, + missing_msbs: 0, + total_bitplanes: 1, + roi_shift: 0, + number_of_coding_passes: 0, + sub_band_type: 0, + style_flags: 1, + strict: 1, + dequantization_step: 1.0, + }; + let output = new_shared_buffer_with_slice(&runtime.device, &[11.0_f32; 4])?; + let command_buffer = new_command_buffer(&runtime.queue)?; + let encoder = new_compute_command_encoder(&command_buffer)?; + let batches = [DistinctClassicBatch { + coded_data: &[], + jobs: core::slice::from_ref(&zero_pass_job), + segments: &[], + output_base: 0, + output_len: 4, + zero_fill: true, + }]; + let mut scratch_buffers = Vec::new(); + let (retained, status) = encode_distinct_classic_batches_to_buffer_in_encoder( + runtime, + &encoder, + batches.into_iter(), + &output, + &mut scratch_buffers, + )?; + encoder.end_encoding(); + commit_and_wait_metal(&command_buffer)?; + validate_direct_status(runtime, status)?; + assert_eq!( + checked_buffer_slice::(&output, 4, "distinct classic zero-fill output")?, + vec![0.0; 4] + ); + drop(retained); + drop(scratch_buffers); + Ok(()) + }) + .expect("distinct empty classic batches"); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/classic_subband.rs b/crates/j2k-metal/src/compute/decode_dispatch/classic_subband.rs index 121113d1..6da1bcee 100644 --- a/crates/j2k-metal/src/compute/decode_dispatch/classic_subband.rs +++ b/crates/j2k-metal/src/compute/decode_dispatch/classic_subband.rs @@ -301,6 +301,7 @@ pub(in crate::compute) fn encode_prepared_classic_sub_band_to_buffer_in_encoder( output, job.width as usize * job.height as usize, )?; + encoder.memory_barrier_with_resources(&[output]); } let (status_check, states_scratch) = dispatch_classic_cleanup_batched_in_encoder( encoder, @@ -354,6 +355,7 @@ pub(in crate::compute) fn encode_prepared_classic_sub_band_group_to_buffer_in_en let coefficients_scratch = take_classic_coefficients_scratch_buffer(runtime, group.jobs.len())?; if group.zero_fill { dispatch_zero_u32_buffer_in_encoder(runtime, encoder, output, group.total_coefficients)?; + encoder.memory_barrier_with_resources(&[output]); } let (status_check, states_scratch) = dispatch_classic_cleanup_batched_in_encoder( encoder, @@ -375,3 +377,6 @@ pub(in crate::compute) fn encode_prepared_classic_sub_band_group_to_buffer_in_en } Ok((retained_buffers, status_check)) } + +#[cfg(all(test, target_os = "macos"))] +mod tests; diff --git a/crates/j2k-metal/src/compute/decode_dispatch/classic_subband/tests.rs b/crates/j2k-metal/src/compute/decode_dispatch/classic_subband/tests.rs new file mode 100644 index 00000000..713b37ff --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/classic_subband/tests.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +fn function_body<'a>(source: &'a str, name: &str) -> &'a str { + source + .split_once(name) + .unwrap_or_else(|| panic!("missing {name}")) + .1 + .split("\n#[cfg(") + .next() + .expect("classic sub-band function body") +} + +#[test] +fn prepared_classic_zero_fill_is_barriered_before_cleanup_dispatch() { + let source = include_str!("../classic_subband.rs"); + for name in [ + "fn encode_prepared_classic_sub_band_to_buffer_in_encoder", + "fn encode_prepared_classic_sub_band_group_to_buffer_in_encoder", + ] { + let body = function_body(source, name); + let zero_fill = body + .find("dispatch_zero_u32_buffer_in_encoder") + .unwrap_or_else(|| panic!("missing zero-fill dispatch in {name}")); + let barrier = body + .find("encoder.memory_barrier_with_resources(&[output]);") + .unwrap_or_else(|| panic!("missing zero-fill resource barrier in {name}")); + let cleanup = body + .find("dispatch_classic_cleanup_batched_in_encoder(") + .unwrap_or_else(|| panic!("missing cleanup dispatch in {name}")); + assert!(zero_fill < barrier && barrier < cleanup, "{name}"); + } +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks.rs new file mode 100644 index 00000000..2b0d4064 --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks.rs @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +mod execution; +mod prepared; + +#[cfg(test)] +mod tests; + +use core::{mem::size_of, num::NonZeroUsize}; + +use j2k_core::{ + plan_ht_gpu_job_chunks, HtGpuJobChunkLimits, HtGpuJobChunkPlan, HtGpuJobChunkPlanError, + HtGpuJobChunkRequest, HtGpuJobPassBucket, +}; +use j2k_native::HtCodeBlockPayloadRanges; + +use std::sync::Arc; + +use super::{Error, J2kHtCleanupBatchJob}; +use crate::compute::{PreparedHtExecutionOwner, PreparedHtPayloadSource}; + +pub(in crate::compute) use execution::{ + encode_metal_ht_batches_in_encoder, encode_repeated_metal_ht_batch_in_command_buffer, +}; +pub(in crate::compute) use prepared::{prepared_metal_ht_execution, PreparedMetalHtExecutionCache}; + +const MAX_METAL_HT_JOBS_PER_CHUNK: usize = 16_384; +const MAX_METAL_HT_PAYLOAD_BYTES_PER_CHUNK: usize = 64 * 1024 * 1024; +const MAX_METAL_HT_DESCRIPTOR_BYTES_PER_CHUNK: usize = + MAX_METAL_HT_JOBS_PER_CHUNK * size_of::(); + +#[derive(Clone, Copy)] +pub(in crate::compute) struct HtBatchInput<'a> { + pub(in crate::compute) source_index: usize, + pub(in crate::compute) payload: HtPayloadSource<'a>, + pub(in crate::compute) jobs: &'a [J2kHtCleanupBatchJob], + pub(in crate::compute) output_base: usize, + pub(in crate::compute) execution_owner: &'a Arc, +} + +#[derive(Clone, Copy)] +pub(in crate::compute) enum HtPayloadSource<'a> { + Contiguous(&'a [u8]), + Referenced { + input: &'a Arc<[u8]>, + ranges: &'a [HtCodeBlockPayloadRanges], + }, +} + +impl PreparedHtPayloadSource { + pub(in crate::compute) fn as_ht_payload_source(&self) -> HtPayloadSource<'_> { + match self { + Self::Contiguous(data) => HtPayloadSource::Contiguous(data), + Self::Referenced { input, ranges } => HtPayloadSource::Referenced { input, ranges }, + } + } +} + +#[derive(Clone, Copy)] +struct FlattenedHtJob<'a> { + source_index: usize, + payload: FlattenedHtPayload<'a>, + job: J2kHtCleanupBatchJob, + output_base: usize, +} + +#[derive(Clone, Copy)] +enum FlattenedHtPayload<'a> { + Contiguous(&'a [u8]), + Referenced { + input: &'a [u8], + ranges: HtCodeBlockPayloadRanges, + }, +} + +pub(in crate::compute) struct MetalHtChunkPlan<'a> { + jobs: Vec>, + plan: HtGpuJobChunkPlan, +} + +pub(in crate::compute) struct PackedMetalHtChunk { + pub(in crate::compute) bucket: HtGpuJobPassBucket, + pub(in crate::compute) coded_data: Vec, + pub(in crate::compute) jobs: Vec, + pub(in crate::compute) source_indices: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::compute) enum MetalHtPipelineKind { + CleanupOnly, + SigProp, + MagRef, +} + +pub(in crate::compute) const fn metal_ht_pipeline_kind_for_bucket( + bucket: HtGpuJobPassBucket, +) -> MetalHtPipelineKind { + match bucket { + HtGpuJobPassBucket::CleanupOnly => MetalHtPipelineKind::CleanupOnly, + HtGpuJobPassBucket::SigProp => MetalHtPipelineKind::SigProp, + HtGpuJobPassBucket::MagRef => MetalHtPipelineKind::MagRef, + } +} + +struct PackedMetalHtChunkOwners { + coded_data: Vec, + jobs: Vec, + source_indices: Vec, +} + +fn allocate_packed_metal_ht_chunk( + payload_bytes: usize, + job_count: usize, + mut budget: crate::batch_allocation::BatchMetadataBudget, +) -> Result { + budget.preflight(&[ + crate::batch_allocation::BatchMetadataRequest::of::(payload_bytes), + crate::batch_allocation::BatchMetadataRequest::of::(job_count), + crate::batch_allocation::BatchMetadataRequest::of::(job_count), + ])?; + Ok(PackedMetalHtChunkOwners { + coded_data: budget.try_vec(payload_bytes, "HTJ2K Metal packed chunk payload")?, + jobs: budget.try_vec(job_count, "HTJ2K Metal packed chunk jobs")?, + source_indices: budget.try_vec(job_count, "HTJ2K Metal packed chunk source indices")?, + }) +} + +pub(in crate::compute) fn default_metal_ht_chunk_limits() -> HtGpuJobChunkLimits { + HtGpuJobChunkLimits::new( + NonZeroUsize::new(MAX_METAL_HT_JOBS_PER_CHUNK).unwrap_or(NonZeroUsize::MIN), + MAX_METAL_HT_PAYLOAD_BYTES_PER_CHUNK, + MAX_METAL_HT_DESCRIPTOR_BYTES_PER_CHUNK, + ) +} + +pub(in crate::compute) fn plan_metal_ht_chunks<'a>( + batches: &[HtBatchInput<'a>], + limits: HtGpuJobChunkLimits, +) -> Result, Error> { + let job_count = crate::batch_allocation::checked_count_sum( + batches.iter().map(|batch| batch.jobs.len()), + "HTJ2K Metal chunk planning jobs", + )?; + let mut budget = + crate::batch_allocation::BatchMetadataBudget::new("HTJ2K Metal chunk planning metadata"); + let mut jobs = budget.try_vec(job_count, "HTJ2K Metal flattened chunk jobs")?; + let mut requests = budget.try_vec(job_count, "HTJ2K Metal chunk planner requests")?; + + for batch in batches { + if let HtPayloadSource::Referenced { ranges, .. } = batch.payload { + if ranges.len() != batch.jobs.len() { + return Err(Error::MetalStateInvariant { + state: "HTJ2K Metal referenced batch input", + reason: "payload range count does not match job count", + }); + } + } + for (job_index, job) in batch.jobs.iter().enumerate() { + let coding_passes = + u8::try_from(job.number_of_coding_passes).map_err(|_| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {} job coding-pass count {} exceeds u8", + batch.source_index, job.number_of_coding_passes + ), + })?; + let payload_bytes = usize::try_from(job.coded_len).map_err(|_| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {} job payload length exceeds usize", + batch.source_index + ), + })?; + requests.push(HtGpuJobChunkRequest::new( + batch.source_index, + coding_passes, + payload_bytes, + size_of::(), + )); + let payload = match batch.payload { + HtPayloadSource::Contiguous(coded_data) => { + FlattenedHtPayload::Contiguous(coded_data) + } + HtPayloadSource::Referenced { input, ranges } => { + let ranges = *ranges.get(job_index).ok_or(Error::MetalStateInvariant { + state: "HTJ2K Metal referenced batch input", + reason: "payload range index is outside the retained table", + })?; + validate_referenced_ht_payload(batch.source_index, input, ranges, job)?; + FlattenedHtPayload::Referenced { + input: input.as_ref(), + ranges, + } + } + }; + jobs.push(FlattenedHtJob { + source_index: batch.source_index, + payload, + job: *job, + output_base: batch.output_base, + }); + } + } + + let plan = plan_ht_gpu_job_chunks(&requests, limits).map_err(chunk_plan_error)?; + Ok(MetalHtChunkPlan { jobs, plan }) +} + +impl MetalHtChunkPlan<'_> { + pub(in crate::compute) fn chunk_count(&self) -> usize { + self.plan.chunks().len() + } + + pub(in crate::compute) fn job_count(&self) -> usize { + self.jobs.len() + } + + pub(in crate::compute) fn pack_chunk( + &self, + chunk_index: usize, + ) -> Result { + let chunk = + self.plan + .chunks() + .get(chunk_index) + .copied() + .ok_or(Error::MetalStateInvariant { + state: "HTJ2K Metal chunk plan", + reason: "requested chunk index is outside the planned range", + })?; + let entries = self + .plan + .chunk_entries(chunk_index) + .ok_or(Error::MetalStateInvariant { + state: "HTJ2K Metal chunk plan", + reason: "chunk entry range is outside the flattened job table", + })?; + let PackedMetalHtChunkOwners { + mut coded_data, + mut jobs, + mut source_indices, + } = allocate_packed_metal_ht_chunk( + chunk.payload_bytes(), + chunk.job_count(), + crate::batch_allocation::BatchMetadataBudget::new("HTJ2K Metal packed chunk metadata"), + )?; + + for entry in entries { + let flattened = + self.jobs + .get(entry.original_job_index()) + .ok_or(Error::MetalStateInvariant { + state: "HTJ2K Metal chunk plan", + reason: "planned job index is outside the flattened job table", + })?; + if flattened.source_index != entry.source_index() { + return Err(Error::MetalStateInvariant { + state: "HTJ2K Metal chunk plan", + reason: "planned source identity does not match the flattened job", + }); + } + append_packed_ht_job(flattened, &mut coded_data, &mut jobs, &mut source_indices)?; + } + + if coded_data.len() != chunk.payload_bytes() + || jobs.len() != chunk.job_count() + || source_indices.len() != chunk.job_count() + { + return Err(Error::MetalStateInvariant { + state: "HTJ2K Metal packed chunk", + reason: "packed arena lengths do not match the shared chunk plan", + }); + } + Ok(PackedMetalHtChunk { + bucket: chunk.bucket(), + coded_data, + jobs, + source_indices, + }) + } +} + +fn append_packed_ht_job( + flattened: &FlattenedHtJob<'_>, + coded_data: &mut Vec, + jobs: &mut Vec, + source_indices: &mut Vec, +) -> Result<(), Error> { + let mut adjusted = flattened.job; + adjusted.coded_offset = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal packed chunk payload exceeds u32".to_string(), + })?; + let output_base = u32::try_from(flattened.output_base).map_err(|_| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {} output base exceeds u32", + flattened.source_index + ), + })?; + adjusted.output_offset = adjusted + .output_offset + .checked_add(output_base) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {} output offset overflow", + flattened.source_index + ), + })?; + match flattened.payload { + FlattenedHtPayload::Contiguous(input) => { + let coded_start = + usize::try_from(flattened.job.coded_offset).map_err(|_| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {} coded offset exceeds usize", + flattened.source_index + ), + })?; + let coded_len = + usize::try_from(flattened.job.coded_len).map_err(|_| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {} coded length exceeds usize", + flattened.source_index + ), + })?; + let coded_end = + coded_start + .checked_add(coded_len) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {} coded payload range overflow", + flattened.source_index + ), + })?; + let payload = input + .get(coded_start..coded_end) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {} coded payload range exceeds its input arena", + flattened.source_index + ), + })?; + coded_data.extend_from_slice(payload); + } + FlattenedHtPayload::Referenced { input, ranges } => { + coded_data.extend_from_slice(referenced_payload_slice( + flattened.source_index, + input, + ranges.cleanup, + )?); + if let Some(refinement) = ranges.refinement { + coded_data.extend_from_slice(referenced_payload_slice( + flattened.source_index, + input, + refinement, + )?); + } + } + } + jobs.push(adjusted); + source_indices.push(flattened.source_index); + Ok(()) +} + +fn validate_referenced_ht_payload( + source_index: usize, + input: &[u8], + ranges: HtCodeBlockPayloadRanges, + job: &J2kHtCleanupBatchJob, +) -> Result<(), Error> { + referenced_payload_slice(source_index, input, ranges.cleanup)?; + if let Some(refinement) = ranges.refinement { + referenced_payload_slice(source_index, input, refinement)?; + } + let refinement_len = ranges.refinement.map_or(0, |range| range.length); + let coded_len = ranges + .cleanup + .length + .checked_add(refinement_len) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {source_index} referenced payload length overflow" + ), + })?; + if usize::try_from(job.cleanup_length).ok() != Some(ranges.cleanup.length) + || usize::try_from(job.refinement_length).ok() != Some(refinement_len) + || usize::try_from(job.coded_len).ok() != Some(coded_len) + { + return Err(Error::MetalKernel { + message: format!( + "HTJ2K Metal source {source_index} referenced payload lengths do not match its job" + ), + }); + } + Ok(()) +} + +fn referenced_payload_slice( + source_index: usize, + input: &[u8], + range: j2k_native::J2kCodestreamRange, +) -> Result<&[u8], Error> { + let end = range.end().ok_or_else(|| Error::MetalKernel { + message: format!("HTJ2K Metal source {source_index} referenced payload range overflow"), + })?; + input + .get(range.offset..end) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "HTJ2K Metal source {source_index} referenced payload range exceeds retained input" + ), + }) +} + +fn chunk_plan_error(source: HtGpuJobChunkPlanError) -> Error { + match source { + HtGpuJobChunkPlanError::BatchInfrastructure(source) => Error::BatchInfrastructure(source), + _ => Error::MetalKernel { + message: format!("HTJ2K Metal chunk planning failed: {source}"), + }, + } +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/execution.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/execution.rs new file mode 100644 index 00000000..e0783f8c --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/execution.rs @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_core::{HtGpuJobChunkLimits, HtGpuJobPassBucket}; + +use crate::compute::resident_codestream::{ + dispatch_ht_cleanup_batched_in_encoder_with_status_offset, + dispatch_ht_cleanup_repeated_batched_in_encoder_with_status_offset, HtCleanupBatchDispatch, + HtCleanupRepeatedBatchDispatch, +}; + +use super::super::{ + dispatch_zero_u32_buffer_in_encoder, new_compute_command_encoder, new_shared_buffer, + size_of as abi_size_of, Buffer, CommandBufferRef, ComputeCommandEncoderRef, DirectStatusCheck, + Error, J2kHtRepeatedBatchParams, MetalRuntime, +}; +use super::{ + metal_ht_pipeline_kind_for_bucket, prepared::PreparedMetalHtExecution, + prepared_metal_ht_execution, HtBatchInput, PackedMetalHtChunk, +}; + +pub(in crate::compute) fn encode_metal_ht_batches_in_encoder( + runtime: &MetalRuntime, + encoder: &ComputeCommandEncoderRef, + batches: &[HtBatchInput<'_>], + decoded: &Buffer, + output_word_count: usize, + limits: HtGpuJobChunkLimits, +) -> Result<(Vec, DirectStatusCheck), Error> { + let output_bytes = output_word_count + .checked_mul(abi_size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal output arena byte length overflow".to_string(), + })?; + if u64::try_from(output_bytes).map_or(true, |bytes| bytes > decoded.length()) { + return Err(Error::MetalStateInvariant { + state: "HTJ2K Metal output arena", + reason: "logical coefficient arena exceeds the destination buffer", + }); + } + let prepared = prepared_metal_ht_execution(runtime, batches, limits)?; + dispatch_zero_u32_buffer_in_encoder(runtime, encoder, decoded, output_word_count)?; + encoder.memory_barrier_with_resources(&[decoded]); + if prepared.job_count() == 0 { + let empty = new_shared_buffer(&runtime.device, 1)?; + return Ok(( + vec![empty.clone()], + DirectStatusCheck::Ht { + buffer: empty, + len: 0, + source_indices: Some(Vec::new()), + recyclable_status: None, + }, + )); + } + + let status_bytes = prepared + .job_count() + .checked_mul(abi_size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal chunk status arena size overflow".to_string(), + })?; + let status_owner = runtime.take_shared_buffer(status_bytes)?; + let status_buffer = status_owner.buffer().clone(); + let mut budget = + crate::batch_allocation::BatchMetadataBudget::new("HTJ2K Metal chunk submission owners"); + let mut retained_buffers = budget.try_vec( + prepared.chunks().len().saturating_mul(2), + "HTJ2K Metal chunk retained buffers", + )?; + let mut source_indices = budget.try_vec( + prepared.job_count(), + "HTJ2K Metal chunk ordered source indices", + )?; + let mut status_offset_bytes = 0usize; + + for chunk in prepared.chunks() { + let status_offset = u64::try_from(status_offset_bytes).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal chunk status offset exceeds u64".to_string(), + })?; + dispatch_ht_cleanup_batched_in_encoder_with_status_offset( + runtime, + encoder, + metal_ht_pipeline_kind_for_bucket(chunk.bucket), + HtCleanupBatchDispatch { + coded_data: &chunk.coded_buffer, + jobs: &chunk.jobs_buffer, + job_count: chunk.job_count(), + decoded, + status_buffer: &status_buffer, + status_offset_bytes: status_offset, + }, + ); + let chunk_status_bytes = chunk + .job_count() + .checked_mul(abi_size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal chunk status offset overflow".to_string(), + })?; + status_offset_bytes = status_offset_bytes + .checked_add(chunk_status_bytes) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal aggregate status offset overflow".to_string(), + })?; + retained_buffers.push(chunk.coded_buffer.clone()); + retained_buffers.push(chunk.jobs_buffer.clone()); + source_indices.extend_from_slice(&chunk.source_indices); + } + + if status_offset_bytes != status_bytes || source_indices.len() != prepared.job_count() { + return Err(Error::MetalStateInvariant { + state: "HTJ2K Metal chunk submission", + reason: "dispatched chunk status ownership does not match the shared plan", + }); + } + Ok(( + retained_buffers, + DirectStatusCheck::Ht { + buffer: status_buffer, + len: prepared.job_count(), + source_indices: Some(source_indices), + recyclable_status: Some(status_owner), + }, + )) +} + +pub(in crate::compute) fn encode_repeated_metal_ht_batch_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + batch: HtBatchInput<'_>, + count: usize, + output_plane_len: usize, + decoded: &Buffer, + limits: HtGpuJobChunkLimits, +) -> Result<(Vec, DirectStatusCheck), Error> { + let prepared = prepared_metal_ht_execution(runtime, &[batch], limits)?; + let total_job_count = + prepared + .job_count() + .checked_mul(count) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal repeated chunk job count overflow".to_string(), + })?; + let decoded_word_count = + output_plane_len + .checked_mul(count) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal repeated chunk output span overflow".to_string(), + })?; + let encoder = new_compute_command_encoder(command_buffer)?; + dispatch_zero_u32_buffer_in_encoder(runtime, &encoder, decoded, decoded_word_count)?; + encoder.memory_barrier_with_resources(&[decoded]); + if total_job_count == 0 { + encoder.end_encoding(); + let empty = new_shared_buffer(&runtime.device, 1)?; + return Ok(( + vec![empty.clone()], + DirectStatusCheck::Ht { + buffer: empty, + len: 0, + source_indices: Some(Vec::new()), + recyclable_status: None, + }, + )); + } + + let status_bytes = total_job_count + .checked_mul(abi_size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal repeated chunk status arena size overflow".to_string(), + })?; + let status_owner = runtime.take_shared_buffer(status_bytes)?; + let status_buffer = status_owner.buffer().clone(); + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "HTJ2K Metal repeated chunk submission owners", + ); + let mut retained_buffers = budget.try_vec( + prepared.chunks().len().saturating_mul(2), + "HTJ2K Metal repeated chunk retained buffers", + )?; + let mut source_indices = budget.try_vec( + total_job_count, + "HTJ2K Metal repeated chunk ordered source indices", + )?; + let output_plane_len = u32::try_from(output_plane_len).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal repeated output plane length exceeds u32".to_string(), + })?; + let batch_count = u32::try_from(count).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal repeated batch count exceeds u32".to_string(), + })?; + let status_offset_bytes = RepeatedHtChunkEncoder { + runtime, + encoder: &encoder, + prepared: &prepared, + count, + output_plane_len, + batch_count, + decoded, + status_buffer: &status_buffer, + } + .encode(&mut retained_buffers, &mut source_indices)?; + encoder.end_encoding(); + + if status_offset_bytes != status_bytes || source_indices.len() != total_job_count { + return Err(Error::MetalStateInvariant { + state: "HTJ2K Metal repeated chunk submission", + reason: "dispatched chunk status ownership does not match the shared plan", + }); + } + Ok(( + retained_buffers, + DirectStatusCheck::Ht { + buffer: status_buffer, + len: total_job_count, + source_indices: Some(source_indices), + recyclable_status: Some(status_owner), + }, + )) +} + +struct RepeatedHtChunkEncoder<'a, 'plan> { + runtime: &'a MetalRuntime, + encoder: &'a ComputeCommandEncoderRef, + prepared: &'plan PreparedMetalHtExecution, + count: usize, + output_plane_len: u32, + batch_count: u32, + decoded: &'a Buffer, + status_buffer: &'a Buffer, +} + +impl RepeatedHtChunkEncoder<'_, '_> { + fn encode( + &self, + retained_buffers: &mut Vec, + source_indices: &mut Vec, + ) -> Result { + let mut status_offset_bytes = 0usize; + for chunk in self.prepared.chunks() { + let status_offset = + u64::try_from(status_offset_bytes).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal repeated chunk status offset exceeds u64".to_string(), + })?; + let job_count = u32::try_from(chunk.job_count()).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal repeated chunk job count exceeds u32".to_string(), + })?; + dispatch_ht_cleanup_repeated_batched_in_encoder_with_status_offset( + self.runtime, + self.encoder, + metal_ht_pipeline_kind_for_bucket(chunk.bucket), + HtCleanupRepeatedBatchDispatch { + coded_data: &chunk.coded_buffer, + jobs: &chunk.jobs_buffer, + base_job_count: chunk.job_count(), + repeated: J2kHtRepeatedBatchParams { + job_count, + output_plane_len: self.output_plane_len, + batch_count: self.batch_count, + }, + decoded: self.decoded, + status_buffer: self.status_buffer, + status_offset_bytes: status_offset, + }, + ); + let chunk_status_count = + chunk + .job_count() + .checked_mul(self.count) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal repeated chunk status count overflow".to_string(), + })?; + let chunk_status_bytes = chunk_status_count + .checked_mul(abi_size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal repeated chunk status offset overflow".to_string(), + })?; + status_offset_bytes = status_offset_bytes + .checked_add(chunk_status_bytes) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal repeated aggregate status offset overflow".to_string(), + })?; + for source_index in 0..self.count { + source_indices.extend(core::iter::repeat_n(source_index, chunk.job_count())); + } + retained_buffers.push(chunk.coded_buffer.clone()); + retained_buffers.push(chunk.jobs_buffer.clone()); + } + Ok(status_offset_bytes) + } +} + +pub(super) fn validate_pass_homogeneous_chunk(chunk: &PackedMetalHtChunk) -> Result<(), Error> { + if chunk.jobs.iter().any(|job| job.number_of_coding_passes > 3) { + return Err(Error::UnsupportedMetalRequest { + reason: "HTJ2K Metal decoding supports at most three coding passes per code block", + }); + } + let matches_bucket = chunk.jobs.iter().all(|job| match chunk.bucket { + HtGpuJobPassBucket::CleanupOnly => job.number_of_coding_passes == 1, + HtGpuJobPassBucket::SigProp => job.number_of_coding_passes == 2, + HtGpuJobPassBucket::MagRef => job.number_of_coding_passes == 3, + }); + if !matches_bucket { + return Err(Error::MetalStateInvariant { + state: "HTJ2K Metal chunk submission", + reason: "shared planner returned a pass-heterogeneous chunk", + }); + } + Ok(()) +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/prepared.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/prepared.rs new file mode 100644 index 00000000..4019915e --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/prepared.rs @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Device-resident immutable arenas for an exact ordered prepared HT group. + +use core::mem::size_of; +use std::sync::Arc; + +use j2k_core::HtGpuJobChunkLimits; +use metal::{Buffer, Device}; + +use super::{ + execution::validate_pass_homogeneous_chunk, HtBatchInput, HtPayloadSource, + J2kHtCleanupBatchJob, PackedMetalHtChunk, +}; +use crate::compute::{copied_slice_buffer, Error, MetalRuntime, PreparedHtExecutionOwner}; +use crate::session::{PREPARED_PLAN_CACHE_MAX_DEVICE_BYTES, PREPARED_PLAN_CACHE_MAX_HOST_BYTES}; + +const PREPARED_METAL_HT_EXECUTION_CACHE_CAP: usize = 128; + +struct PreparedMetalHtInputKey { + owner: Arc, + payload: PreparedMetalHtPayloadKey, + jobs_ptr: usize, + jobs_len: usize, + source_index: usize, + output_base: usize, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum PreparedMetalHtPayloadKey { + Contiguous { + data_ptr: usize, + data_len: usize, + }, + Referenced { + input_ptr: usize, + input_len: usize, + ranges_ptr: usize, + ranges_len: usize, + }, +} + +impl PreparedMetalHtPayloadKey { + fn from_source(source: HtPayloadSource<'_>) -> Self { + match source { + HtPayloadSource::Contiguous(data) => Self::Contiguous { + data_ptr: data.as_ptr() as usize, + data_len: data.len(), + }, + HtPayloadSource::Referenced { input, ranges } => Self::Referenced { + input_ptr: input.as_ptr() as usize, + input_len: input.len(), + ranges_ptr: ranges.as_ptr() as usize, + ranges_len: ranges.len(), + }, + } + } +} + +struct PreparedMetalHtExecutionCacheEntry { + inputs: Vec, + limits: HtGpuJobChunkLimits, + execution: Arc, + host_bytes: usize, + device_bytes: usize, + last_used: u64, +} + +pub(in crate::compute) struct PreparedMetalHtExecutionCache { + entries: Vec, + retained_host_bytes: usize, + retained_device_bytes: usize, + access_clock: u64, +} + +pub(in crate::compute) struct PreparedMetalHtExecution { + chunks: Vec, + job_count: usize, +} + +pub(in crate::compute) struct PreparedMetalHtChunk { + pub(in crate::compute) bucket: j2k_core::HtGpuJobPassBucket, + coded_data: Vec, + jobs: Vec, + pub(in crate::compute) source_indices: Vec, + pub(in crate::compute) coded_buffer: Buffer, + pub(in crate::compute) jobs_buffer: Buffer, +} + +impl PreparedMetalHtExecutionCache { + pub(in crate::compute) const fn new() -> Self { + Self { + entries: Vec::new(), + retained_host_bytes: 0, + retained_device_bytes: 0, + access_clock: 0, + } + } + + fn get_or_prepare( + &mut self, + device: &Device, + batches: &[HtBatchInput<'_>], + limits: HtGpuJobChunkLimits, + ) -> Result, Error> { + if let Some(index) = self.find(batches, limits) { + self.access_clock = self.access_clock.saturating_add(1); + self.entries[index].last_used = self.access_clock; + return Ok(self.entries[index].execution.clone()); + } + + let execution = Arc::new(PreparedMetalHtExecution::prepare(device, batches, limits)?); + let (execution_host_bytes, device_bytes) = execution.retained_bytes()?; + let mut inputs = Vec::new(); + inputs.try_reserve_exact(batches.len()).map_err(|source| { + Error::PreparedPlanCacheAllocation { + context: "J2K Metal prepared HT execution cache key", + source, + } + })?; + inputs.extend(batches.iter().map(|batch| PreparedMetalHtInputKey { + owner: batch.execution_owner.clone(), + payload: PreparedMetalHtPayloadKey::from_source(batch.payload), + jobs_ptr: batch.jobs.as_ptr() as usize, + jobs_len: batch.jobs.len(), + source_index: batch.source_index, + output_base: batch.output_base, + })); + let key_host_bytes = + prepared_metal_ht_input_key_host_bytes(inputs.capacity(), inputs.len())?; + let host_bytes = execution_host_bytes + .checked_add(key_host_bytes) + .and_then(|bytes| { + bytes + .checked_add(size_of::()) + .and_then(|bytes| bytes.checked_add(2 * size_of::())) + }) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "prepared execution host byte count overflow", + })?; + self.ensure_metadata_capacity()?; + let metadata_host_bytes = self + .entries + .capacity() + .checked_mul(size_of::()) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "entry metadata byte count overflow", + })?; + if !prepared_metal_ht_execution_fits_empty_cache( + metadata_host_bytes, + host_bytes, + device_bytes, + ) { + return Ok(execution); + } + + self.evict_until_fits(host_bytes, device_bytes)?; + self.access_clock = self.access_clock.saturating_add(1); + self.entries + .try_reserve(1) + .map_err(|source| Error::PreparedPlanCacheAllocation { + context: "J2K Metal prepared HT execution cache entry", + source, + })?; + self.entries.push(PreparedMetalHtExecutionCacheEntry { + inputs, + limits, + execution: execution.clone(), + host_bytes, + device_bytes, + last_used: self.access_clock, + }); + self.retained_host_bytes = + self.retained_host_bytes + .checked_add(host_bytes) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "retained host byte count overflow", + })?; + self.retained_device_bytes = self.retained_device_bytes.checked_add(device_bytes).ok_or( + Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "retained device byte count overflow", + }, + )?; + Ok(execution) + } + + fn find(&self, batches: &[HtBatchInput<'_>], limits: HtGpuJobChunkLimits) -> Option { + self.entries.iter().position(|entry| { + same_limits(entry.limits, limits) + && entry.inputs.len() == batches.len() + && entry.inputs.iter().zip(batches).all(|(key, batch)| { + Arc::ptr_eq(&key.owner, batch.execution_owner) + && key.payload == PreparedMetalHtPayloadKey::from_source(batch.payload) + && key.jobs_ptr == batch.jobs.as_ptr() as usize + && key.jobs_len == batch.jobs.len() + && key.source_index == batch.source_index + && key.output_base == batch.output_base + }) + }) + } + + fn ensure_metadata_capacity(&mut self) -> Result<(), Error> { + if self.entries.capacity() >= PREPARED_METAL_HT_EXECUTION_CACHE_CAP { + return Ok(()); + } + if !self.entries.is_empty() { + return Err(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "entry metadata capacity changed after cache insertion", + }); + } + let mut entries = Vec::new(); + entries + .try_reserve_exact(PREPARED_METAL_HT_EXECUTION_CACHE_CAP) + .map_err(|source| Error::PreparedPlanCacheAllocation { + context: "J2K Metal prepared HT execution cache metadata", + source, + })?; + let metadata_bytes = entries + .capacity() + .checked_mul(size_of::()) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "entry metadata byte count overflow", + })?; + if metadata_bytes > PREPARED_PLAN_CACHE_MAX_HOST_BYTES { + return Err(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "entry metadata exceeds the shared host cache limit", + }); + } + self.entries = entries; + self.retained_host_bytes = metadata_bytes; + Ok(()) + } + + fn evict_until_fits( + &mut self, + incoming_host_bytes: usize, + incoming_device_bytes: usize, + ) -> Result<(), Error> { + while self.entries.len() >= PREPARED_METAL_HT_EXECUTION_CACHE_CAP + || self + .retained_host_bytes + .checked_add(incoming_host_bytes) + .is_none_or(|bytes| bytes > PREPARED_PLAN_CACHE_MAX_HOST_BYTES) + || self + .retained_device_bytes + .checked_add(incoming_device_bytes) + .is_none_or(|bytes| bytes > PREPARED_PLAN_CACHE_MAX_DEVICE_BYTES) + { + let index = self + .entries + .iter() + .enumerate() + .min_by_key(|(index, entry)| (entry.last_used, *index)) + .map(|(index, _)| index) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "cache limits require eviction but no entry is retained", + })?; + let entry = self.entries.remove(index); + self.retained_host_bytes = self + .retained_host_bytes + .checked_sub(entry.host_bytes) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "retained host byte count underflow", + })?; + self.retained_device_bytes = self + .retained_device_bytes + .checked_sub(entry.device_bytes) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "retained device byte count underflow", + })?; + } + Ok(()) + } +} + +fn prepared_metal_ht_input_key_host_bytes(capacity: usize, len: usize) -> Result { + capacity + .checked_mul(size_of::()) + .and_then(|bytes| { + len.checked_mul(2 * size_of::()) + .and_then(|owners| bytes.checked_add(owners)) + }) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "prepared execution key byte count overflow", + }) +} + +fn prepared_metal_ht_execution_fits_empty_cache( + metadata_host_bytes: usize, + entry_host_bytes: usize, + entry_device_bytes: usize, +) -> bool { + metadata_host_bytes + .checked_add(entry_host_bytes) + .is_some_and(|bytes| bytes <= PREPARED_PLAN_CACHE_MAX_HOST_BYTES) + && entry_device_bytes <= PREPARED_PLAN_CACHE_MAX_DEVICE_BYTES +} + +impl PreparedMetalHtExecution { + fn prepare( + device: &Device, + batches: &[HtBatchInput<'_>], + limits: HtGpuJobChunkLimits, + ) -> Result { + let plan = super::plan_metal_ht_chunks(batches, limits)?; + let mut chunks = Vec::new(); + chunks + .try_reserve_exact(plan.chunk_count()) + .map_err(|source| Error::PreparedPlanCacheAllocation { + context: "J2K Metal prepared HT execution chunks", + source, + })?; + for chunk_index in 0..plan.chunk_count() { + let packed = plan.pack_chunk(chunk_index)?; + validate_pass_homogeneous_chunk(&packed)?; + #[cfg(test)] + crate::compute::test_counters::record_ht_immutable_payload_upload(); + let coded_buffer = copied_slice_buffer(device, &packed.coded_data)?; + #[cfg(test)] + crate::compute::test_counters::record_ht_immutable_job_upload(); + let jobs_buffer = copied_slice_buffer(device, &packed.jobs)?; + chunks.push(PreparedMetalHtChunk::new(packed, coded_buffer, jobs_buffer)); + } + Ok(Self { + chunks, + job_count: plan.job_count(), + }) + } + + pub(in crate::compute) fn chunks(&self) -> &[PreparedMetalHtChunk] { + &self.chunks + } + + pub(in crate::compute) const fn job_count(&self) -> usize { + self.job_count + } + + fn retained_bytes(&self) -> Result<(usize, usize), Error> { + let mut host_bytes = self + .chunks + .capacity() + .checked_mul(size_of::()) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "chunk owner host byte count overflow", + })?; + let mut device_bytes = 0usize; + for chunk in &self.chunks { + host_bytes = + host_bytes + .checked_add(chunk.host_bytes()?) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "aggregate chunk host byte count overflow", + })?; + device_bytes = device_bytes.checked_add(chunk.device_bytes()?).ok_or( + Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "aggregate chunk device byte count overflow", + }, + )?; + } + Ok((host_bytes, device_bytes)) + } +} + +impl PreparedMetalHtChunk { + fn new(packed: PackedMetalHtChunk, coded_buffer: Buffer, jobs_buffer: Buffer) -> Self { + Self { + bucket: packed.bucket, + coded_data: packed.coded_data, + jobs: packed.jobs, + source_indices: packed.source_indices, + coded_buffer, + jobs_buffer, + } + } + + pub(in crate::compute) fn job_count(&self) -> usize { + self.jobs.len() + } + + fn host_bytes(&self) -> Result { + self.coded_data + .capacity() + .checked_add( + self.jobs + .capacity() + .checked_mul(size_of::()) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "job arena host byte count overflow", + })?, + ) + .and_then(|bytes| { + self.source_indices + .capacity() + .checked_mul(size_of::()) + .and_then(|indices| bytes.checked_add(indices)) + }) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "chunk host byte count overflow", + }) + } + + fn device_bytes(&self) -> Result { + usize::try_from(self.coded_buffer.length()) + .ok() + .and_then(|coded| { + usize::try_from(self.jobs_buffer.length()) + .ok() + .and_then(|jobs| coded.checked_add(jobs)) + }) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal prepared HT execution cache", + reason: "chunk device byte count overflow", + }) + } +} + +pub(in crate::compute) fn prepared_metal_ht_execution( + runtime: &MetalRuntime, + batches: &[HtBatchInput<'_>], + limits: HtGpuJobChunkLimits, +) -> Result, Error> { + runtime + .prepared_ht_execution_cache + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "J2K Metal prepared HT execution cache", + })? + .get_or_prepare(&runtime.device, batches, limits) +} + +fn same_limits(left: HtGpuJobChunkLimits, right: HtGpuJobChunkLimits) -> bool { + left.max_jobs() == right.max_jobs() + && left.max_payload_bytes() == right.max_payload_bytes() + && left.max_descriptor_bytes() == right.max_descriptor_bytes() +} + +#[cfg(test)] +mod tests; diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/prepared/tests.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/prepared/tests.rs new file mode 100644 index 00000000..d75ec475 --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/prepared/tests.rs @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::{mem::size_of, num::NonZeroUsize}; +use std::sync::Arc; + +use j2k_core::HtGpuJobChunkLimits; + +use super::{ + prepared_metal_ht_execution, prepared_metal_ht_execution_fits_empty_cache, + prepared_metal_ht_input_key_host_bytes, Error, HtBatchInput, HtPayloadSource, + J2kHtCleanupBatchJob, MetalRuntime, PreparedHtExecutionOwner, PreparedMetalHtExecutionCache, + PreparedMetalHtInputKey, PreparedMetalHtPayloadKey, PREPARED_PLAN_CACHE_MAX_DEVICE_BYTES, + PREPARED_PLAN_CACHE_MAX_HOST_BYTES, +}; +use crate::compute::test_counters::{ + ht_immutable_job_uploads_for_test, ht_immutable_payload_uploads_for_test, + reset_ht_immutable_job_uploads_for_test, reset_ht_immutable_payload_uploads_for_test, +}; + +fn cleanup_job(output_offset: u32) -> J2kHtCleanupBatchJob { + J2kHtCleanupBatchJob { + coded_offset: 0, + width: 1, + height: 1, + coded_len: 1, + cleanup_length: 1, + refinement_length: 0, + missing_msbs: 0, + num_bitplanes: 1, + roi_shift: 0, + number_of_coding_passes: 1, + output_stride: 1, + output_offset, + dequantization_step: 1.0, + stripe_causal: 0, + } +} + +#[test] +fn host_pressure_evicts_the_oldest_prepared_execution_instead_of_bypassing_cache() { + let Some(device) = metal::Device::system_default() else { + return; + }; + let limits = HtGpuJobChunkLimits::new( + NonZeroUsize::new(1).expect("nonzero job cap"), + 1, + size_of::(), + ); + let payload = [0_u8]; + let first_jobs = [cleanup_job(0)]; + let second_jobs = [cleanup_job(1)]; + let first_owner = Arc::new(PreparedHtExecutionOwner); + let second_owner = Arc::new(PreparedHtExecutionOwner); + let first = [HtBatchInput { + source_index: 0, + payload: HtPayloadSource::Contiguous(&payload), + jobs: &first_jobs, + output_base: 0, + execution_owner: &first_owner, + }]; + let second = [HtBatchInput { + source_index: 1, + payload: HtPayloadSource::Contiguous(&payload), + jobs: &second_jobs, + output_base: 1, + execution_owner: &second_owner, + }]; + + let mut cache = PreparedMetalHtExecutionCache::new(); + cache + .get_or_prepare(&device, &first, limits) + .expect("seed prepared execution cache"); + + let metadata_bytes = cache + .retained_host_bytes + .checked_sub(cache.entries[0].host_bytes) + .expect("cache metadata accounting"); + let pressured_entry_bytes = PREPARED_PLAN_CACHE_MAX_HOST_BYTES + .checked_sub(metadata_bytes) + .expect("cache metadata fits host limit"); + cache.entries[0].host_bytes = pressured_entry_bytes; + cache.retained_host_bytes = PREPARED_PLAN_CACHE_MAX_HOST_BYTES; + + cache + .get_or_prepare(&device, &second, limits) + .expect("rotate prepared execution cache under host pressure"); + reset_ht_immutable_payload_uploads_for_test(); + reset_ht_immutable_job_uploads_for_test(); + cache + .get_or_prepare(&device, &second, limits) + .expect("reuse rotated prepared execution"); + + assert_eq!(ht_immutable_payload_uploads_for_test(), 0); + assert_eq!(ht_immutable_job_uploads_for_test(), 0); +} + +#[test] +fn prepared_execution_key_weight_uses_allocated_capacity() { + let owner = Arc::new(PreparedHtExecutionOwner); + let mut inputs = Vec::with_capacity(8); + inputs.push(PreparedMetalHtInputKey { + owner, + payload: PreparedMetalHtPayloadKey::Contiguous { + data_ptr: 1, + data_len: 2, + }, + jobs_ptr: 3, + jobs_len: 4, + source_index: 5, + output_base: 6, + }); + + assert_eq!( + prepared_metal_ht_input_key_host_bytes(inputs.capacity(), inputs.len()) + .expect("key weight"), + inputs.capacity() * size_of::() + 2 * size_of::() + ); +} + +#[test] +fn otherwise_identical_prepared_inputs_are_isolated_by_owner_identity() { + let Some(device) = metal::Device::system_default() else { + return; + }; + let limits = HtGpuJobChunkLimits::new( + NonZeroUsize::new(1).expect("nonzero job cap"), + 1, + size_of::(), + ); + let payload = [0_u8]; + let jobs = [cleanup_job(0)]; + let first_owner = Arc::new(PreparedHtExecutionOwner); + let second_owner = Arc::new(PreparedHtExecutionOwner); + let first = [HtBatchInput { + source_index: 0, + payload: HtPayloadSource::Contiguous(&payload), + jobs: &jobs, + output_base: 0, + execution_owner: &first_owner, + }]; + let second = [HtBatchInput { + execution_owner: &second_owner, + ..first[0] + }]; + + let mut cache = PreparedMetalHtExecutionCache::new(); + cache + .get_or_prepare(&device, &first, limits) + .expect("cache first owner"); + reset_ht_immutable_payload_uploads_for_test(); + reset_ht_immutable_job_uploads_for_test(); + cache + .get_or_prepare(&device, &second, limits) + .expect("prepare identical slices under second owner"); + assert_eq!(ht_immutable_payload_uploads_for_test(), 1); + assert_eq!(ht_immutable_job_uploads_for_test(), 1); + + reset_ht_immutable_payload_uploads_for_test(); + reset_ht_immutable_job_uploads_for_test(); + cache + .get_or_prepare(&device, &first, limits) + .expect("reuse first owner after isolated miss"); + assert_eq!(ht_immutable_payload_uploads_for_test(), 0); + assert_eq!(ht_immutable_job_uploads_for_test(), 0); +} + +#[test] +fn poisoned_prepared_execution_cache_reports_poisoned_state() { + let runtime = MetalRuntime::new().expect("isolated Metal runtime"); + let poison = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = runtime + .prepared_ht_execution_cache + .lock() + .expect("cache starts healthy"); + panic!("poison prepared HT cache for test"); + })); + assert!(poison.is_err()); + + let limits = HtGpuJobChunkLimits::new( + NonZeroUsize::new(1).expect("nonzero job cap"), + 1, + size_of::(), + ); + let Err(error) = prepared_metal_ht_execution(&runtime, &[], limits) else { + panic!("poisoned cache must fail") + }; + assert!(matches!( + error, + Error::MetalStatePoisoned { + state: "J2K Metal prepared HT execution cache" + } + )); +} + +#[test] +fn oversized_prepared_execution_is_not_cacheable() { + assert!(prepared_metal_ht_execution_fits_empty_cache( + 1, + PREPARED_PLAN_CACHE_MAX_HOST_BYTES - 1, + PREPARED_PLAN_CACHE_MAX_DEVICE_BYTES, + )); + assert!(!prepared_metal_ht_execution_fits_empty_cache( + 1, + PREPARED_PLAN_CACHE_MAX_HOST_BYTES, + 0, + )); + assert!(!prepared_metal_ht_execution_fits_empty_cache( + 0, + 0, + PREPARED_PLAN_CACHE_MAX_DEVICE_BYTES + 1, + )); +} + +#[test] +fn thousand_prepared_cache_hits_keep_retained_memory_stable() { + let Some(device) = metal::Device::system_default() else { + return; + }; + let limits = HtGpuJobChunkLimits::new( + NonZeroUsize::new(1).expect("nonzero job cap"), + 1, + size_of::(), + ); + let payload = [0_u8]; + let jobs = [cleanup_job(0)]; + let owner = Arc::new(PreparedHtExecutionOwner); + let input = [HtBatchInput { + source_index: 0, + payload: HtPayloadSource::Contiguous(&payload), + jobs: &jobs, + output_base: 0, + execution_owner: &owner, + }]; + let mut cache = PreparedMetalHtExecutionCache::new(); + cache + .get_or_prepare(&device, &input, limits) + .expect("warm prepared execution cache"); + let retained = ( + cache.entries.len(), + cache.retained_host_bytes, + cache.retained_device_bytes, + ); + reset_ht_immutable_payload_uploads_for_test(); + reset_ht_immutable_job_uploads_for_test(); + + for _ in 0..1_000 { + cache + .get_or_prepare(&device, &input, limits) + .expect("reuse prepared execution cache"); + } + + assert_eq!( + ( + cache.entries.len(), + cache.retained_host_bytes, + cache.retained_device_bytes, + ), + retained + ); + assert_eq!(ht_immutable_payload_uploads_for_test(), 0); + assert_eq!(ht_immutable_job_uploads_for_test(), 0); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests.rs new file mode 100644 index 00000000..9756be36 --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +mod cache; +mod parity; +mod planning; +mod status; diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/cache.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/cache.rs new file mode 100644 index 00000000..74fe370e --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/cache.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::mem::size_of; + +use j2k_native::{DecodeSettings, DecoderContext, EncodeOptions, Image}; + +use super::super::{ + default_metal_ht_chunk_limits, encode_metal_ht_batches_in_encoder, HtBatchInput, +}; +use crate::compute::test_counters::{ + ht_immutable_job_uploads_for_test, ht_immutable_payload_uploads_for_test, + reset_ht_immutable_job_uploads_for_test, reset_ht_immutable_payload_uploads_for_test, +}; +use crate::compute::{ + checked_buffer_slice, commit_and_wait_metal, decode_prepared_ht_sub_band_group_on_cpu_profile, + new_command_buffer, new_compute_command_encoder, new_shared_buffer, + prepare_direct_grayscale_plan, validate_direct_status, with_runtime, +}; + +mod referenced; + +#[test] +fn second_prepared_ht_submission_reuses_immutable_gpu_arenas() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let pixels = (0..64_u8).collect::>(); + let bytes = j2k_native::encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + .expect("encode prepared-reuse fixture"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("fixture image"); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct fixture plan"); + let prepared = prepare_direct_grayscale_plan(&direct).expect("prepared fixture plan"); + let group = prepared.ht_groups.first().expect("prepared HT group"); + let input = HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }; + let cpu = decode_prepared_ht_sub_band_group_on_cpu_profile(group, None) + .expect("prepared-reuse CPU coefficient oracle"); + + with_runtime(|runtime| { + reset_ht_immutable_payload_uploads_for_test(); + reset_ht_immutable_job_uploads_for_test(); + for submission in 0..2 { + if submission == 1 { + assert!(ht_immutable_payload_uploads_for_test() > 0); + assert!(ht_immutable_job_uploads_for_test() > 0); + reset_ht_immutable_payload_uploads_for_test(); + reset_ht_immutable_job_uploads_for_test(); + } + let output = + new_shared_buffer(&runtime.device, group.total_coefficients * size_of::())?; + let command_buffer = new_command_buffer(&runtime.queue)?; + let encoder = new_compute_command_encoder(&command_buffer)?; + let (retained, status) = encode_metal_ht_batches_in_encoder( + runtime, + &encoder, + &[input], + &output, + group.total_coefficients, + default_metal_ht_chunk_limits(), + )?; + encoder.end_encoding(); + commit_and_wait_metal(&command_buffer)?; + validate_direct_status(runtime, status)?; + let coefficients = checked_buffer_slice::( + &output, + group.total_coefficients, + "prepared HT cache reuse parity", + )?; + assert_eq!( + coefficients, cpu, + "submission {submission} must decode the cached immutable arenas exactly" + ); + drop(retained); + } + assert_eq!(ht_immutable_payload_uploads_for_test(), 0); + assert_eq!(ht_immutable_job_uploads_for_test(), 0); + Ok(()) + }) + .expect("prepared HT arena reuse"); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/cache/referenced.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/cache/referenced.rs new file mode 100644 index 00000000..9e067459 --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/cache/referenced.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::mem::size_of; +use std::sync::Arc; + +use super::super::super::{ + default_metal_ht_chunk_limits, encode_metal_ht_batches_in_encoder, HtBatchInput, +}; +use crate::compute::test_counters::{ + ht_immutable_job_uploads_for_test, ht_immutable_payload_uploads_for_test, + reset_ht_immutable_job_uploads_for_test, reset_ht_immutable_payload_uploads_for_test, +}; + +#[test] +fn second_referenced_prepared_submission_uploads_no_immutable_arenas() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let runtime = crate::compute::MetalRuntime::new().expect("isolated Metal runtime"); + let pixels = (0..64_u8).collect::>(); + let bytes = Arc::<[u8]>::from( + j2k_native::encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode referenced prepared fixture"), + ); + let image = j2k_native::Image::new(&bytes, &j2k_native::DecodeSettings::strict()) + .expect("referenced prepared image"); + let mut context = j2k_native::DecoderContext::default(); + let referenced = image + .build_referenced_htj2k_plan_region_with_context(&mut context, (0, 0, 8, 8)) + .expect("referenced prepared plan"); + let prepared = crate::compute::prepare_referenced_htj2k_grayscale_plan(&referenced, &bytes) + .expect("Metal referenced prepared plan"); + let group = prepared.ht_groups.first().expect("prepared HT group"); + let input = [HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }]; + + reset_ht_immutable_payload_uploads_for_test(); + reset_ht_immutable_job_uploads_for_test(); + for submission in 0..2 { + if submission == 1 { + assert_eq!(ht_immutable_payload_uploads_for_test(), 1); + assert_eq!(ht_immutable_job_uploads_for_test(), 1); + reset_ht_immutable_payload_uploads_for_test(); + reset_ht_immutable_job_uploads_for_test(); + } + let output = crate::compute::new_shared_buffer( + &runtime.device, + group.total_coefficients * size_of::(), + ) + .expect("referenced prepared output"); + let command_buffer = crate::compute::new_command_buffer(&runtime.queue) + .expect("referenced prepared command buffer"); + let encoder = crate::compute::new_compute_command_encoder(&command_buffer) + .expect("referenced prepared encoder"); + let (retained, status) = encode_metal_ht_batches_in_encoder( + &runtime, + &encoder, + &input, + &output, + group.total_coefficients, + default_metal_ht_chunk_limits(), + ) + .expect("submit referenced prepared execution"); + encoder.end_encoding(); + crate::compute::commit_and_wait_metal(&command_buffer) + .expect("complete referenced prepared execution"); + crate::compute::validate_direct_status(&runtime, status) + .expect("validate referenced prepared execution"); + drop(retained); + } + + assert_eq!(ht_immutable_payload_uploads_for_test(), 0); + assert_eq!(ht_immutable_job_uploads_for_test(), 0); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/parity.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/parity.rs new file mode 100644 index 00000000..7ca8af11 --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/parity.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::{mem::size_of, num::NonZeroUsize}; + +use j2k_core::HtGpuJobChunkLimits; +use j2k_native::{DecodeSettings, DecoderContext, EncodeOptions, Image}; + +use super::super::{ + encode_metal_ht_batches_in_encoder, plan_metal_ht_chunks, HtBatchInput, J2kHtCleanupBatchJob, +}; +use crate::compute::{ + checked_buffer_slice, commit_and_wait_metal, decode_prepared_ht_sub_band_group_on_cpu_profile, + new_command_buffer, new_compute_command_encoder, new_shared_buffer, + prepare_direct_grayscale_plan, validate_direct_status, with_runtime, +}; + +#[test] +fn ht_zero_fill_is_barriered_before_distinct_and_repeated_tier1_dispatch() { + let source = include_str!("../execution.rs"); + let distinct = source + .split_once("fn encode_metal_ht_batches_in_encoder") + .expect("distinct HT execution function") + .1 + .split_once("fn encode_repeated_metal_ht_batch_in_command_buffer") + .expect("repeated HT execution function follows distinct execution") + .0; + let repeated = source + .split_once("fn encode_repeated_metal_ht_batch_in_command_buffer") + .expect("repeated HT execution function") + .1 + .split_once("struct RepeatedHtChunkEncoder") + .expect("repeated HT encoder follows repeated execution") + .0; + + for (route, body, tier1_dispatch) in [ + ( + "distinct", + distinct, + "dispatch_ht_cleanup_batched_in_encoder_with_status_offset", + ), + ("repeated", repeated, "RepeatedHtChunkEncoder {"), + ] { + let zero_fill = body + .find("dispatch_zero_u32_buffer_in_encoder") + .unwrap_or_else(|| panic!("{route} HT execution must zero the decoded buffer")); + let barrier = body + .find("memory_barrier_with_resources(&[decoded])") + .unwrap_or_else(|| { + panic!("{route} HT execution must barrier the zero-filled decoded buffer") + }); + let tier1 = body + .find(tier1_dispatch) + .unwrap_or_else(|| panic!("{route} HT execution must dispatch Tier-1 work")); + assert!( + zero_fill < barrier && barrier < tier1, + "{route} HT execution must barrier the decoded buffer after zero-fill and before Tier-1" + ); + } +} + +#[test] +fn forced_multi_chunk_metal_output_matches_cpu_coefficients() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let mut pixels = Vec::new(); + pixels + .try_reserve_exact(64) + .expect("fixture pixel allocation"); + pixels.extend(0..64_u8); + let bytes = j2k_native::encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + .expect("encode forced-chunk fixture"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("fixture image"); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct fixture plan"); + let prepared = prepare_direct_grayscale_plan(&direct).expect("prepared fixture plan"); + let group = prepared.ht_groups.first().expect("prepared HT group"); + let max_payload = group + .jobs + .iter() + .map(|job| job.coded_len as usize) + .max() + .unwrap_or(0); + let tiny_limits = HtGpuJobChunkLimits::new( + NonZeroUsize::MIN, + max_payload, + size_of::(), + ); + let input = HtBatchInput { + source_index: 4, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }; + let chunk_plan = plan_metal_ht_chunks(&[input], tiny_limits).expect("forced chunk plan"); + assert!( + chunk_plan.chunk_count() > 1, + "fixture must split into chunks" + ); + let cpu = decode_prepared_ht_sub_band_group_on_cpu_profile(group, None) + .expect("CPU coefficient oracle"); + + let gpu = with_runtime(|runtime| { + let output = + new_shared_buffer(&runtime.device, group.total_coefficients * size_of::())?; + let command_buffer = new_command_buffer(&runtime.queue)?; + let encoder = new_compute_command_encoder(&command_buffer)?; + let (retained, status) = encode_metal_ht_batches_in_encoder( + runtime, + &encoder, + &[input], + &output, + group.total_coefficients, + tiny_limits, + )?; + encoder.end_encoding(); + commit_and_wait_metal(&command_buffer)?; + validate_direct_status(runtime, status)?; + let coefficients = + checked_buffer_slice::(&output, group.total_coefficients, "chunk parity")?; + drop(retained); + Ok(coefficients) + }) + .expect("forced multi-chunk Metal decode"); + + assert_eq!(gpu, cpu); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning.rs new file mode 100644 index 00000000..115d7233 --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::{mem::size_of, num::NonZeroUsize}; +use std::sync::Arc; + +use j2k_core::{BatchInfrastructureError, HtGpuJobChunkLimits, HtGpuJobPassBucket}; + +use super::super::execution::validate_pass_homogeneous_chunk; +use super::super::{ + allocate_packed_metal_ht_chunk, metal_ht_pipeline_kind_for_bucket, plan_metal_ht_chunks, Error, + HtBatchInput, HtPayloadSource, J2kHtCleanupBatchJob, MetalHtPipelineKind, PackedMetalHtChunk, +}; +use crate::batch_allocation::BatchMetadataBudget; +use crate::compute::PreparedHtExecutionOwner; + +mod fixtures; +mod referenced; + +fn limits(jobs: usize, payload: usize) -> HtGpuJobChunkLimits { + HtGpuJobChunkLimits::new( + NonZeroUsize::new(jobs).expect("test job cap is nonzero"), + payload, + jobs * size_of::(), + ) +} + +fn job(coded_offset: u32, coded_len: u32, passes: u32, output_offset: u32) -> J2kHtCleanupBatchJob { + J2kHtCleanupBatchJob { + coded_offset, + width: 1, + height: 1, + coded_len, + cleanup_length: coded_len, + refinement_length: 0, + missing_msbs: 0, + num_bitplanes: 1, + roi_shift: 0, + number_of_coding_passes: passes, + output_stride: 1, + output_offset, + dequantization_step: 1.0, + stripe_causal: 0, + } +} + +#[test] +fn pass_buckets_select_dedicated_metal_ht_pipelines() { + assert_eq!( + metal_ht_pipeline_kind_for_bucket(HtGpuJobPassBucket::CleanupOnly), + MetalHtPipelineKind::CleanupOnly + ); + assert_eq!( + metal_ht_pipeline_kind_for_bucket(HtGpuJobPassBucket::SigProp), + MetalHtPipelineKind::SigProp + ); + assert_eq!( + metal_ht_pipeline_kind_for_bucket(HtGpuJobPassBucket::MagRef), + MetalHtPipelineKind::MagRef + ); +} + +#[test] +fn magref_specialization_rejects_more_than_three_coding_passes() { + let chunk = PackedMetalHtChunk { + bucket: HtGpuJobPassBucket::MagRef, + coded_data: vec![0], + jobs: vec![job(0, 1, 4, 0)], + source_indices: vec![0], + }; + + assert!(matches!( + validate_pass_homogeneous_chunk(&chunk), + Err(Error::UnsupportedMetalRequest { + reason: "HTJ2K Metal decoding supports at most three coding passes per code block" + }) + )); +} + +#[test] +fn tiny_caps_split_jobs_in_pass_order_and_keep_source_mapping() { + let first_payload = [10, 11, 12, 13]; + let second_payload = [20, 21, 22, 23]; + let first_jobs = [job(0, 1, 3, 0), job(1, 1, 1, 1)]; + let second_jobs = [job(0, 1, 2, 0), job(1, 1, 1, 1)]; + let first_owner = Arc::new(PreparedHtExecutionOwner); + let second_owner = Arc::new(PreparedHtExecutionOwner); + let batches = [ + HtBatchInput { + source_index: 7, + payload: HtPayloadSource::Contiguous(&first_payload), + jobs: &first_jobs, + output_base: 0, + execution_owner: &first_owner, + }, + HtBatchInput { + source_index: 11, + payload: HtPayloadSource::Contiguous(&second_payload), + jobs: &second_jobs, + output_base: 8, + execution_owner: &second_owner, + }, + ]; + + let plan = plan_metal_ht_chunks(&batches, limits(1, 1)).expect("tiny chunk plan"); + assert_eq!(plan.job_count(), 4); + assert_eq!(plan.chunk_count(), 4); + + let packed = [0, 1, 2, 3].map(|index| plan.pack_chunk(index).expect("packed chunk")); + assert_eq!( + [ + packed[0].bucket, + packed[1].bucket, + packed[2].bucket, + packed[3].bucket, + ], + [ + HtGpuJobPassBucket::CleanupOnly, + HtGpuJobPassBucket::CleanupOnly, + HtGpuJobPassBucket::SigProp, + HtGpuJobPassBucket::MagRef, + ] + ); + assert!(packed + .iter() + .flat_map(|chunk| chunk.source_indices.iter().copied()) + .eq([7, 11, 11, 7])); + assert!(packed + .iter() + .flat_map(|chunk| chunk.coded_data.iter().copied()) + .eq([11, 21, 20, 10])); + assert!(packed + .iter() + .flat_map(|chunk| chunk.jobs.iter().map(|job| job.output_offset)) + .eq([1, 9, 8, 0])); +} + +#[test] +fn packed_chunk_rebases_payload_offsets_without_exceeding_caps() { + let payload = [1, 2, 3, 4, 5, 6]; + let jobs = [job(1, 2, 1, 0), job(4, 2, 1, 1)]; + let owner = Arc::new(PreparedHtExecutionOwner); + let batches = [HtBatchInput { + source_index: 3, + payload: HtPayloadSource::Contiguous(&payload), + jobs: &jobs, + output_base: 16, + execution_owner: &owner, + }]; + + let plan = plan_metal_ht_chunks(&batches, limits(2, 4)).expect("bounded chunk plan"); + let packed = plan.pack_chunk(0).expect("packed chunk"); + + assert_eq!(packed.coded_data, [2, 3, 5, 6]); + assert_eq!(packed.jobs.len(), 2); + assert_eq!(packed.jobs[0].coded_offset, 0); + assert_eq!(packed.jobs[1].coded_offset, 2); + assert_eq!(packed.jobs[0].output_offset, 16); + assert_eq!(packed.jobs[1].output_offset, 17); + assert_eq!(packed.source_indices, [3, 3]); + assert!(packed.coded_data.len() <= 4); + assert!( + packed.jobs.len() * size_of::() + <= 2 * size_of::() + ); +} + +#[test] +fn packed_ht_chunk_metadata_honors_exact_cap_and_one_byte_over() { + let payload_bytes = 7; + let job_count = 3; + let exact_cap = + payload_bytes + job_count * (size_of::() + size_of::()); + let owners = allocate_packed_metal_ht_chunk( + payload_bytes, + job_count, + BatchMetadataBudget::with_cap("HTJ2K Metal packed chunk metadata", exact_cap), + ) + .expect("exact packed HT chunk metadata cap"); + assert_eq!(owners.coded_data.capacity(), payload_bytes); + assert_eq!(owners.jobs.capacity(), job_count); + assert_eq!(owners.source_indices.capacity(), job_count); + + assert!(matches!( + allocate_packed_metal_ht_chunk( + payload_bytes, + job_count, + BatchMetadataBudget::with_cap("HTJ2K Metal packed chunk metadata", exact_cap - 1,), + ), + Err(Error::BatchInfrastructure( + BatchInfrastructureError::AllocationTooLarge { requested, cap, .. } + )) if requested == exact_cap && cap == exact_cap - 1 + )); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning/fixtures.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning/fixtures.rs new file mode 100644 index 00000000..2dc9177c --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning/fixtures.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_native::{DecodeSettings, DecoderContext, EncodeOptions, Image}; + +use super::super::super::{ + default_metal_ht_chunk_limits, metal_ht_pipeline_kind_for_bucket, plan_metal_ht_chunks, + HtBatchInput, MetalHtPipelineKind, +}; +use crate::compute::{ + prepare_direct_color_plan, prepare_direct_grayscale_plan, PreparedDirectGrayscalePlan, +}; + +#[test] +fn prepared_fixtures_select_expected_dedicated_metal_ht_pipelines() { + let cleanup_pixels = j2k_test_support::patterned_gray8(32, 32); + let cleanup_bytes = j2k_native::encode_htj2k( + &cleanup_pixels, + 32, + 32, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }, + ) + .expect("encode cleanup-only fixture"); + let cleanup = prepare_grayscale_fixture(&cleanup_bytes, "cleanup-only"); + assert_fixture_selects_pipeline( + "generated cleanup-only", + &prepared_fixture_pipeline_kinds(core::slice::from_ref(&cleanup)), + MetalHtPipelineKind::CleanupOnly, + ); + + let sigprop_bytes = j2k_test_support::openhtj2k_sigprop_fixture(); + let sigprop_image = Image::new(sigprop_bytes, &DecodeSettings::strict()) + .expect("independent OpenHTJ2K SigProp fixture"); + let mut sigprop_context = DecoderContext::default(); + let sigprop_direct = sigprop_image + .build_direct_color_plan_with_context(&mut sigprop_context) + .expect("independent OpenHTJ2K SigProp direct plan"); + let sigprop = prepare_direct_color_plan(&sigprop_direct) + .expect("prepare independent OpenHTJ2K SigProp plan"); + assert_fixture_selects_pipeline( + "independent OpenHTJ2K SigProp", + &prepared_fixture_pipeline_kinds(&sigprop.component_plans), + MetalHtPipelineKind::SigProp, + ); + + let magref = prepare_grayscale_fixture( + j2k_test_support::openhtj2k_refinement_fixture(), + "OpenHTJ2K MagRef", + ); + assert_fixture_selects_pipeline( + "OpenHTJ2K MagRef", + &prepared_fixture_pipeline_kinds(core::slice::from_ref(&magref)), + MetalHtPipelineKind::MagRef, + ); +} + +fn prepare_grayscale_fixture(bytes: &[u8], name: &str) -> PreparedDirectGrayscalePlan { + let image = Image::new(bytes, &DecodeSettings::strict()) + .unwrap_or_else(|error| panic!("parse {name} fixture: {error}")); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .unwrap_or_else(|error| panic!("build {name} direct plan: {error}")); + prepare_direct_grayscale_plan(&direct) + .unwrap_or_else(|error| panic!("prepare {name} Metal plan: {error}")) +} + +fn prepared_fixture_pipeline_kinds( + component_plans: &[PreparedDirectGrayscalePlan], +) -> Vec { + let mut kinds = Vec::new(); + for (source_index, component) in component_plans.iter().enumerate() { + for group in &component.ht_groups { + let input = HtBatchInput { + source_index, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }; + let plan = plan_metal_ht_chunks(&[input], default_metal_ht_chunk_limits()) + .expect("plan prepared fixture HT chunks"); + for chunk_index in 0..plan.chunk_count() { + let chunk = plan + .pack_chunk(chunk_index) + .expect("pack prepared fixture HT chunk"); + let kind = metal_ht_pipeline_kind_for_bucket(chunk.bucket); + if !kinds.contains(&kind) { + kinds.push(kind); + } + } + } + } + assert!( + !kinds.is_empty(), + "prepared fixture must contain planned HT jobs" + ); + kinds +} + +fn assert_fixture_selects_pipeline( + name: &str, + actual: &[MetalHtPipelineKind], + expected: MetalHtPipelineKind, +) { + assert!( + actual.contains(&expected), + "{name} planned pipelines {actual:?} must include {expected:?}" + ); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning/referenced.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning/referenced.rs new file mode 100644 index 00000000..ae5b406e --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/planning/referenced.rs @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::{mem::size_of, num::NonZeroUsize}; +use std::sync::Arc; + +use j2k_core::HtGpuJobChunkLimits; +use j2k_native::{HtCodeBlockPayloadRanges, J2kCodestreamRange}; + +use super::super::super::{ + plan_metal_ht_chunks, Error, HtBatchInput, HtPayloadSource, J2kHtCleanupBatchJob, +}; +use crate::compute::PreparedHtExecutionOwner; + +fn job( + coded_len: u32, + cleanup_length: u32, + refinement_length: u32, + output_offset: u32, +) -> J2kHtCleanupBatchJob { + J2kHtCleanupBatchJob { + coded_offset: 999, + width: 1, + height: 1, + coded_len, + cleanup_length, + refinement_length, + missing_msbs: 0, + num_bitplanes: 1, + roi_shift: 0, + number_of_coding_passes: if refinement_length == 0 { 1 } else { 2 }, + output_stride: 1, + output_offset, + dequantization_step: 1.0, + stripe_causal: 0, + } +} + +fn limits(jobs: usize, payload: usize) -> HtGpuJobChunkLimits { + HtGpuJobChunkLimits::new( + NonZeroUsize::new(jobs).expect("nonzero test job cap"), + payload, + jobs * size_of::(), + ) +} + +#[test] +fn referenced_payload_ranges_are_validated_before_chunk_packing() { + let bytes = Arc::<[u8]>::from([10, 11, 12]); + let ranges = [HtCodeBlockPayloadRanges { + cleanup: J2kCodestreamRange { + offset: 2, + length: 2, + }, + refinement: None, + }]; + let jobs = [job(2, 2, 0, 0)]; + let owner = Arc::new(PreparedHtExecutionOwner); + let input = [HtBatchInput { + source_index: 7, + payload: HtPayloadSource::Referenced { + input: &bytes, + ranges: &ranges, + }, + jobs: &jobs, + output_base: 0, + execution_owner: &owner, + }]; + + let Err(error) = plan_metal_ht_chunks(&input, limits(1, 2)) else { + panic!("out-of-bounds referenced payload must fail before packing") + }; + assert!(matches!(error, Error::MetalKernel { .. })); + assert!(error.to_string().contains("exceeds retained input")); +} + +#[test] +fn referenced_payload_pack_preserves_job_order_and_rebases_offsets() { + let bytes = Arc::<[u8]>::from([0, 10, 0, 11, 0, 20, 21, 0, 22]); + let ranges = [ + HtCodeBlockPayloadRanges { + cleanup: J2kCodestreamRange { + offset: 1, + length: 1, + }, + refinement: Some(J2kCodestreamRange { + offset: 3, + length: 1, + }), + }, + HtCodeBlockPayloadRanges { + cleanup: J2kCodestreamRange { + offset: 5, + length: 2, + }, + refinement: Some(J2kCodestreamRange { + offset: 8, + length: 1, + }), + }, + ]; + let jobs = [job(2, 1, 1, 2), job(3, 2, 1, 4)]; + let owner = Arc::new(PreparedHtExecutionOwner); + let input = [HtBatchInput { + source_index: 3, + payload: HtPayloadSource::Referenced { + input: &bytes, + ranges: &ranges, + }, + jobs: &jobs, + output_base: 16, + execution_owner: &owner, + }]; + + let plan = plan_metal_ht_chunks(&input, limits(2, 5)).expect("referenced chunk plan"); + let packed = plan.pack_chunk(0).expect("referenced packed chunk"); + + assert_eq!(packed.coded_data, [10, 11, 20, 21, 22]); + assert_eq!(packed.jobs[0].coded_offset, 0); + assert_eq!(packed.jobs[1].coded_offset, 2); + assert_eq!(packed.jobs[0].output_offset, 18); + assert_eq!(packed.jobs[1].output_offset, 20); + assert_eq!(packed.source_indices, [3, 3]); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/status.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/status.rs new file mode 100644 index 00000000..d1386e18 --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests/status.rs @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::mem::size_of; +use std::sync::Arc; + +use j2k_native::{DecodeSettings, DecoderContext, EncodeOptions, Image}; +use metal::foreign_types::ForeignType; + +use super::super::{ + default_metal_ht_chunk_limits, encode_metal_ht_batches_in_encoder, + encode_repeated_metal_ht_batch_in_command_buffer, HtBatchInput, +}; +use crate::compute::{ + checked_buffer_slice, commit_and_wait_metal, decode_prepared_ht_sub_band_group_on_cpu_profile, + new_command_buffer, new_compute_command_encoder, new_shared_buffer, + prepare_direct_grayscale_plan, validate_direct_status, wait_for_completion_metal, + DirectStatusCheck, MetalRuntime, PreparedHtExecutionOwner, +}; + +#[test] +fn completed_ht_status_storage_returns_to_the_shared_pool() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let pixels = (0..64_u8).rev().collect::>(); + let bytes = j2k_native::encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + .expect("encode status-pool fixture"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("fixture image"); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct fixture plan"); + let prepared = prepare_direct_grayscale_plan(&direct).expect("prepared fixture plan"); + let group = prepared.ht_groups.first().expect("prepared HT group"); + let input = HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }; + let runtime = MetalRuntime::new().expect("isolated Metal runtime"); + + for submission in 0..2 { + let output = + new_shared_buffer(&runtime.device, group.total_coefficients * size_of::()) + .expect("status-pool output"); + let command_buffer = new_command_buffer(&runtime.queue).expect("status-pool command"); + let encoder = new_compute_command_encoder(&command_buffer).expect("status-pool encoder"); + let (retained, status) = encode_metal_ht_batches_in_encoder( + &runtime, + &encoder, + &[input], + &output, + group.total_coefficients, + default_metal_ht_chunk_limits(), + ) + .expect("status-pool encode"); + encoder.end_encoding(); + commit_and_wait_metal(&command_buffer).expect("status-pool completion"); + validate_direct_status(&runtime, status).expect("status-pool validation"); + drop(retained); + + let diagnostics = runtime + .buffer_pool_diagnostics() + .expect("status-pool diagnostics"); + assert_eq!( + diagnostics.shared.cached_buffers, 1, + "submission {submission} must retire exactly one reusable HT status buffer" + ); + } +} + +#[test] +fn reused_ht_status_storage_is_overwritten_by_every_dispatched_job() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let image = Image::new( + j2k_test_support::openhtj2k_refinement_fixture(), + &DecodeSettings::default(), + ) + .expect("refinement status-overwrite fixture image"); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct fixture plan"); + let prepared = prepare_direct_grayscale_plan(&direct).expect("prepared fixture plan"); + let group = prepared.ht_groups.first().expect("prepared HT group"); + assert!( + group.jobs.iter().any(|job| job.number_of_coding_passes > 1), + "status-overwrite fixture must exercise refinement jobs" + ); + let mut invalid_jobs = group.jobs.clone(); + for job in &mut invalid_jobs { + job.num_bitplanes = 0; + } + let invalid_owner = Arc::new(PreparedHtExecutionOwner); + let runtime = MetalRuntime::new().expect("isolated Metal runtime"); + + let invalid_output = + new_shared_buffer(&runtime.device, group.total_coefficients * size_of::()) + .expect("invalid status output"); + let invalid_command = new_command_buffer(&runtime.queue).expect("invalid status command"); + let invalid_encoder = + new_compute_command_encoder(&invalid_command).expect("invalid status encoder"); + let (_, invalid_status) = encode_metal_ht_batches_in_encoder( + &runtime, + &invalid_encoder, + &[HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &invalid_jobs, + output_base: 0, + execution_owner: &invalid_owner, + }], + &invalid_output, + group.total_coefficients, + default_metal_ht_chunk_limits(), + ) + .expect("invalid status encode"); + invalid_encoder.end_encoding(); + commit_and_wait_metal(&invalid_command).expect("invalid status completion"); + let DirectStatusCheck::Ht { + buffer: invalid_status_buffer, + .. + } = &invalid_status + else { + panic!("invalid distinct submission must retain HT status") + }; + let invalid_status_ptr = invalid_status_buffer.as_ptr(); + assert!( + validate_direct_status(&runtime, invalid_status).is_err(), + "invalid first submission must seed every status slot with a failure" + ); + + let valid_output = + new_shared_buffer(&runtime.device, group.total_coefficients * size_of::()) + .expect("valid status output"); + let valid_command = new_command_buffer(&runtime.queue).expect("valid status command"); + let valid_encoder = new_compute_command_encoder(&valid_command).expect("valid status encoder"); + let (_, valid_status) = encode_metal_ht_batches_in_encoder( + &runtime, + &valid_encoder, + &[HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }], + &valid_output, + group.total_coefficients, + default_metal_ht_chunk_limits(), + ) + .expect("valid status encode"); + valid_encoder.end_encoding(); + commit_and_wait_metal(&valid_command).expect("valid status completion"); + let DirectStatusCheck::Ht { + buffer: valid_status_buffer, + .. + } = &valid_status + else { + panic!("valid distinct submission must retain HT status") + }; + assert_eq!( + valid_status_buffer.as_ptr(), + invalid_status_ptr, + "valid distinct dispatch must overwrite the recycled failure-status allocation" + ); + validate_direct_status(&runtime, valid_status) + .expect("every valid dispatch must overwrite its recycled failure status"); +} + +#[test] +fn reused_repeated_ht_status_storage_is_overwritten_by_every_dispatched_job() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let image = Image::new( + j2k_test_support::openhtj2k_refinement_fixture(), + &DecodeSettings::default(), + ) + .expect("repeated refinement status fixture image"); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("repeated refinement direct fixture plan"); + let prepared = prepare_direct_grayscale_plan(&direct).expect("prepared refinement plan"); + let group = prepared.ht_groups.first().expect("prepared HT group"); + assert!( + group.jobs.iter().any(|job| job.number_of_coding_passes > 1), + "repeated status-overwrite fixture must exercise refinement jobs" + ); + let mut invalid_jobs = group.jobs.clone(); + for job in &mut invalid_jobs { + job.num_bitplanes = 0; + } + let invalid_owner = Arc::new(PreparedHtExecutionOwner); + let runtime = MetalRuntime::new().expect("isolated repeated Metal runtime"); + let count = 2; + let output_words = group + .total_coefficients + .checked_mul(count) + .expect("repeated status output word count"); + + let invalid_output = new_shared_buffer(&runtime.device, output_words * size_of::()) + .expect("invalid repeated status output"); + let invalid_command = + new_command_buffer(&runtime.queue).expect("invalid repeated status command"); + let (_, invalid_status) = encode_repeated_metal_ht_batch_in_command_buffer( + &runtime, + &invalid_command, + HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &invalid_jobs, + output_base: 0, + execution_owner: &invalid_owner, + }, + count, + group.total_coefficients, + &invalid_output, + default_metal_ht_chunk_limits(), + ) + .expect("invalid repeated status encode"); + commit_and_wait_metal(&invalid_command).expect("invalid repeated status completion"); + let DirectStatusCheck::Ht { + buffer: invalid_status_buffer, + .. + } = &invalid_status + else { + panic!("invalid repeated submission must retain HT status") + }; + let invalid_status_ptr = invalid_status_buffer.as_ptr(); + assert!( + validate_direct_status(&runtime, invalid_status).is_err(), + "invalid repeated submission must seed every status slot with a failure" + ); + + let valid_output = new_shared_buffer(&runtime.device, output_words * size_of::()) + .expect("valid repeated status output"); + let valid_command = new_command_buffer(&runtime.queue).expect("valid repeated status command"); + let (_, valid_status) = encode_repeated_metal_ht_batch_in_command_buffer( + &runtime, + &valid_command, + HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }, + count, + group.total_coefficients, + &valid_output, + default_metal_ht_chunk_limits(), + ) + .expect("valid repeated status encode"); + commit_and_wait_metal(&valid_command).expect("valid repeated status completion"); + let DirectStatusCheck::Ht { + buffer: valid_status_buffer, + .. + } = &valid_status + else { + panic!("valid repeated submission must retain HT status") + }; + assert_eq!( + valid_status_buffer.as_ptr(), + invalid_status_ptr, + "valid repeated dispatch must overwrite the recycled failure-status allocation" + ); + validate_direct_status(&runtime, valid_status) + .expect("every valid repeated dispatch must overwrite recycled failure status"); +} + +#[test] +#[expect( + clippy::too_many_lines, + reason = "the overlap-lifetime scenario keeps both pending submissions and ownership assertions visible" +)] +fn overlapping_prepared_ht_submissions_keep_distinct_status_owners() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let pixels = (0..64_u8).collect::>(); + let bytes = j2k_native::encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + .expect("encode overlapping-status fixture"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("fixture image"); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct fixture plan"); + let prepared = prepare_direct_grayscale_plan(&direct).expect("prepared fixture plan"); + let group = prepared.ht_groups.first().expect("prepared HT group"); + let input = HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }; + let cpu = decode_prepared_ht_sub_band_group_on_cpu_profile(group, None) + .expect("overlapping CPU coefficient oracle"); + let runtime = MetalRuntime::new().expect("isolated Metal runtime"); + + let first_output = + new_shared_buffer(&runtime.device, group.total_coefficients * size_of::()) + .expect("first overlapping output"); + let first_command = new_command_buffer(&runtime.queue).expect("first overlapping command"); + let first_encoder = + new_compute_command_encoder(&first_command).expect("first overlapping encoder"); + let (first_retained, first_status) = encode_metal_ht_batches_in_encoder( + &runtime, + &first_encoder, + &[input], + &first_output, + group.total_coefficients, + default_metal_ht_chunk_limits(), + ) + .expect("first overlapping encode"); + first_encoder.end_encoding(); + first_command.commit(); + + let second_output = + new_shared_buffer(&runtime.device, group.total_coefficients * size_of::()) + .expect("second overlapping output"); + let second_command = new_command_buffer(&runtime.queue).expect("second overlapping command"); + let second_encoder = + new_compute_command_encoder(&second_command).expect("second overlapping encoder"); + let (second_retained, second_status) = encode_metal_ht_batches_in_encoder( + &runtime, + &second_encoder, + &[input], + &second_output, + group.total_coefficients, + default_metal_ht_chunk_limits(), + ) + .expect("second overlapping encode"); + second_encoder.end_encoding(); + second_command.commit(); + + let DirectStatusCheck::Ht { + buffer: first_buffer, + .. + } = &first_status + else { + panic!("first prepared HT submission must retain HT status") + }; + let DirectStatusCheck::Ht { + buffer: second_buffer, + .. + } = &second_status + else { + panic!("second prepared HT submission must retain HT status") + }; + assert_ne!( + first_buffer.as_ptr(), + second_buffer.as_ptr(), + "overlapping submissions must not alias in-flight status storage" + ); + + wait_for_completion_metal(&first_command).expect("first overlapping completion"); + wait_for_completion_metal(&second_command).expect("second overlapping completion"); + validate_direct_status(&runtime, first_status).expect("first overlapping status"); + validate_direct_status(&runtime, second_status).expect("second overlapping status"); + let first_coefficients = checked_buffer_slice::( + &first_output, + group.total_coefficients, + "first overlapping HT output", + ) + .expect("read first overlapping HT output"); + let second_coefficients = checked_buffer_slice::( + &second_output, + group.total_coefficients, + "second overlapping HT output", + ) + .expect("read second overlapping HT output"); + assert_eq!(first_coefficients, cpu); + assert_eq!(second_coefficients, cpu); + drop((first_retained, second_retained)); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_distinct.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_distinct.rs index 39f8fe48..32971993 100644 --- a/crates/j2k-metal/src/compute/decode_dispatch/ht_distinct.rs +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_distinct.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - copied_slice_buffer, dispatch_ht_cleanup_batched_in_command_buffer, ht_batch_output_word_count, - new_shared_buffer, Buffer, CommandBufferRef, DirectStatusCheck, Error, J2kHtCleanupBatchJob, - MetalRuntime, PreparedHtSubBand, PreparedHtSubBandGroup, + default_metal_ht_chunk_limits, encode_metal_ht_batches_in_encoder, new_compute_command_encoder, + new_shared_buffer, Buffer, CommandBufferRef, DirectStatusCheck, Error, HtBatchInput, + J2kHtCleanupBatchJob, MetalRuntime, PreparedHtSubBand, PreparedHtSubBandGroup, }; #[cfg(target_os = "macos")] @@ -12,6 +12,20 @@ pub(in crate::compute) fn encode_distinct_ht_sub_bands_to_buffer_in_command_buff command_buffer: &CommandBufferRef, sub_bands: &[&PreparedHtSubBand], output: &Buffer, +) -> Result<(Vec, DirectStatusCheck), Error> { + let encoder = new_compute_command_encoder(command_buffer)?; + let result = + encode_distinct_ht_sub_bands_to_buffer_in_encoder(runtime, &encoder, sub_bands, output); + encoder.end_encoding(); + result +} + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn encode_distinct_ht_sub_bands_to_buffer_in_encoder( + runtime: &MetalRuntime, + encoder: &metal::ComputeCommandEncoderRef, + sub_bands: &[&PreparedHtSubBand], + output: &Buffer, ) -> Result<(Vec, DirectStatusCheck), Error> { let Some(first) = sub_bands.first() else { let empty = new_shared_buffer(&runtime.device, 1)?; @@ -20,22 +34,31 @@ pub(in crate::compute) fn encode_distinct_ht_sub_bands_to_buffer_in_command_buff DirectStatusCheck::Ht { buffer: empty, len: 0, + source_indices: None, + recyclable_status: None, }, )); }; let per_instance_len = first.width as usize * first.height as usize; - encode_distinct_ht_batches_to_buffer_in_command_buffer( + let output_word_count = per_instance_len + .checked_mul(sub_bands.len()) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect distinct sub-band output length overflow".to_string(), + })?; + encode_distinct_ht_batches_to_buffer_in_encoder( runtime, - command_buffer, + encoder, sub_bands .iter() .enumerate() .map(|(index, sub_band)| DistinctHtBatch { - coded_data: &sub_band.coded_data, + payload: sub_band.payload_source.as_ht_payload_source(), jobs: &sub_band.jobs, output_base: index * per_instance_len, + execution_owner: &sub_band.execution_owner, }), output, + output_word_count, ) } @@ -45,6 +68,20 @@ pub(in crate::compute) fn encode_distinct_ht_sub_band_groups_to_buffer_in_comman command_buffer: &CommandBufferRef, groups: &[&PreparedHtSubBandGroup], output: &Buffer, +) -> Result<(Vec, DirectStatusCheck), Error> { + let encoder = new_compute_command_encoder(command_buffer)?; + let result = + encode_distinct_ht_sub_band_groups_to_buffer_in_encoder(runtime, &encoder, groups, output); + encoder.end_encoding(); + result +} + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn encode_distinct_ht_sub_band_groups_to_buffer_in_encoder( + runtime: &MetalRuntime, + encoder: &metal::ComputeCommandEncoderRef, + groups: &[&PreparedHtSubBandGroup], + output: &Buffer, ) -> Result<(Vec, DirectStatusCheck), Error> { let Some(first) = groups.first() else { let empty = new_shared_buffer(&runtime.device, 1)?; @@ -53,181 +90,103 @@ pub(in crate::compute) fn encode_distinct_ht_sub_band_groups_to_buffer_in_comman DirectStatusCheck::Ht { buffer: empty, len: 0, + source_indices: None, + recyclable_status: None, }, )); }; let per_instance_len = first.total_coefficients; - encode_distinct_ht_batches_to_buffer_in_command_buffer( + let output_word_count = + per_instance_len + .checked_mul(groups.len()) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect distinct group output length overflow".to_string(), + })?; + encode_distinct_ht_batches_to_buffer_in_encoder( runtime, - command_buffer, + encoder, groups .iter() .enumerate() .map(|(index, group)| DistinctHtBatch { - coded_data: &group.coded_arena.data, + payload: group.payload_source.as_ht_payload_source(), jobs: &group.jobs, output_base: index * per_instance_len, + execution_owner: &group.execution_owner, }), output, + output_word_count, ) } #[cfg(target_os = "macos")] +#[derive(Clone, Copy)] pub(in crate::compute) struct DistinctHtBatch<'a> { - pub(in crate::compute) coded_data: &'a [u8], + pub(in crate::compute) payload: super::HtPayloadSource<'a>, pub(in crate::compute) jobs: &'a [J2kHtCleanupBatchJob], pub(in crate::compute) output_base: usize, + pub(in crate::compute) execution_owner: + &'a std::sync::Arc, } #[cfg(target_os = "macos")] -struct DistinctHtMetadata { - coded_data: Vec, - jobs: Vec, -} - -#[cfg(target_os = "macos")] -fn allocate_distinct_ht_metadata( - coded_len: usize, - job_count: usize, - mut budget: crate::batch_allocation::BatchMetadataBudget, -) -> Result { - let requests = [ - crate::batch_allocation::BatchMetadataRequest::of::(coded_len), - crate::batch_allocation::BatchMetadataRequest::of::(job_count), - ]; - budget.preflight(&requests)?; - Ok(DistinctHtMetadata { - coded_data: budget.try_vec( - coded_len, - "HTJ2K MetalDirect distinct grayscale coded payload", - )?, - jobs: budget.try_vec(job_count, "HTJ2K MetalDirect distinct grayscale jobs")?, - }) -} - -#[cfg(target_os = "macos")] -pub(in crate::compute) fn encode_distinct_ht_batches_to_buffer_in_command_buffer<'a>( +fn encode_distinct_ht_batches_to_buffer_in_encoder<'a>( runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, + encoder: &metal::ComputeCommandEncoderRef, batches: impl Iterator> + Clone, output: &Buffer, + output_word_count: usize, ) -> Result<(Vec, DirectStatusCheck), Error> { - let coded_len = crate::batch_allocation::checked_count_sum( - batches.clone().map(|batch| batch.coded_data.len()), - "HTJ2K MetalDirect distinct grayscale coded payload", + let batch_count = batches.clone().count(); + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "HTJ2K MetalDirect distinct chunk submission", + ); + let mut inputs = budget.try_vec( + batch_count, + "HTJ2K MetalDirect distinct chunk input descriptors", )?; - let job_count = crate::batch_allocation::checked_count_sum( - batches.clone().map(|batch| batch.jobs.len()), - "HTJ2K MetalDirect distinct grayscale jobs", - )?; - let DistinctHtMetadata { - mut coded_data, - mut jobs, - } = allocate_distinct_ht_metadata( - coded_len, - job_count, - crate::batch_allocation::BatchMetadataBudget::new( - "HTJ2K MetalDirect distinct grayscale submission", - ), - )?; - - for batch in batches { - let coded_base = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect distinct grayscale coded payload exceeds u32".to_string(), - })?; - coded_data.extend_from_slice(batch.coded_data); - let output_base = u32::try_from(batch.output_base).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect distinct grayscale output offset exceeds u32".to_string(), - })?; - for job in batch.jobs { - let mut adjusted = *job; - adjusted.coded_offset = - adjusted - .coded_offset - .checked_add(coded_base) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect distinct grayscale job coded offset overflow" - .to_string(), - })?; - adjusted.output_offset = - adjusted - .output_offset - .checked_add(output_base) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect distinct grayscale job output offset overflow" - .to_string(), - })?; - jobs.push(adjusted); + inputs.extend( + batches + .enumerate() + .map(|(source_index, batch)| HtBatchInput { + source_index, + payload: batch.payload, + jobs: batch.jobs, + output_base: batch.output_base, + execution_owner: batch.execution_owner, + }), + ); + for input in &inputs { + for job in input.jobs { + let output_offset = (job.output_offset as usize) + .checked_add(input.output_base) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect distinct output span overflow".to_string(), + })?; + let output_offset = u32::try_from(output_offset).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect distinct output span exceeds u32".to_string(), + })?; + let job_output_end = super::ht_output_word_count( + output_offset, + job.output_stride, + job.width, + job.height, + )?; + if job_output_end > output_word_count { + return Err(Error::MetalStateInvariant { + state: "HTJ2K MetalDirect distinct output", + reason: "code-block output span exceeds the logical coefficient arena", + }); + } } } - if jobs.is_empty() { - let empty = new_shared_buffer(&runtime.device, 1)?; - return Ok(( - vec![empty.clone()], - DirectStatusCheck::Ht { - buffer: empty, - len: 0, - }, - )); - } - - let coded_buffer = copied_slice_buffer(&runtime.device, &coded_data)?; - let jobs_buffer = copied_slice_buffer(&runtime.device, &jobs)?; - let status_check = dispatch_ht_cleanup_batched_in_command_buffer( + encode_metal_ht_batches_in_encoder( runtime, - command_buffer, - &coded_buffer, - &jobs_buffer, - jobs.len(), + encoder, + &inputs, output, - ht_batch_output_word_count(&jobs)?, - )?; - Ok((vec![coded_buffer, jobs_buffer], status_check)) -} - -#[cfg(all(test, target_os = "macos"))] -mod tests { - use core::mem::size_of; - - use j2k_core::BatchInfrastructureError; - - use super::{allocate_distinct_ht_metadata, Error, J2kHtCleanupBatchJob}; - use crate::batch_allocation::BatchMetadataBudget; - - #[test] - fn distinct_ht_metadata_honors_exact_cap_and_one_byte_over() { - let coded_len = 7; - let job_count = 3; - let exact_cap = coded_len + job_count * size_of::(); - let owners = allocate_distinct_ht_metadata( - coded_len, - job_count, - BatchMetadataBudget::with_cap( - "HTJ2K MetalDirect distinct grayscale submission", - exact_cap, - ), - ) - .expect("exact distinct HT metadata cap"); - assert_eq!(owners.coded_data.capacity(), coded_len); - assert_eq!(owners.jobs.capacity(), job_count); - - assert!(matches!( - allocate_distinct_ht_metadata( - coded_len, - job_count, - BatchMetadataBudget::with_cap( - "HTJ2K MetalDirect distinct grayscale submission", - exact_cap - 1, - ), - ), - Err(Error::BatchInfrastructure( - BatchInfrastructureError::AllocationTooLarge { - requested, - cap, - .. - } - )) if requested == exact_cap && cap == exact_cap - 1 - )); - } + output_word_count, + default_metal_ht_chunk_limits(), + ) } diff --git a/crates/j2k-metal/src/compute/decode_dispatch/ht_subband.rs b/crates/j2k-metal/src/compute/decode_dispatch/ht_subband.rs index b56baa03..ce122b3c 100644 --- a/crates/j2k-metal/src/compute/decode_dispatch/ht_subband.rs +++ b/crates/j2k-metal/src/compute/decode_dispatch/ht_subband.rs @@ -1,11 +1,10 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - dispatch_1d_pipeline, dispatch_ht_cleanup_batched_in_encoder, - dispatch_ht_cleanup_repeated_batched_in_command_buffer, new_shared_buffer, prepared_ht_buffer, - size_of, Buffer, CommandBufferRef, ComputeCommandEncoderRef, DirectStatusCheck, Error, - HtCodeBlockDecodeJob, HtRepeatedCleanupDispatch, J2kHtCleanupBatchJob, MetalRuntime, - PreparedHtSubBand, PreparedHtSubBandGroup, + default_metal_ht_chunk_limits, dispatch_1d_pipeline, encode_metal_ht_batches_in_encoder, + encode_repeated_metal_ht_batch_in_command_buffer, size_of, Buffer, CommandBufferRef, + ComputeCommandEncoderRef, DirectStatusCheck, Error, HtBatchInput, HtCodeBlockDecodeJob, + J2kHtCleanupBatchJob, MetalRuntime, PreparedHtSubBand, PreparedHtSubBandGroup, }; #[cfg(target_os = "macos")] @@ -32,38 +31,21 @@ pub(in crate::compute) fn encode_repeated_ht_sub_band_to_buffer_in_command_buffe count: usize, output: &Buffer, ) -> Result<(Vec, DirectStatusCheck), Error> { - if count == 0 || job.jobs.is_empty() { - let empty = new_shared_buffer(&runtime.device, 1)?; - return Ok(( - vec![empty.clone()], - DirectStatusCheck::Ht { - buffer: empty, - len: 0, - }, - )); - } - - let total_jobs = job - .jobs - .len() - .checked_mul(count) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect repeated job count overflow".to_string(), - })?; - let coded_buffer = prepared_ht_buffer(job.coded_buffer.as_ref(), "coded")?.clone(); - let jobs_buffer = prepared_ht_buffer(job.jobs_buffer.as_ref(), "jobs")?.clone(); - let status_check = - dispatch_ht_cleanup_repeated_batched_in_command_buffer(HtRepeatedCleanupDispatch { - runtime, - command_buffer, - coded_data: &coded_buffer, - jobs: &jobs_buffer, - base_job_count: job.jobs.len(), - total_job_count: total_jobs, - output_plane_len: job.width as usize * job.height as usize, - decoded: output, - })?; - Ok((vec![coded_buffer, jobs_buffer], status_check)) + encode_repeated_metal_ht_batch_in_command_buffer( + runtime, + command_buffer, + HtBatchInput { + source_index: 0, + payload: job.payload_source.as_ht_payload_source(), + jobs: &job.jobs, + output_base: 0, + execution_owner: &job.execution_owner, + }, + count, + job.width as usize * job.height as usize, + output, + default_metal_ht_chunk_limits(), + ) } #[cfg(target_os = "macos")] @@ -74,38 +56,21 @@ pub(in crate::compute) fn encode_repeated_ht_sub_band_group_to_buffer_in_command count: usize, output: &Buffer, ) -> Result<(Vec, DirectStatusCheck), Error> { - if count == 0 || group.jobs.is_empty() { - let empty = new_shared_buffer(&runtime.device, 1)?; - return Ok(( - vec![empty.clone()], - DirectStatusCheck::Ht { - buffer: empty, - len: 0, - }, - )); - } - - let total_jobs = group - .jobs - .len() - .checked_mul(count) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect repeated grouped job count overflow".to_string(), - })?; - let coded_buffer = group.coded_arena.buffer.clone(); - let jobs_buffer = group.jobs_buffer.clone(); - let status_check = - dispatch_ht_cleanup_repeated_batched_in_command_buffer(HtRepeatedCleanupDispatch { - runtime, - command_buffer, - coded_data: &coded_buffer, - jobs: &jobs_buffer, - base_job_count: group.jobs.len(), - total_job_count: total_jobs, - output_plane_len: group.total_coefficients, - decoded: output, - })?; - Ok((vec![coded_buffer, jobs_buffer], status_check)) + encode_repeated_metal_ht_batch_in_command_buffer( + runtime, + command_buffer, + HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }, + count, + group.total_coefficients, + output, + default_metal_ht_chunk_limits(), + ) } #[cfg(target_os = "macos")] @@ -115,35 +80,20 @@ pub(in crate::compute) fn encode_prepared_ht_sub_band_to_buffer_in_encoder( job: &PreparedHtSubBand, output: &Buffer, ) -> Result<(Vec, DirectStatusCheck), Error> { - if job.jobs.is_empty() { - dispatch_zero_u32_buffer_in_encoder( - runtime, - encoder, - output, - job.width as usize * job.height as usize, - )?; - let empty = new_shared_buffer(&runtime.device, 1)?; - return Ok(( - vec![empty.clone()], - DirectStatusCheck::Ht { - buffer: empty, - len: 0, - }, - )); - } - - let coded_buffer = prepared_ht_buffer(job.coded_buffer.as_ref(), "coded")?.clone(); - let jobs_buffer = prepared_ht_buffer(job.jobs_buffer.as_ref(), "jobs")?.clone(); - let status_check = dispatch_ht_cleanup_batched_in_encoder( + encode_metal_ht_batches_in_encoder( runtime, encoder, - &coded_buffer, - &jobs_buffer, - job.jobs.len(), + &[HtBatchInput { + source_index: 0, + payload: job.payload_source.as_ht_payload_source(), + jobs: &job.jobs, + output_base: 0, + execution_owner: &job.execution_owner, + }], output, job.width as usize * job.height as usize, - )?; - Ok((vec![coded_buffer, jobs_buffer], status_check)) + default_metal_ht_chunk_limits(), + ) } #[cfg(target_os = "macos")] @@ -153,30 +103,20 @@ pub(in crate::compute) fn encode_prepared_ht_sub_band_group_to_buffer_in_encoder group: &PreparedHtSubBandGroup, output: &Buffer, ) -> Result<(Vec, DirectStatusCheck), Error> { - if group.jobs.is_empty() { - dispatch_zero_u32_buffer_in_encoder(runtime, encoder, output, group.total_coefficients)?; - let empty = new_shared_buffer(&runtime.device, 1)?; - return Ok(( - vec![empty.clone()], - DirectStatusCheck::Ht { - buffer: empty, - len: 0, - }, - )); - } - - let coded_buffer = group.coded_arena.buffer.clone(); - let jobs_buffer = group.jobs_buffer.clone(); - let status_check = dispatch_ht_cleanup_batched_in_encoder( + encode_metal_ht_batches_in_encoder( runtime, encoder, - &coded_buffer, - &jobs_buffer, - group.jobs.len(), + &[HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }], output, group.total_coefficients, - )?; - Ok((vec![coded_buffer, jobs_buffer], status_check)) + default_metal_ht_chunk_limits(), + ) } #[cfg(target_os = "macos")] diff --git a/crates/j2k-metal/src/compute/decode_dispatch/idwt.rs b/crates/j2k-metal/src/compute/decode_dispatch/idwt.rs index f9056a66..ec2a519e 100644 --- a/crates/j2k-metal/src/compute/decode_dispatch/idwt.rs +++ b/crates/j2k-metal/src/compute/decode_dispatch/idwt.rs @@ -230,6 +230,7 @@ pub(in crate::compute) fn dispatch_reversible53_single_decomposition_buffers_in_ &runtime.idwt_interleave, (params.width, params.height), ); + encoder.memory_barrier_with_resources(&[decoded]); encoder.set_compute_pipeline_state(&runtime.idwt_reversible53_horizontal); encoder.set_buffer(0, Some(decoded), decoded_offset as u64); @@ -254,6 +255,7 @@ pub(in crate::compute) fn dispatch_reversible53_single_decomposition_buffers_in_ depth: 1, }, ); + encoder.memory_barrier_with_resources(&[decoded]); encoder.set_compute_pipeline_state(&runtime.idwt_reversible53_vertical); encoder.set_buffer(0, Some(decoded), decoded_offset as u64); @@ -377,3 +379,92 @@ pub(in crate::compute) fn dispatch_reversible53_repeated_buffers_in_command_buff encoder.end_encoding(); Ok(()) } + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn dispatch_reversible53_repeated_buffers_in_encoder_with_offsets( + encoder: &ComputeCommandEncoderRef, + dispatch: RepeatedIdwtDispatch<'_>, +) { + let RepeatedIdwtDispatch { + runtime, + sub_bands, + params, + decoded, + } = dispatch; + let IdwtSubBandBuffers { + ll, + ll_offset, + hl, + hl_offset, + lh, + lh_offset, + hh, + hh_offset, + } = sub_bands; + encoder.set_compute_pipeline_state(&runtime.idwt_interleave_batched); + encoder.set_buffer(0, Some(ll), ll_offset as u64); + encoder.set_buffer(1, Some(hl), hl_offset as u64); + encoder.set_buffer(2, Some(lh), lh_offset as u64); + encoder.set_buffer(3, Some(hh), hh_offset as u64); + encoder.set_buffer(4, Some(decoded), 0); + encoder.set_bytes( + 5, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_3d_pipeline( + encoder, + &runtime.idwt_interleave_batched, + (params.width, params.height, params.batch_count), + ); + encoder.memory_barrier_with_resources(&[decoded]); + + encoder.set_compute_pipeline_state(&runtime.idwt_reversible53_horizontal_batched); + encoder.set_buffer(0, Some(decoded), 0); + encoder.set_bytes( + 1, + size_of::() as u64, + (&raw const params).cast(), + ); + let horizontal_width = runtime + .idwt_reversible53_horizontal_batched + .thread_execution_width() + .max(1); + encoder.dispatch_threads( + MTLSize { + width: u64::from(params.height), + height: u64::from(params.batch_count), + depth: 1, + }, + MTLSize { + width: horizontal_width, + height: 1, + depth: 1, + }, + ); + encoder.memory_barrier_with_resources(&[decoded]); + + encoder.set_compute_pipeline_state(&runtime.idwt_reversible53_vertical_batched); + encoder.set_buffer(0, Some(decoded), 0); + encoder.set_bytes( + 1, + size_of::() as u64, + (&raw const params).cast(), + ); + let vertical_width = runtime + .idwt_reversible53_vertical_batched + .thread_execution_width() + .max(1); + encoder.dispatch_threads( + MTLSize { + width: u64::from(params.width), + height: u64::from(params.batch_count), + depth: 1, + }, + MTLSize { + width: vertical_width, + height: 1, + depth: 1, + }, + ); +} diff --git a/crates/j2k-metal/src/compute/decode_dispatch/store.rs b/crates/j2k-metal/src/compute/decode_dispatch/store.rs index 3e93f607..dd131102 100644 --- a/crates/j2k-metal/src/compute/decode_dispatch/store.rs +++ b/crates/j2k-metal/src/compute/decode_dispatch/store.rs @@ -1,13 +1,21 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - checked_buffer_slice, checked_metal_surface_len, commit_and_wait_metal, copied_slice_buffer, - dispatch_2d_pipeline, dispatch_3d_pipeline, hybrid_stage_signpost, label_compute_encoder, - new_command_buffer, new_compute_command_encoder, new_shared_buffer, size_of, with_runtime, - Buffer, CommandBufferRef, ComputeCommandEncoderRef, Error, J2kGrayStoreParams, + checked_buffer_slice, commit_and_wait_metal, copied_slice_buffer, dispatch_2d_pipeline, + dispatch_3d_pipeline, hybrid_stage_signpost, label_compute_encoder, new_command_buffer, + new_compute_command_encoder, new_shared_buffer, size_of, with_runtime, Buffer, + CommandBufferRef, ComputeCommandEncoderRef, Error, J2kGrayStoreParams, J2kRepeatedGrayStoreParams, J2kRepeatedStoreParams, J2kStoreComponentJob, J2kStoreParams, MTLSize, MetalRuntime, PixelFormat, Surface, SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE, }; +use crate::compute::direct_surface_pack::checked_metal_surface_len; +#[cfg(target_os = "macos")] +mod destination; + +#[cfg(target_os = "macos")] +pub(in crate::compute) use self::destination::{ + encode_gray_store_to_destination_in_encoder, GrayStoreDestinationRequest, +}; #[cfg(target_os = "macos")] pub(crate) fn decode_store_component_and_capture( @@ -172,6 +180,26 @@ pub(in crate::compute) fn dispatch_store_component_repeated_in_command_buffer( let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE); let encoder = new_compute_command_encoder(command_buffer)?; label_compute_encoder(&encoder, "J2K decode hybrid repeated component store"); + dispatch_store_component_repeated_in_encoder( + runtime, + &encoder, + input, + input_offset_bytes, + output, + params, + ); + encoder.end_encoding(); + Ok(()) +} + +pub(in crate::compute) fn dispatch_store_component_repeated_in_encoder( + runtime: &MetalRuntime, + encoder: &ComputeCommandEncoderRef, + input: &Buffer, + input_offset_bytes: usize, + output: &Buffer, + params: J2kRepeatedStoreParams, +) { encoder.set_compute_pipeline_state(&runtime.store_component_repeated); encoder.set_buffer(0, Some(input), input_offset_bytes as u64); encoder.set_buffer(1, Some(output), 0); @@ -181,12 +209,10 @@ pub(in crate::compute) fn dispatch_store_component_repeated_in_command_buffer( (&raw const params).cast(), ); dispatch_3d_pipeline( - &encoder, + encoder, &runtime.store_component_repeated, (params.copy_width, params.copy_height, params.batch_count), ); - encoder.end_encoding(); - Ok(()) } #[cfg(target_os = "macos")] @@ -309,6 +335,7 @@ pub(in crate::compute) fn encode_gray_store_to_surface_in_encoder( let pipeline = match fmt { PixelFormat::Gray8 => &runtime.store_component_gray_u8, PixelFormat::Gray16 => &runtime.store_component_gray_u16, + PixelFormat::GrayI16 => &runtime.store_component_gray_i16, _ => { return Err(Error::MetalKernel { message: format!("J2K Metal grayscale fused store does not support {fmt:?}"), diff --git a/crates/j2k-metal/src/compute/decode_dispatch/store/destination.rs b/crates/j2k-metal/src/compute/decode_dispatch/store/destination.rs new file mode 100644 index 00000000..10843ce2 --- /dev/null +++ b/crates/j2k-metal/src/compute/decode_dispatch/store/destination.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + dispatch_2d_pipeline, size_of, Buffer, ComputeCommandEncoderRef, Error, J2kGrayStoreParams, + MetalRuntime, PixelFormat, +}; +use j2k_metal_support::MetalImageDestination; + +#[derive(Clone, Copy)] +pub(in crate::compute) struct GrayStoreDestinationRequest<'a> { + pub(in crate::compute) runtime: &'a MetalRuntime, + pub(in crate::compute) encoder: &'a ComputeCommandEncoderRef, + pub(in crate::compute) input: &'a Buffer, + pub(in crate::compute) input_offset_bytes: usize, + pub(in crate::compute) params: J2kGrayStoreParams, + pub(in crate::compute) dims: (u32, u32), + pub(in crate::compute) fmt: PixelFormat, + pub(in crate::compute) destination: &'a MetalImageDestination, + pub(in crate::compute) destination_item_index: usize, +} + +pub(in crate::compute) fn encode_gray_store_to_destination_in_encoder( + request: GrayStoreDestinationRequest<'_>, +) -> Result<(), Error> { + let GrayStoreDestinationRequest { + runtime, + encoder, + input, + input_offset_bytes, + mut params, + dims, + fmt, + destination, + destination_item_index, + } = request; + destination + .validate_device(&runtime.device) + .and_then(|()| destination.validate_image(dims, fmt)) + .map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K Metal grayscale final-store destination validation failed", + source, + ) + })?; + let layout = destination.layout(); + let bytes_per_sample = fmt.bytes_per_sample(); + if !layout.pitch_bytes().is_multiple_of(bytes_per_sample) { + return Err(Error::MetalKernel { + message: "J2K Metal grayscale destination pitch is not sample aligned".to_string(), + }); + } + params.output_stride = + u32::try_from(layout.pitch_bytes() / bytes_per_sample).map_err(|_| Error::MetalKernel { + message: "J2K Metal grayscale destination stride exceeds u32".to_string(), + })?; + let item_offset_bytes = layout + .image_offset_bytes(destination_item_index) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal grayscale destination item index exceeds group".to_string(), + })?; + if !item_offset_bytes.is_multiple_of(bytes_per_sample) { + return Err(Error::MetalKernel { + message: "J2K Metal grayscale destination item stride is not sample aligned" + .to_string(), + }); + } + params.output_item_offset = + u32::try_from(item_offset_bytes / bytes_per_sample).map_err(|_| Error::MetalKernel { + message: "J2K Metal grayscale destination item offset exceeds u32".to_string(), + })?; + let pipeline = match fmt { + PixelFormat::Gray8 => &runtime.store_component_gray_u8, + PixelFormat::Gray16 => &runtime.store_component_gray_u16, + PixelFormat::GrayI16 => &runtime.store_component_gray_i16, + _ => { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal external final-store currently supports Gray8/Gray16/GrayI16", + }); + } + }; + + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(input), input_offset_bytes as u64); + // SAFETY: `MetalImageDestination` validated this exact allocation range, + // and the decode-into submission retains exclusive access until command + // completion or a same-device consumer dependency is registered. The raw + // handle does not escape this encoder binding. + encoder.set_buffer( + 1, + Some(unsafe { destination.raw_buffer() }), + layout.byte_offset() as u64, + ); + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline(encoder, pipeline, (params.copy_width, params.copy_height)); + Ok(()) +} diff --git a/crates/j2k-metal/src/compute/direct_buffers.rs b/crates/j2k-metal/src/compute/direct_buffers.rs index 37335344..4ede44da 100644 --- a/crates/j2k-metal/src/compute/direct_buffers.rs +++ b/crates/j2k-metal/src/compute/direct_buffers.rs @@ -12,9 +12,10 @@ use metal::{Buffer, Device}; use crate::{error::metal_kernel_support_error, Error}; +use super::abi::J2K_CLASSIC_MAX_COEFF_COUNT; use super::{ direct_scratch::{take_recyclable_shared_buffer, DirectScratchBuffer}, - MetalRuntime, J2K_CLASSIC_MAX_COEFF_COUNT, + MetalRuntime, }; fn buffer_access_error(context: &str, error: MetalSupportError) -> Error { diff --git a/crates/j2k-metal/src/compute/direct_commands.rs b/crates/j2k-metal/src/compute/direct_commands.rs index 8dd49d40..1b2d6d09 100644 --- a/crates/j2k-metal/src/compute/direct_commands.rs +++ b/crates/j2k-metal/src/compute/direct_commands.rs @@ -1,10 +1,44 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use metal::{CommandBuffer, CommandBufferRef}; +use j2k_metal_support::{ + checked_blit_command_encoder, checked_command_buffer, checked_compute_command_encoder, +}; +use metal::{ + BlitCommandEncoder, CommandBuffer, CommandBufferRef, CommandQueueRef, ComputeCommandEncoder, +}; +use crate::error::metal_kernel_support_error; use crate::profile_env::label_command_buffer; -use super::{new_command_buffer, Error, MetalRuntime}; +use super::{Error, MetalRuntime}; + +pub(in crate::compute) fn new_command_buffer( + queue: &CommandQueueRef, +) -> Result { + #[cfg(test)] + super::test_counters::record_metal_command_buffer(); + checked_command_buffer(queue).map_err(|source| { + metal_kernel_support_error("J2K Metal command buffer creation failed", source) + }) +} + +pub(in crate::compute) fn new_compute_command_encoder( + command_buffer: &CommandBufferRef, +) -> Result { + #[cfg(test)] + super::test_counters::record_metal_compute_encoder(); + checked_compute_command_encoder(command_buffer).map_err(|source| { + metal_kernel_support_error("J2K Metal compute encoder creation failed", source) + }) +} + +pub(in crate::compute) fn new_blit_command_encoder( + command_buffer: &CommandBufferRef, +) -> Result { + checked_blit_command_encoder(command_buffer).map_err(|source| { + metal_kernel_support_error("J2K Metal blit encoder creation failed", source) + }) +} #[derive(Clone, Copy)] pub(super) struct DirectIdwtCommandBuffers<'a> { diff --git a/crates/j2k-metal/src/compute/direct_cpu.rs b/crates/j2k-metal/src/compute/direct_cpu.rs index 4ca3a4ac..3a51a882 100644 --- a/crates/j2k-metal/src/compute/direct_cpu.rs +++ b/crates/j2k-metal/src/compute/direct_cpu.rs @@ -13,20 +13,24 @@ use j2k_native::{ }; use rayon::prelude::*; +use crate::profile_env::{hybrid_stage_signpost, SIGNPOST_DECODE_HYBRID_CPU_TIER1}; use crate::{error::native_decode_error, Error}; -use super::{ - checked_coefficient_len, hybrid_stage_signpost, packed_cpu_decode_coefficients, - packed_cpu_decode_coefficients_in, packed_cpu_decode_output_len, - record_hybrid_cpu_decode_inputs, record_hybrid_cpu_decode_worker_init, - required_classic_output_len, required_ht_output_len, CpuTier1DecodeSubstageCounters, - J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob, PreparedClassicSubBand, - PreparedClassicSubBandGroup, PreparedDirectGrayscalePlan, PreparedHtSubBand, - PreparedHtSubBandGroup, HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK, +use super::abi::{ + J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob, J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES, J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT, - SIGNPOST_DECODE_HYBRID_CPU_TIER1, +}; +use super::decode_dispatch::required_ht_output_len; +use super::direct_grayscale_execute::checked_coefficient_len; +use super::{ + packed_cpu_decode_coefficients, packed_cpu_decode_coefficients_in, + packed_cpu_decode_output_len, record_hybrid_cpu_decode_inputs, + record_hybrid_cpu_decode_worker_init, required_classic_output_len, + CpuTier1DecodeSubstageCounters, PreparedClassicSubBand, PreparedClassicSubBandGroup, + PreparedDirectGrayscalePlan, PreparedHtPayloadSource, PreparedHtSubBand, + PreparedHtSubBandGroup, HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK, }; #[cfg(test)] @@ -290,17 +294,18 @@ pub(super) fn decode_prepared_ht_sub_band_on_cpu_profile( "HTJ2K MetalDirect hybrid sub-band size overflow", )?; let mut output = packed_cpu_decode_coefficients(1, len)?; + let coded_data = sub_band.payload_source.materialize_for_cpu()?; if let Some(counters) = profile_counters { let mut workspace = HtCodeBlockDecodeWorkspace::default(); decode_prepared_ht_jobs_on_cpu_with_workspace_profiled( - &sub_band.coded_data, + coded_data.as_ref(), &sub_band.jobs, &mut output, &mut workspace, counters, )?; } else { - decode_prepared_ht_jobs_on_cpu(&sub_band.coded_data, &sub_band.jobs, &mut output)?; + decode_prepared_ht_jobs_on_cpu(coded_data.as_ref(), &sub_band.jobs, &mut output)?; } Ok(output) } @@ -311,17 +316,18 @@ pub(super) fn decode_prepared_ht_sub_band_group_on_cpu_profile( ) -> Result, Error> { let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_CPU_TIER1); let mut output = packed_cpu_decode_coefficients(1, group.total_coefficients)?; + let coded_data = group.payload_source.materialize_for_cpu()?; if let Some(counters) = profile_counters { let mut workspace = HtCodeBlockDecodeWorkspace::default(); decode_prepared_ht_jobs_on_cpu_with_workspace_profiled( - &group.coded_arena.data, + coded_data.as_ref(), &group.jobs, &mut output, &mut workspace, counters, )?; } else { - decode_prepared_ht_jobs_on_cpu(&group.coded_arena.data, &group.jobs, &mut output)?; + decode_prepared_ht_jobs_on_cpu(coded_data.as_ref(), &group.jobs, &mut output)?; } Ok(output) } @@ -450,7 +456,7 @@ pub(super) struct ClassicCpuDecodeInput<'a> { } pub(super) struct HtCpuDecodeInput<'a> { - pub(super) coded_data: &'a [u8], + pub(super) payload_source: &'a PreparedHtPayloadSource, pub(super) jobs: &'a [J2kHtCleanupBatchJob], pub(super) output_len: usize, } @@ -528,9 +534,10 @@ fn decode_ht_inputs_on_cpu_parallel( HtCodeBlockDecodeWorkspace::default() }, |workspace, (output, input)| { + let coded_data = input.payload_source.materialize_for_cpu()?; if let Some(counters) = profile_counters { decode_prepared_ht_jobs_on_cpu_with_workspace_profiled( - input.coded_data, + coded_data.as_ref(), input.jobs, output, workspace, @@ -538,7 +545,7 @@ fn decode_ht_inputs_on_cpu_parallel( ) } else { decode_prepared_ht_jobs_on_cpu_with_workspace( - input.coded_data, + coded_data.as_ref(), input.jobs, output, workspace, diff --git a/crates/j2k-metal/src/compute/direct_execute.rs b/crates/j2k-metal/src/compute/direct_execute.rs index d3bf455e..33265c5a 100644 --- a/crates/j2k-metal/src/compute/direct_execute.rs +++ b/crates/j2k-metal/src/compute/direct_execute.rs @@ -59,6 +59,8 @@ fn prepare_direct_color_plan_with_tier1_mode( Ok(PreparedDirectColorPlan { dimensions: plan.dimensions, bit_depths: plan.bit_depths, + alpha_bit_depth: None, + signed: false, mct: plan.mct, transform: plan.transform, component_plans, diff --git a/crates/j2k-metal/src/compute/direct_flattened.rs b/crates/j2k-metal/src/compute/direct_flattened.rs index 6fdd75c0..a8413cef 100644 --- a/crates/j2k-metal/src/compute/direct_flattened.rs +++ b/crates/j2k-metal/src/compute/direct_flattened.rs @@ -5,20 +5,21 @@ use std::{sync::Arc, time::Instant}; use j2k_native::HtCodeBlockDecodeWorkspace; use metal::Buffer; +use crate::profile_env::{hybrid_stage_signpost, SIGNPOST_DECODE_HYBRID_CPU_TIER1}; use crate::Error; +use super::abi::{J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob}; +use super::direct_grayscale_execute::{checked_coefficient_len, upload_cpu_decoded_coefficients}; use super::{ - checked_coefficient_len, decode_prepared_classic_jobs_on_cpu_with_scratch, + decode_prepared_classic_jobs_on_cpu_with_scratch, decode_prepared_classic_jobs_on_cpu_with_scratch_profiled, decode_prepared_ht_jobs_on_cpu_with_workspace, - decode_prepared_ht_jobs_on_cpu_with_workspace_profiled, elapsed_us, hybrid_stage_signpost, + decode_prepared_ht_jobs_on_cpu_with_workspace_profiled, elapsed_us, metal_profile_stages_enabled, record_flattened_hybrid_cpu_decode_batch, - record_hybrid_cpu_decode_inputs, record_hybrid_cpu_decode_worker_init, - upload_cpu_decoded_coefficients, ClassicCpuDecodeScratch, CpuTier1DecodeSubstageCounters, - DirectHybridStageTimings, J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob, - MetalRuntime, PreparedDirectColorPlan, PreparedDirectGrayscalePlan, - PreparedDirectGrayscaleStep, HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK, - SIGNPOST_DECODE_HYBRID_CPU_TIER1, + record_hybrid_cpu_decode_inputs, record_hybrid_cpu_decode_worker_init, ClassicCpuDecodeScratch, + CpuTier1DecodeSubstageCounters, DirectHybridStageTimings, MetalRuntime, + PreparedDirectColorPlan, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, + PreparedHtPayloadSource, HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK, }; #[cfg(target_os = "macos")] @@ -37,7 +38,7 @@ enum FlattenedCpuTier1Source<'a> { jobs: &'a [J2kClassicCleanupBatchJob], }, Ht { - coded_data: &'a [u8], + payload_source: &'a PreparedHtPayloadSource, jobs: &'a [J2kHtCleanupBatchJob], }, } @@ -276,7 +277,7 @@ fn collect_flattened_cpu_tier1_bucket_specs( .to_string(), })?; inputs.push(FlattenedCpuTier1Source::Ht { - coded_data: &group.coded_arena.data, + payload_source: &group.payload_source, jobs: &group.jobs, }); } @@ -354,7 +355,7 @@ fn collect_flattened_cpu_tier1_bucket_specs( match &plan.steps[step_idx] { PreparedDirectGrayscaleStep::HtSubBand(other) => { inputs.push(FlattenedCpuTier1Source::Ht { - coded_data: &other.coded_data, + payload_source: &other.payload_source, jobs: &other.jobs, }); } @@ -561,10 +562,14 @@ fn decode_flattened_cpu_tier1_work_item( ) } } - FlattenedCpuTier1Source::Ht { coded_data, jobs } => { + FlattenedCpuTier1Source::Ht { + payload_source, + jobs, + } => { + let coded_data = payload_source.materialize_for_cpu()?; if let Some(counters) = profile_counters { decode_prepared_ht_jobs_on_cpu_with_workspace_profiled( - coded_data, + coded_data.as_ref(), jobs, output, &mut scratch.ht, @@ -572,7 +577,7 @@ fn decode_flattened_cpu_tier1_work_item( ) } else { decode_prepared_ht_jobs_on_cpu_with_workspace( - coded_data, + coded_data.as_ref(), jobs, output, &mut scratch.ht, diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute.rs index 9545379e..51803b82 100644 --- a/crates/j2k-metal/src/compute/direct_grayscale_execute.rs +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute.rs @@ -1,92 +1,75 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::{ - commit_and_wait_metal, completed_command_buffer_gpu_duration, copied_slice_buffer, - decode_prepared_classic_sub_band_group_on_cpu_profile, - decode_prepared_classic_sub_band_on_cpu_profile, - decode_prepared_ht_sub_band_group_on_cpu_profile, decode_prepared_ht_sub_band_on_cpu_profile, +use std::{sync::Arc, time::Instant}; + +use crate::profile_env::{ + hybrid_stage_signpost, label_command_buffer, metal_profile_decode_split_commands_enabled, + SIGNPOST_DECODE_HYBRID_COMMAND_WAIT, +}; + +use super::abi::{J2kGrayStoreParams, J2kStoreParams}; +use super::decode_dispatch::{ dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets, dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets, - dispatch_store_component_buffer_in_encoder_with_offsets, elapsed_us, - emit_direct_hybrid_stage_timings, encode_gray_plane_to_surface_in_encoder, - encode_gray_store_to_surface_in_encoder, + dispatch_store_component_buffer_in_encoder_with_offsets, + encode_gray_store_to_destination_in_encoder, encode_gray_store_to_surface_in_encoder, encode_prepared_classic_sub_band_group_to_buffer_in_encoder, encode_prepared_classic_sub_band_to_buffer_in_encoder, - encode_prepared_direct_color_plan_in_command_buffer, encode_prepared_ht_sub_band_group_to_buffer_in_encoder, - encode_prepared_ht_sub_band_to_buffer_in_encoder, + encode_prepared_ht_sub_band_to_buffer_in_encoder, GrayStoreDestinationRequest, + IdwtSubBandBuffers, SingleIdwtDispatch, +}; +use super::direct_roi::{ + idwt_input_windows_from_slices, prepared_idwt_output_len, prepared_idwt_params, + BandRequiredRegion, +}; +use super::{ + commit_and_wait_metal, completed_command_buffer_gpu_duration, elapsed_us, + emit_direct_hybrid_stage_timings, encode_gray_plane_to_surface_in_encoder, + encode_prepared_direct_color_plan_in_command_buffer, encode_repeated_direct_grayscale_plan_in_command_buffer, encode_repeated_gray_plane_to_surfaces_in_command_buffer, - encode_stacked_direct_component_plane_batch, hybrid_stage_signpost, - idwt_input_windows_from_slices, j2k_scalar_pack_params, label_command_buffer, - lookup_direct_band_slice, lookup_direct_band_slice_entry, - metal_profile_decode_split_commands_enabled, metal_profile_stages_enabled, new_command_buffer, + encode_stacked_direct_component_plane_batch, j2k_scalar_pack_params, lookup_direct_band_slice, + lookup_direct_band_slice_entry, metal_profile_stages_enabled, new_command_buffer, new_compute_command_encoder, prepared_direct_color_plan_supports_runtime, - prepared_idwt_output_len, prepared_idwt_params, record_completed_decode_split_gpu_stages, - recycle_scratch_buffers, size_of, supports_stacked_direct_component_plane_batch, - take_f32_scratch_buffer, try_encode_stacked_mct_rgb8_direct_color_batch, - validate_direct_status, wait_for_completion_metal, with_runtime, with_runtime_for_device, Arc, - BandRequiredRegion, Buffer, CommandBufferRef, CpuTier1DecodeSubstageCounters, + record_completed_decode_split_gpu_stages, recycle_scratch_buffers, retire_direct_status_checks, + supports_stacked_direct_component_plane_batch, take_f32_scratch_buffer, + try_encode_stacked_mct_rgb8_direct_color_batch, wait_for_completion_metal, with_runtime, + with_runtime_for_device, Buffer, CommandBuffer, CommandBufferRef, DecodeHybridSplitCommandBuffers, Device, DirectBandSlice, DirectColorBatchCommandBuffers, DirectColorPlanRequest, DirectHybridStageTimings, DirectScratchBuffer, DirectStatusCheck, - DirectTier1Mode, Error, IdwtSubBandBuffers, Instant, J2kGrayStoreParams, J2kStoreParams, - J2kWaveletTransform, MetalRuntime, PixelFormat, PreparedDirectColorPlan, - PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, RepeatedDirectGrayscalePlanRequest, - SingleIdwtDispatch, StackedDirectColorBatchRequest, StackedDirectComponentPlaneBatchRequest, - Surface, SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD, SIGNPOST_DECODE_HYBRID_COMMAND_WAIT, + DirectStatusRetirementMode, DirectTier1Mode, Error, J2kWaveletTransform, MetalRuntime, + PixelFormat, PreparedDirectColorPlan, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, + RepeatedDirectGrayscalePlanRequest, StackedDirectColorBatchRequest, + StackedDirectComponentPlaneBatchRequest, Surface, }; - mod allocation; +mod color_destination; mod component_plane; +mod destination; +mod destination_index_validation; +mod grayscale_batch; mod single; use self::allocation::{allocate_direct_execution_metadata, DirectExecutionMetadata}; +pub(crate) use self::color_destination::submit_prepared_direct_color_plan_batch_into_group; pub(in crate::compute) use self::component_plane::{ checked_coefficient_len, encode_prepared_direct_component_plane_in_command_buffer, - upload_cpu_decoded_coefficients, DirectComponentPlaneRequest, + encode_prepared_direct_component_plane_in_encoder, upload_cpu_decoded_coefficients, + DirectComponentPlaneRequest, +}; +pub(crate) use self::destination::{ + submit_prepared_direct_grayscale_plan_batch_into_group, DirectDestinationConsumerOrdering, + SubmittedDirectDestination, +}; +pub(crate) use self::grayscale_batch::{ + execute_prepared_direct_grayscale_plan_batch, execute_repeated_prepared_direct_grayscale_plan, }; pub(in crate::compute) use self::single::encode_prepared_direct_grayscale_plan_in_command_buffer; - -#[cfg(target_os = "macos")] -pub(crate) fn execute_repeated_prepared_direct_grayscale_plan( - plan: &PreparedDirectGrayscalePlan, - fmt: PixelFormat, - count: usize, -) -> Result, Error> { - with_runtime(|runtime| { - let DirectExecutionMetadata { - mut retained_buffers, - mut status_checks, - mut scratch_buffers, - } = allocate_direct_execution_metadata( - plan.steps.len(), - 0, - crate::batch_allocation::BatchMetadataBudget::new( - "J2K Metal repeated direct execution resources", - ), - )?; - let command_buffer = new_command_buffer(&runtime.queue)?; - let surfaces = encode_repeated_direct_grayscale_plan_in_command_buffer( - RepeatedDirectGrayscalePlanRequest { - runtime, - command_buffer: &command_buffer, - plan, - fmt, - count, - retained_buffers: &mut retained_buffers, - status_checks: &mut status_checks, - scratch_buffers: &mut scratch_buffers, - }, - )?; - commit_and_wait_metal(&command_buffer)?; - for status_check in status_checks { - validate_direct_status(status_check)?; - } - drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers)?; - Ok(surfaces) - }) -} +pub(in crate::compute) use self::single::{ + encode_prepared_direct_grayscale_plan_into_in_encoder, + DirectGrayscaleDestinationExecutionRequest, +}; #[cfg(target_os = "macos")] pub(crate) fn execute_prepared_direct_grayscale_plan( @@ -115,12 +98,19 @@ pub(crate) fn execute_prepared_direct_grayscale_plan( &mut status_checks, &mut scratch_buffers, )?; - commit_and_wait_metal(&command_buffer)?; - for status_check in status_checks { - validate_direct_status(status_check)?; - } + let completion = commit_and_wait_metal(&command_buffer); + let status_retirement = retire_direct_status_checks( + runtime, + status_checks, + if completion.is_ok() { + DirectStatusRetirementMode::Validate + } else { + DirectStatusRetirementMode::RecycleWithoutRead + }, + ); drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers)?; + let scratch_retirement = recycle_scratch_buffers(runtime, scratch_buffers); + completion.and(status_retirement).and(scratch_retirement)?; Ok(surface) }) } @@ -136,102 +126,6 @@ pub(crate) fn execute_prepared_direct_grayscale_plan_with_device( }) } -#[cfg(target_os = "macos")] -pub(crate) fn execute_prepared_direct_grayscale_plan_batch( - plans: &[Arc], - fmt: PixelFormat, -) -> Result, Error> { - if plans.is_empty() { - return Ok(Vec::new()); - } - - with_runtime(|runtime| { - let step_count = crate::batch_allocation::checked_count_sum( - plans.iter().map(|plan| plan.steps.len()), - "J2K Metal direct grayscale batch step metadata", - )?; - let DirectExecutionMetadata { - mut retained_buffers, - mut status_checks, - mut scratch_buffers, - } = allocate_direct_execution_metadata( - step_count, - 0, - crate::batch_allocation::BatchMetadataBudget::new( - "J2K Metal direct grayscale batch execution resources", - ), - )?; - let command_buffer = new_command_buffer(&runtime.queue)?; - let mut stage_timings = DirectHybridStageTimings::default(); - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "J2K Metal direct grayscale batch execution", - ); - budget.preflight(&[ - crate::batch_allocation::BatchMetadataRequest::of::(plans.len()), - crate::batch_allocation::BatchMetadataRequest::of::<&PreparedDirectGrayscalePlan>( - plans.len(), - ), - ])?; - let mut surfaces = budget.try_vec(plans.len(), "J2K Metal direct grayscale surfaces")?; - let mut component_plan_refs = budget.try_vec( - plans.len(), - "J2K Metal direct grayscale component plan references", - )?; - component_plan_refs.extend(plans.iter().map(Arc::as_ref)); - if plans.len() > 1 && supports_stacked_direct_component_plane_batch(&component_plan_refs) { - let stacked_plane = encode_stacked_direct_component_plane_batch( - StackedDirectComponentPlaneBatchRequest { - runtime, - command_buffers: DirectColorBatchCommandBuffers::single(&command_buffer), - plans: &component_plan_refs, - component_idx: 0, - flattened_cpu_tier1_cache: None, - tier1_mode: DirectTier1Mode::Metal, - stage_timings: &mut stage_timings, - retained_buffers: &mut retained_buffers, - status_checks: &mut status_checks, - scratch_buffers: &mut scratch_buffers, - }, - )?; - let first = plans.first().expect("plans is not empty"); - if stacked_plane.dimensions == first.dimensions && stacked_plane.count == plans.len() { - surfaces = encode_repeated_gray_plane_to_surfaces_in_command_buffer( - runtime, - &command_buffer, - &stacked_plane.buffer, - first.dimensions, - first.bit_depth, - fmt, - plans.len(), - )?; - } - } - - for plan in plans { - if !surfaces.is_empty() { - break; - } - surfaces.push(encode_prepared_direct_grayscale_plan_in_command_buffer( - runtime, - &command_buffer, - plan, - fmt, - &mut retained_buffers, - &mut status_checks, - &mut scratch_buffers, - )?); - } - - commit_and_wait_metal(&command_buffer)?; - for status_check in status_checks { - validate_direct_status(status_check)?; - } - drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers)?; - Ok(surfaces) - }) -} - #[cfg(target_os = "macos")] pub(crate) fn execute_prepared_direct_color_plan( plan: Arc, @@ -379,25 +273,40 @@ pub(super) fn execute_direct_color_plan_batch_with_tier1_options( split_command_buffers.commit_in_order(); let wait_started = Instant::now(); let _wait_signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_COMMAND_WAIT); - wait_for_completion_metal(&split_command_buffers.mct_pack)?; - stage_timings.command_wait += elapsed_us(wait_started); - record_completed_decode_split_gpu_stages( - &mut stage_timings, - &split_command_buffers, - ); - for status_check in status_checks { - validate_direct_status(status_check)?; + let completion = wait_for_completion_metal(&split_command_buffers.mct_pack); + if completion.is_ok() { + stage_timings.command_wait += elapsed_us(wait_started); + record_completed_decode_split_gpu_stages( + &mut stage_timings, + &split_command_buffers, + ); } - emit_direct_hybrid_stage_timings(&stage_timings, fmt, plans.len()); + let status_retirement = retire_direct_status_checks( + runtime, + status_checks, + if completion.is_ok() { + DirectStatusRetirementMode::Validate + } else { + DirectStatusRetirementMode::RecycleWithoutRead + }, + ); drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers)?; + let scratch_retirement = recycle_scratch_buffers(runtime, scratch_buffers); + completion.and(status_retirement).and(scratch_retirement)?; + emit_direct_hybrid_stage_timings(&stage_timings, fmt, plans.len()); return Ok(surfaces); } drop(split_command_buffers); retained_buffers.clear(); - status_checks.clear(); - scratch_buffers.clear(); + let abandoned_status_retirement = retire_direct_status_checks( + runtime, + core::mem::take(&mut status_checks), + DirectStatusRetirementMode::RecycleWithoutRead, + ); + let abandoned_scratch_retirement = + recycle_scratch_buffers(runtime, core::mem::take(&mut scratch_buffers)); + abandoned_status_retirement.and(abandoned_scratch_retirement)?; stage_timings = DirectHybridStageTimings::default(); } @@ -423,23 +332,34 @@ pub(super) fn execute_direct_color_plan_batch_with_tier1_options( command_buffer.commit(); let wait_started = profile_hybrid_stages.then(Instant::now); let _wait_signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_COMMAND_WAIT); - wait_for_completion_metal(&command_buffer)?; - if let Some(started) = wait_started { - stage_timings.command_wait += elapsed_us(started); - } - if profile_hybrid_stages { - if let Some(duration) = completed_command_buffer_gpu_duration(&command_buffer) { - stage_timings.gpu_command += duration.as_micros(); + let completion = wait_for_completion_metal(&command_buffer); + if completion.is_ok() { + if let Some(started) = wait_started { + stage_timings.command_wait += elapsed_us(started); + } + if profile_hybrid_stages { + if let Some(duration) = + completed_command_buffer_gpu_duration(&command_buffer) + { + stage_timings.gpu_command += duration.as_micros(); + } } } - for status_check in status_checks { - validate_direct_status(status_check)?; - } + let status_retirement = retire_direct_status_checks( + runtime, + status_checks, + if completion.is_ok() { + DirectStatusRetirementMode::Validate + } else { + DirectStatusRetirementMode::RecycleWithoutRead + }, + ); + drop(retained_buffers); + let scratch_retirement = recycle_scratch_buffers(runtime, scratch_buffers); + completion.and(status_retirement).and(scratch_retirement)?; if tier1_mode == DirectTier1Mode::CpuUpload { emit_direct_hybrid_stage_timings(&stage_timings, fmt, plans.len()); } - drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers)?; return Ok(surfaces); } } @@ -468,23 +388,32 @@ pub(super) fn execute_direct_color_plan_batch_with_tier1_options( command_buffer.commit(); let wait_started = profile_hybrid_stages.then(Instant::now); let _wait_signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_COMMAND_WAIT); - wait_for_completion_metal(&command_buffer)?; - if let Some(started) = wait_started { - stage_timings.command_wait += elapsed_us(started); - } - if profile_hybrid_stages { - if let Some(duration) = completed_command_buffer_gpu_duration(&command_buffer) { - stage_timings.gpu_command += duration.as_micros(); + let completion = wait_for_completion_metal(&command_buffer); + if completion.is_ok() { + if let Some(started) = wait_started { + stage_timings.command_wait += elapsed_us(started); + } + if profile_hybrid_stages { + if let Some(duration) = completed_command_buffer_gpu_duration(&command_buffer) { + stage_timings.gpu_command += duration.as_micros(); + } } } - for status_check in status_checks { - validate_direct_status(status_check)?; - } + let status_retirement = retire_direct_status_checks( + runtime, + status_checks, + if completion.is_ok() { + DirectStatusRetirementMode::Validate + } else { + DirectStatusRetirementMode::RecycleWithoutRead + }, + ); + drop(retained_buffers); + let scratch_retirement = recycle_scratch_buffers(runtime, scratch_buffers); + completion.and(status_retirement).and(scratch_retirement)?; if tier1_mode == DirectTier1Mode::CpuUpload { emit_direct_hybrid_stage_timings(&stage_timings, fmt, plans.len()); } - drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers)?; Ok(surfaces) }) } diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination.rs new file mode 100644 index 00000000..0af07ce3 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{mem::size_of, sync::Arc}; + +use j2k::BatchLayout; +use j2k_metal_support::{dispatch_3d_pipeline, MetalImageDestination}; + +use crate::compute::abi::J2kNativeColorBatchStoreParams; + +use super::{ + allocation::{allocate_direct_execution_metadata, DirectExecutionMetadata}, + destination::{commit_direct_destination, DirectDestinationConsumerOrdering}, + encode_prepared_direct_component_plane_in_encoder, encode_stacked_direct_component_plane_batch, + new_command_buffer, new_compute_command_encoder, prepared_direct_color_plan_supports_runtime, + supports_stacked_direct_component_plane_batch, Buffer, DirectColorBatchCommandBuffers, + DirectComponentPlaneRequest, DirectHybridStageTimings, DirectTier1Mode, Error, + J2kWaveletTransform, MetalRuntime, PixelFormat, PreparedDirectColorPlan, + PreparedDirectGrayscalePlan, StackedDirectComponentPlaneBatchRequest, + SubmittedDirectDestination, +}; + +mod encoder; +mod store; +mod validation; + +use encoder::{color_component_plan_refs, ColorGroupEncoder}; +use store::encode_exact_native_color_batch_store_in_encoder; +use validation::validate_color_group; + +#[cfg(target_os = "macos")] +pub(crate) fn submit_prepared_direct_color_plan_batch_into_group( + runtime: Arc, + plans: &[Arc], + fmt: PixelFormat, + layout: BatchLayout, + destination: &MetalImageDestination, + source_indices: Option<&[usize]>, + consumer_ordering: DirectDestinationConsumerOrdering, +) -> Result { + let Some(_) = plans.first() else { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal exact RGB destination requires at least one image", + }); + }; + validate_color_group(&runtime, plans, fmt, layout, destination)?; + if source_indices.is_some_and(|indices| indices.len() != plans.len()) { + return Err(Error::MetalStateInvariant { + state: "J2K submitted RGB source attribution", + reason: "source index count does not match prepared plan count", + }); + } + + let step_count = crate::batch_allocation::checked_count_sum( + plans + .iter() + .flat_map(|plan| plan.component_plans.iter()) + .map(|component| component.steps.len()), + "J2K Metal exact RGB destination batch step metadata", + )?; + let mut metadata = allocate_direct_execution_metadata( + step_count, + 0, + crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal exact RGB destination batch execution resources", + ), + )?; + let command_buffer = new_command_buffer(&runtime.queue)?; + let compute_encoder = new_compute_command_encoder(&command_buffer)?; + let component_plan_refs = color_component_plan_refs(plans)?; + let use_stacked = plans.len() > 1 + && component_plan_refs + .iter() + .all(|refs| supports_stacked_direct_component_plane_batch(refs)); + let mut encoder = ColorGroupEncoder { + runtime: &runtime, + command_buffer: &command_buffer, + compute_encoder: &compute_encoder, + plans, + fmt, + layout, + destination, + source_indices, + metadata: &mut metadata, + stage_timings: DirectHybridStageTimings::default(), + }; + let result = if use_stacked { + encoder.encode_coalesced(&component_plan_refs) + } else { + encoder.encode_individually() + }; + compute_encoder.end_encoding(); + result?; + + commit_direct_destination(runtime, command_buffer, metadata, consumer_ordering) +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/encoder.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/encoder.rs new file mode 100644 index 00000000..f9620d6e --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/encoder.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + encode_exact_native_color_batch_store_in_encoder, + encode_prepared_direct_component_plane_in_encoder, encode_stacked_direct_component_plane_batch, + Arc, BatchLayout, Buffer, DirectColorBatchCommandBuffers, DirectComponentPlaneRequest, + DirectExecutionMetadata, DirectHybridStageTimings, DirectTier1Mode, Error, + MetalImageDestination, MetalRuntime, PixelFormat, PreparedDirectColorPlan, + StackedDirectComponentPlaneBatchRequest, +}; + +pub(super) fn color_component_plan_refs( + plans: &[Arc], +) -> Result>, Error> { + let component_count = plans[0].component_plans.len(); + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal stacked exact color plan references", + ); + let mut grouped = budget.try_vec( + component_count, + "J2K Metal stacked exact color component reference groups", + )?; + for component_index in 0..component_count { + let mut refs = budget.try_vec( + plans.len(), + "J2K Metal stacked exact color component references", + )?; + refs.extend( + plans + .iter() + .map(|plan| &plan.component_plans[component_index]), + ); + grouped.push(refs); + } + Ok(grouped) +} + +#[cfg(target_os = "macos")] +pub(super) struct ColorGroupEncoder<'a> { + pub(super) runtime: &'a MetalRuntime, + pub(super) command_buffer: &'a metal::CommandBufferRef, + pub(super) compute_encoder: &'a metal::ComputeCommandEncoderRef, + pub(super) plans: &'a [Arc], + pub(super) fmt: PixelFormat, + pub(super) layout: BatchLayout, + pub(super) destination: &'a MetalImageDestination, + pub(super) source_indices: Option<&'a [usize]>, + pub(super) metadata: &'a mut DirectExecutionMetadata, + pub(super) stage_timings: DirectHybridStageTimings, +} + +#[cfg(target_os = "macos")] +impl ColorGroupEncoder<'_> { + pub(super) fn encode_coalesced( + &mut self, + component_plan_refs: &[Vec<&super::PreparedDirectGrayscalePlan>], + ) -> Result<(), Error> { + let broadcast = self + .plans + .first() + .is_some_and(|first| self.plans.iter().all(|plan| Arc::ptr_eq(plan, first))); + let status_start = self.metadata.status_checks.len(); + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal stacked exact color output planes", + ); + let mut planes = budget.try_vec( + component_plan_refs.len(), + "J2K Metal stacked exact color output planes", + )?; + if broadcast { + self.encode_repeated_planes(&mut planes)?; + } else { + self.encode_distinct_planes(component_plan_refs, &mut planes)?; + } + encode_exact_native_color_batch_store_in_encoder( + self.runtime, + self.compute_encoder, + &planes, + &self.plans[0], + self.fmt, + self.layout, + self.plans.len(), + broadcast, + 0, + self.destination, + )?; + self.remap_coalesced_status(status_start, broadcast) + } + + fn encode_repeated_planes(&mut self, planes: &mut Vec) -> Result<(), Error> { + // Identical prepared inputs share one immutable plan. Decode each + // component once and broadcast those planes in the group final-store. + for component in &self.plans[0].component_plans { + planes.push(encode_prepared_direct_component_plane_in_encoder( + DirectComponentPlaneRequest { + runtime: self.runtime, + command_buffer: self.command_buffer, + plan: component, + tier1_mode: component.tier1_prepare_mode, + stage_timings: &mut self.stage_timings, + retained_buffers: &mut self.metadata.retained_buffers, + status_checks: &mut self.metadata.status_checks, + scratch_buffers: &mut self.metadata.scratch_buffers, + }, + self.compute_encoder, + )?); + } + Ok(()) + } + + fn encode_distinct_planes( + &mut self, + component_plan_refs: &[Vec<&super::PreparedDirectGrayscalePlan>], + planes: &mut Vec, + ) -> Result<(), Error> { + for (component_index, refs) in component_plan_refs.iter().enumerate() { + let stacked = encode_stacked_direct_component_plane_batch( + StackedDirectComponentPlaneBatchRequest { + runtime: self.runtime, + command_buffers: DirectColorBatchCommandBuffers::single(self.command_buffer), + compute_encoder: Some(self.compute_encoder), + plans: refs, + component_idx: component_index, + flattened_cpu_tier1_cache: None, + tier1_mode: DirectTier1Mode::Metal, + stage_timings: &mut self.stage_timings, + retained_buffers: &mut self.metadata.retained_buffers, + status_checks: &mut self.metadata.status_checks, + scratch_buffers: &mut self.metadata.scratch_buffers, + }, + )?; + if stacked.dimensions != self.plans[0].dimensions || stacked.count != self.plans.len() { + return Err(Error::MetalStateInvariant { + state: "J2K Metal stacked exact color destination", + reason: "stacked component output does not match prepared group", + }); + } + planes.push(stacked.buffer); + } + Ok(()) + } + + fn remap_coalesced_status(&mut self, start: usize, broadcast: bool) -> Result<(), Error> { + if broadcast { + let source = self.source_indices.map_or(0, |indices| indices[0]); + for status in &mut self.metadata.status_checks[start..] { + status.remap_ht_source(source)?; + } + } else if let Some(sources) = self.source_indices { + for status in &mut self.metadata.status_checks[start..] { + status.remap_ht_sources(sources)?; + } + } + Ok(()) + } + + pub(super) fn encode_individually(&mut self) -> Result<(), Error> { + for (image_index, plan) in self.plans.iter().enumerate() { + let status_start = self.metadata.status_checks.len(); + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal exact color component plane handles", + ); + let mut planes = budget.try_vec( + plan.component_plans.len(), + "J2K Metal exact color component plane handles", + )?; + for component in &plan.component_plans { + planes.push(encode_prepared_direct_component_plane_in_encoder( + DirectComponentPlaneRequest { + runtime: self.runtime, + command_buffer: self.command_buffer, + plan: component, + tier1_mode: component.tier1_prepare_mode, + stage_timings: &mut self.stage_timings, + retained_buffers: &mut self.metadata.retained_buffers, + status_checks: &mut self.metadata.status_checks, + scratch_buffers: &mut self.metadata.scratch_buffers, + }, + self.compute_encoder, + )?); + } + encode_exact_native_color_batch_store_in_encoder( + self.runtime, + self.compute_encoder, + &planes, + plan, + self.fmt, + self.layout, + 1, + false, + image_index, + self.destination, + )?; + let source = self + .source_indices + .map_or(image_index, |indices| indices[image_index]); + for status in &mut self.metadata.status_checks[status_start..] { + status.remap_ht_source(source)?; + } + } + Ok(()) + } +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/store.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/store.rs new file mode 100644 index 00000000..09694560 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/store.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::destination_index_validation::validate_stacked_color_destination_indices; +use super::{ + dispatch_3d_pipeline, size_of, BatchLayout, Buffer, Error, J2kNativeColorBatchStoreParams, + J2kWaveletTransform, MetalImageDestination, MetalRuntime, PixelFormat, PreparedDirectColorPlan, +}; + +#[cfg(target_os = "macos")] +#[expect( + clippy::too_many_arguments, + reason = "the stacked exact store keeps destination layout and native codec metadata explicit" +)] +#[expect( + clippy::too_many_lines, + reason = "validation, ABI construction, resource binding, and dispatch form one ordered Metal encoder transaction" +)] +pub(super) fn encode_exact_native_color_batch_store_in_encoder( + runtime: &MetalRuntime, + encoder: &metal::ComputeCommandEncoderRef, + planes: &[Buffer], + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, + layout: BatchLayout, + count: usize, + broadcast_planes: bool, + destination_image_index: usize, + destination: &MetalImageDestination, +) -> Result<(), Error> { + let channels = fmt.channels(); + if planes.len() != channels { + return Err(Error::MetalStateInvariant { + state: "J2K Metal stacked exact color store", + reason: "stacked plane count does not match output channels", + }); + } + let destination_layout = destination.layout(); + let bytes_per_sample = fmt.bytes_per_sample(); + let plane_stride = usize::try_from(plan.dimensions.0) + .ok() + .and_then(|width| width.checked_mul(plan.dimensions.1 as usize)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal stacked exact color plane size overflow".to_string(), + })?; + validate_stacked_color_destination_indices( + plan.dimensions, + channels, + layout, + count, + broadcast_planes, + )?; + let destination_end = + destination_image_index + .checked_add(count) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal exact color destination image range overflow".to_string(), + })?; + if destination_end > destination_layout.image_count() { + return Err(Error::MetalStateInvariant { + state: "J2K Metal exact color store", + reason: "destination image range exceeds the validated output group", + }); + } + let destination_offset = destination_layout + .image_offset_bytes(destination_image_index) + .and_then(|offset| destination_layout.byte_offset().checked_add(offset)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal exact color destination offset overflow".to_string(), + })?; + let params = J2kNativeColorBatchStoreParams { + width: plan.dimensions.0, + height: plan.dimensions.1, + plane_stride: if broadcast_planes { + 0 + } else { + u32::try_from(plane_stride).map_err(|_| Error::MetalKernel { + message: "J2K Metal stacked exact color plane stride exceeds u32".to_string(), + })? + }, + output_row_stride: u32::try_from(destination_layout.pitch_bytes() / bytes_per_sample) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal stacked exact color row stride exceeds u32".to_string(), + })?, + output_item_stride: u32::try_from( + destination_layout.image_stride_bytes() / bytes_per_sample, + ) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal stacked exact color image stride exceeds u32".to_string(), + })?, + batch_count: u32::try_from(count).map_err(|_| Error::MetalKernel { + message: "J2K Metal stacked exact color batch count exceeds u32".to_string(), + })?, + layout: match layout { + BatchLayout::Nchw => 0, + BatchLayout::Nhwc => 1, + _ => { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal exact color destination received an unknown batch layout", + }) + } + }, + mct: u32::from(plan.mct), + transform: match plan.transform { + J2kWaveletTransform::Reversible53 => 0, + J2kWaveletTransform::Irreversible97 => 1, + }, + signed: u32::from(plan.signed), + bit_depths: [ + u32::from(plan.bit_depths[0]), + u32::from(plan.bit_depths[1]), + u32::from(plan.bit_depths[2]), + plan.alpha_bit_depth.map_or(0, u32::from), + ], + }; + let pipeline = match fmt { + PixelFormat::Rgb8 => &runtime.store_native_rgb_batch_u8, + PixelFormat::Rgb16 => &runtime.store_native_rgb_batch_u16, + PixelFormat::RgbI16 => &runtime.store_native_rgb_batch_i16, + PixelFormat::Rgba8 => &runtime.store_native_rgba_batch_u8, + PixelFormat::Rgba16 => &runtime.store_native_rgba_batch_u16, + PixelFormat::RgbaI16 => &runtime.store_native_rgba_batch_i16, + _ => return Err(Error::UnsupportedMetalRequest { + reason: + "J2K Metal stacked exact color destination supports native RGB/RGBA integers only", + }), + }; + match planes { + [r, g, b] => encoder.memory_barrier_with_resources(&[r, g, b]), + [r, g, b, a] => encoder.memory_barrier_with_resources(&[r, g, b, a]), + _ => unreachable!("plane count was validated against the native color format"), + } + encoder.set_compute_pipeline_state(pipeline); + for (index, plane) in planes.iter().enumerate() { + encoder.set_buffer(index as u64, Some(plane), 0); + } + // SAFETY: the checked destination owns this exact dense group range until + // the submitted command buffer has completed. + encoder.set_buffer( + channels as u64, + Some(unsafe { destination.raw_buffer() }), + u64::try_from(destination_offset).map_err(|_| Error::MetalKernel { + message: "J2K Metal stacked exact color destination offset exceeds u64".to_string(), + })?, + ); + encoder.set_bytes( + channels as u64 + 1, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_3d_pipeline( + encoder, + pipeline, + (params.width, params.height, params.batch_count), + ); + Ok(()) +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/validation.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/validation.rs new file mode 100644 index 00000000..bf55bd41 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/validation.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + prepared_direct_color_plan_supports_runtime, Arc, BatchLayout, Error, MetalImageDestination, + MetalRuntime, PixelFormat, PreparedDirectColorPlan, +}; + +pub(super) fn validate_color_group( + runtime: &MetalRuntime, + plans: &[Arc], + fmt: PixelFormat, + layout: BatchLayout, + destination: &MetalImageDestination, +) -> Result<(), Error> { + let first = &plans[0]; + if !matches!( + fmt, + PixelFormat::Rgb8 + | PixelFormat::Rgb16 + | PixelFormat::RgbI16 + | PixelFormat::Rgba8 + | PixelFormat::Rgba16 + | PixelFormat::RgbaI16 + ) { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal exact color destination supports native RGB/RGBA integers only", + }); + } + if !matches!(layout, BatchLayout::Nchw | BatchLayout::Nhwc) { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal exact RGB destination received an unknown batch layout", + }); + } + let bit_depths = first + .bit_depths + .iter() + .copied() + .chain(first.alpha_bit_depth); + let bit_depths_supported = match fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 => { + !first.signed && bit_depths.clone().all(|depth| (1..=8).contains(&depth)) + } + PixelFormat::Rgb16 | PixelFormat::Rgba16 => { + !first.signed && bit_depths.clone().all(|depth| (9..=16).contains(&depth)) + } + PixelFormat::RgbI16 | PixelFormat::RgbaI16 => { + first.signed && bit_depths.clone().all(|depth| (1..=16).contains(&depth)) + } + _ => false, + }; + if !bit_depths_supported { + return Err(Error::UnsupportedMetalRequest { + reason: + "J2K Metal exact RGB destination sample width does not match component precision", + }); + } + let expected_components = fmt.channels(); + let expects_alpha = expected_components == 4; + if plans.iter().any(|plan| { + plan.dimensions != first.dimensions + || plan.bit_depths != first.bit_depths + || plan.alpha_bit_depth != first.alpha_bit_depth + || plan.signed != first.signed + || plan.component_plans.len() != expected_components + || plan.alpha_bit_depth.is_some() != expects_alpha + || plan + .component_plans + .iter() + .any(|component| component.dimensions != first.dimensions) + }) { + return Err(Error::UnsupportedMetalRequest { + reason: + "J2K Metal exact color destination requires homogeneous matching component plans", + }); + } + if plans + .iter() + .any(|plan| !prepared_direct_color_plan_supports_runtime(plan, fmt)) + { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal exact color destination contains a code-block plan unsupported by the Metal runtime", + }); + } + destination + .validate_device(&runtime.device) + .and_then(|()| destination.validate_batch(first.dimensions, fmt, plans.len())) + .map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K Metal exact RGB group destination validation failed", + source, + ) + })?; + + let destination_layout = destination.layout(); + let tight_row_bytes = usize::try_from(first.dimensions.0) + .ok() + .and_then(|width| width.checked_mul(fmt.bytes_per_pixel())) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal exact RGB row byte count overflow".to_string(), + })?; + let tight_image_bytes = tight_row_bytes + .checked_mul(first.dimensions.1 as usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal exact RGB image byte count overflow".to_string(), + })?; + if destination_layout.pitch_bytes() != tight_row_bytes + || destination_layout.image_stride_bytes() != tight_image_bytes + { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal exact RGB destination must be one dense contiguous group", + }); + } + Ok(()) +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane.rs index d0f7f7fe..65d9b6e1 100644 --- a/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane.rs +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane.rs @@ -1,43 +1,29 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use j2k_native::J2kDirectStoreStep; -use metal::ComputeCommandEncoderRef; +use metal::{Buffer, CommandBufferRef}; -use super::{ - copied_slice_buffer, decode_prepared_classic_sub_band_group_on_cpu_profile, - decode_prepared_classic_sub_band_on_cpu_profile, - decode_prepared_ht_sub_band_group_on_cpu_profile, decode_prepared_ht_sub_band_on_cpu_profile, - dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets, - dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets, - dispatch_store_component_buffer_in_encoder_with_offsets, elapsed_us, - encode_prepared_classic_sub_band_group_to_buffer_in_encoder, - encode_prepared_classic_sub_band_to_buffer_in_encoder, - encode_prepared_ht_sub_band_group_to_buffer_in_encoder, - encode_prepared_ht_sub_band_to_buffer_in_encoder, hybrid_stage_signpost, - idwt_input_windows_from_slices, lookup_direct_band_slice, lookup_direct_band_slice_entry, - metal_profile_stages_enabled, new_compute_command_encoder, prepared_idwt_output_len, - prepared_idwt_params, size_of, take_f32_scratch_buffer, BandRequiredRegion, Buffer, - CommandBufferRef, CpuTier1DecodeSubstageCounters, DirectBandSlice, DirectHybridStageTimings, - DirectScratchBuffer, DirectStatusCheck, DirectTier1Mode, Error, IdwtSubBandBuffers, Instant, - J2kStoreParams, J2kWaveletTransform, MetalRuntime, PreparedDirectGrayscalePlan, - PreparedDirectGrayscaleStep, SingleIdwtDispatch, SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD, -}; use crate::compute::{ - PreparedClassicSubBand, PreparedClassicSubBandGroup, PreparedDirectIdwt, PreparedHtSubBand, - PreparedHtSubBandGroup, + direct_buffers::copied_slice_buffer, direct_commands::new_compute_command_encoder, + direct_plan_types::PreparedDirectGrayscalePlan, direct_profile::DirectHybridStageTimings, + direct_roi::checked_f32_span, direct_scratch::DirectScratchBuffer, + direct_status::DirectStatusCheck, direct_tier1::DirectTier1Mode, MetalRuntime, +}; +use crate::{ + profile_env::{hybrid_stage_signpost, SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD}, + Error, }; +mod execution; + +pub(in crate::compute) use self::execution::encode_prepared_direct_component_plane_in_encoder; + #[cfg(target_os = "macos")] pub(in crate::compute) fn checked_coefficient_len( width: u32, height: u32, message: &str, ) -> Result { - (width as usize) - .checked_mul(height as usize) - .ok_or_else(|| Error::MetalKernel { - message: message.to_string(), - }) + checked_f32_span(width as usize, height as usize, message).map(|span| span.elements) } #[cfg(target_os = "macos")] @@ -64,330 +50,13 @@ pub(in crate::compute) struct DirectComponentPlaneRequest<'a> { pub(in crate::compute) scratch_buffers: &'a mut Vec, } -struct ComponentPlaneExecution<'a> { - runtime: &'a MetalRuntime, - encoder: &'a ComputeCommandEncoderRef, - tier1_mode: DirectTier1Mode, - profile_stages: bool, - stage_timings: &'a mut DirectHybridStageTimings, - retained_buffers: &'a mut Vec, - status_checks: &'a mut Vec, - scratch_buffers: &'a mut Vec, - bands: Vec, - final_plane: Option, -} - -impl ComponentPlaneExecution<'_> { - fn decode_and_upload_cpu( - &mut self, - decode: impl FnOnce(Option<&CpuTier1DecodeSubstageCounters>) -> Result, Error>, - ) -> Result { - let decode_started = self.profile_stages.then(Instant::now); - let cpu_tier1_counters = self - .profile_stages - .then(CpuTier1DecodeSubstageCounters::default); - let coefficients = decode(cpu_tier1_counters.as_ref())?; - if let Some(started) = decode_started { - self.stage_timings.cpu_tier1 += elapsed_us(started); - } - if let Some(counters) = &cpu_tier1_counters { - counters.add_to_stage_timings(self.stage_timings); - } - let upload_started = self.profile_stages.then(Instant::now); - let buffer = - upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; - if let Some(started) = upload_started { - self.stage_timings.coefficient_upload += elapsed_us(started); - } - Ok(buffer) - } - - fn encode_classic_group(&mut self, group: &PreparedClassicSubBandGroup) -> Result<(), Error> { - let buffer = match self.tier1_mode { - DirectTier1Mode::Metal => { - let output = take_f32_scratch_buffer(self.runtime, group.total_coefficients)?; - let (buffers, status_check) = - encode_prepared_classic_sub_band_group_to_buffer_in_encoder( - self.runtime, - self.encoder, - group, - &output.buffer, - self.scratch_buffers, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - let buffer = output.buffer.clone(); - self.scratch_buffers.push(output); - buffer - } - DirectTier1Mode::CpuUpload => self.decode_and_upload_cpu(|counters| { - decode_prepared_classic_sub_band_group_on_cpu_profile(group, counters) - })?, - }; - for member in &group.members { - self.bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, - }); - } - Ok(()) - } - - fn encode_ht_group(&mut self, group: &PreparedHtSubBandGroup) -> Result<(), Error> { - let buffer = match self.tier1_mode { - DirectTier1Mode::Metal => { - let output = take_f32_scratch_buffer(self.runtime, group.total_coefficients)?; - let (buffers, status_check) = - encode_prepared_ht_sub_band_group_to_buffer_in_encoder( - self.runtime, - self.encoder, - group, - &output.buffer, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - let buffer = output.buffer.clone(); - self.scratch_buffers.push(output); - buffer - } - DirectTier1Mode::CpuUpload => self.decode_and_upload_cpu(|counters| { - decode_prepared_ht_sub_band_group_on_cpu_profile(group, counters) - })?, - }; - for member in &group.members { - self.bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, - }); - } - Ok(()) - } - - fn encode_classic_sub_band(&mut self, sub_band: &PreparedClassicSubBand) -> Result<(), Error> { - let buffer = match self.tier1_mode { - DirectTier1Mode::Metal => { - let output = take_f32_scratch_buffer( - self.runtime, - sub_band.width as usize * sub_band.height as usize, - )?; - let (buffers, status_check) = - encode_prepared_classic_sub_band_to_buffer_in_encoder( - self.runtime, - self.encoder, - sub_band, - &output.buffer, - self.scratch_buffers, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - let buffer = output.buffer.clone(); - self.scratch_buffers.push(output); - buffer - } - DirectTier1Mode::CpuUpload => self.decode_and_upload_cpu(|counters| { - decode_prepared_classic_sub_band_on_cpu_profile(sub_band, counters) - })?, - }; - self.bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer, - offset_bytes: 0, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - Ok(()) - } - - fn encode_ht_sub_band(&mut self, sub_band: &PreparedHtSubBand) -> Result<(), Error> { - let buffer = match self.tier1_mode { - DirectTier1Mode::Metal => { - let output = take_f32_scratch_buffer( - self.runtime, - sub_band.width as usize * sub_band.height as usize, - )?; - let (buffers, status_check) = encode_prepared_ht_sub_band_to_buffer_in_encoder( - self.runtime, - self.encoder, - sub_band, - &output.buffer, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - let buffer = output.buffer.clone(); - self.scratch_buffers.push(output); - buffer - } - DirectTier1Mode::CpuUpload => self.decode_and_upload_cpu(|counters| { - decode_prepared_ht_sub_band_on_cpu_profile(sub_band, counters) - })?, - }; - self.bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer, - offset_bytes: 0, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - Ok(()) - } - - fn encode_idwt(&mut self, idwt: &PreparedDirectIdwt) -> Result<(), Error> { - let ll = lookup_direct_band_slice_entry(&self.bands, idwt.step.ll_band_id, idwt.step.ll)?; - let hl = lookup_direct_band_slice_entry(&self.bands, idwt.step.hl_band_id, idwt.step.hl)?; - let lh = lookup_direct_band_slice_entry(&self.bands, idwt.step.lh_band_id, idwt.step.lh)?; - let hh = lookup_direct_band_slice_entry(&self.bands, idwt.step.hh_band_id, idwt.step.hh)?; - let params = prepared_idwt_params(idwt, idwt_input_windows_from_slices(&ll, &hl, &lh, &hh)); - let output = take_f32_scratch_buffer(self.runtime, prepared_idwt_output_len(idwt))?; - let encode_started = self.profile_stages.then(Instant::now); - let dispatch = SingleIdwtDispatch { - runtime: self.runtime, - sub_bands: IdwtSubBandBuffers { - ll: &ll.buffer, - ll_offset: ll.offset_bytes, - hl: &hl.buffer, - hl_offset: hl.offset_bytes, - lh: &lh.buffer, - lh_offset: lh.offset_bytes, - hh: &hh.buffer, - hh_offset: hh.offset_bytes, - }, - params, - decoded: &output.buffer, - decoded_offset: 0, - }; - match idwt.step.transform { - J2kWaveletTransform::Reversible53 => { - dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( - self.encoder, - dispatch, - ); - } - J2kWaveletTransform::Irreversible97 => { - self.status_checks.push( - dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets( - self.encoder, - dispatch, - )?, - ); - } - } - if let Some(started) = encode_started { - self.stage_timings.metal_idwt_encode += elapsed_us(started); - } - self.bands.push(DirectBandSlice { - band_id: idwt.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: idwt.output_window, - }); - self.scratch_buffers.push(output); - Ok(()) - } - - fn encode_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { - let (input, input_offset) = - lookup_direct_band_slice(&self.bands, store.input_band_id, store.input_rect)?; - let output = take_f32_scratch_buffer( - self.runtime, - store.output_width as usize * store.output_height as usize, - )?; - let encode_started = self.profile_stages.then(Instant::now); - dispatch_store_component_buffer_in_encoder_with_offsets( - self.runtime, - self.encoder, - &input, - input_offset, - &output.buffer, - 0, - J2kStoreParams { - input_width: store.input_rect.width(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - }, - ); - if let Some(started) = encode_started { - self.stage_timings.metal_store_encode += elapsed_us(started); - } - self.final_plane = Some(output.buffer.clone()); - self.scratch_buffers.push(output); - Ok(()) - } - - fn encode_step(&mut self, step: &PreparedDirectGrayscaleStep) -> Result<(), Error> { - match step { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - self.encode_classic_sub_band(sub_band) - } - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => self.encode_ht_sub_band(sub_band), - PreparedDirectGrayscaleStep::Idwt(idwt) => self.encode_idwt(idwt), - PreparedDirectGrayscaleStep::Store(store) => self.encode_store(store), - } - } - - fn finish(self) -> Result { - self.final_plane.ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect component plan did not produce a stored plane".to_string(), - }) - } -} - #[cfg(target_os = "macos")] pub(in crate::compute) fn encode_prepared_direct_component_plane_in_command_buffer( request: DirectComponentPlaneRequest<'_>, ) -> Result { - let DirectComponentPlaneRequest { - runtime, - command_buffer, - plan, - tier1_mode, - stage_timings, - retained_buffers, - status_checks, - scratch_buffers, - } = request; - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "J2K MetalDirect component band metadata", - ); - let bands = budget.try_vec(plan.steps.len(), "J2K MetalDirect component band metadata")?; + let command_buffer = request.command_buffer; let encoder = new_compute_command_encoder(command_buffer)?; - let mut execution = ComponentPlaneExecution { - runtime, - encoder: &encoder, - tier1_mode, - profile_stages: metal_profile_stages_enabled(), - stage_timings, - retained_buffers, - status_checks, - scratch_buffers, - bands, - final_plane: None, - }; - let result = (|| { - let mut step_idx = 0; - while step_idx < plan.steps.len() { - if let Some(group) = plan.classic_group_starting_at(step_idx) { - execution.encode_classic_group(group)?; - step_idx = group.end_step; - continue; - } - if let Some(group) = plan.ht_group_starting_at(step_idx) { - execution.encode_ht_group(group)?; - step_idx = group.end_step; - continue; - } - execution.encode_step(&plan.steps[step_idx])?; - step_idx += 1; - } - execution.finish() - })(); + let result = encode_prepared_direct_component_plane_in_encoder(request, &encoder); encoder.end_encoding(); result } diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution.rs new file mode 100644 index 00000000..35287a70 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution.rs @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::time::Instant; + +use j2k_native::{J2kDirectStoreStep, J2kWaveletTransform}; +use metal::{Buffer, ComputeCommandEncoderRef}; + +use super::{upload_cpu_decoded_coefficients, DirectComponentPlaneRequest}; +use crate::compute::{ + abi::J2kStoreParams, + decode_dispatch::{ + encode_prepared_classic_sub_band_group_to_buffer_in_encoder, + encode_prepared_classic_sub_band_to_buffer_in_encoder, + encode_prepared_ht_sub_band_group_to_buffer_in_encoder, + encode_prepared_ht_sub_band_to_buffer_in_encoder, + idwt::{ + dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets, + dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets, + IdwtSubBandBuffers, SingleIdwtDispatch, + }, + store::dispatch_store_component_buffer_in_encoder_with_offsets, + }, + direct_cpu::{ + decode_prepared_classic_sub_band_group_on_cpu_profile, + decode_prepared_classic_sub_band_on_cpu_profile, + decode_prepared_ht_sub_band_group_on_cpu_profile, + decode_prepared_ht_sub_band_on_cpu_profile, + }, + direct_plan_types::{ + PreparedClassicSubBand, PreparedClassicSubBandGroup, PreparedDirectGrayscaleStep, + PreparedDirectIdwt, PreparedHtSubBand, PreparedHtSubBandGroup, + }, + direct_profile::{elapsed_us, CpuTier1DecodeSubstageCounters, DirectHybridStageTimings}, + direct_roi::{ + checked_f32_span, idwt_input_windows_from_slices, prepared_idwt_output_len, + prepared_idwt_params, BandRequiredRegion, + }, + direct_scratch::{take_f32_scratch_buffer, DirectScratchBuffer}, + direct_stacked_batch::{ + lookup_direct_band_slice, lookup_direct_band_slice_entry, DirectBandSlice, + }, + direct_status::DirectStatusCheck, + direct_tier1::DirectTier1Mode, + MetalRuntime, +}; +use crate::{profile_env::metal_profile_stages_enabled, Error}; + +mod final_plane; + +use self::final_plane::FinalComponentPlane; + +struct ComponentPlaneExecution<'a> { + runtime: &'a MetalRuntime, + encoder: &'a ComputeCommandEncoderRef, + tier1_mode: DirectTier1Mode, + profile_stages: bool, + stage_timings: &'a mut DirectHybridStageTimings, + retained_buffers: &'a mut Vec, + status_checks: &'a mut Vec, + scratch_buffers: &'a mut Vec, + bands: Vec, + final_plane: FinalComponentPlane, +} + +impl ComponentPlaneExecution<'_> { + fn decode_and_upload_cpu( + &mut self, + decode: impl FnOnce(Option<&CpuTier1DecodeSubstageCounters>) -> Result, Error>, + ) -> Result { + let decode_started = self.profile_stages.then(Instant::now); + let cpu_tier1_counters = self + .profile_stages + .then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode(cpu_tier1_counters.as_ref())?; + if let Some(started) = decode_started { + self.stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(self.stage_timings); + } + let upload_started = self.profile_stages.then(Instant::now); + let buffer = + upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; + if let Some(started) = upload_started { + self.stage_timings.coefficient_upload += elapsed_us(started); + } + Ok(buffer) + } + + fn encode_classic_group(&mut self, group: &PreparedClassicSubBandGroup) -> Result<(), Error> { + let output_span = checked_f32_span( + group.total_coefficients, + 1, + "classic J2K MetalDirect grouped coefficients", + )?; + let buffer = match self.tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let (buffers, status_check) = + encode_prepared_classic_sub_band_group_to_buffer_in_encoder( + self.runtime, + self.encoder, + group, + &output.buffer, + self.scratch_buffers, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + let buffer = output.buffer.clone(); + self.scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => self.decode_and_upload_cpu(|counters| { + decode_prepared_classic_sub_band_group_on_cpu_profile(group, counters) + })?, + }; + for member in &group.members { + let offset_bytes = checked_f32_span( + member.offset_elements, + 1, + "classic J2K MetalDirect grouped member offset", + )? + .bytes; + self.bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes, + window: member.window, + }); + } + Ok(()) + } + + fn encode_ht_group(&mut self, group: &PreparedHtSubBandGroup) -> Result<(), Error> { + let output_span = checked_f32_span( + group.total_coefficients, + 1, + "HTJ2K MetalDirect grouped coefficients", + )?; + let buffer = match self.tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let (buffers, status_check) = + encode_prepared_ht_sub_band_group_to_buffer_in_encoder( + self.runtime, + self.encoder, + group, + &output.buffer, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + let buffer = output.buffer.clone(); + self.scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => self.decode_and_upload_cpu(|counters| { + decode_prepared_ht_sub_band_group_on_cpu_profile(group, counters) + })?, + }; + for member in &group.members { + let offset_bytes = checked_f32_span( + member.offset_elements, + 1, + "HTJ2K MetalDirect grouped member offset", + )? + .bytes; + self.bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes, + window: member.window, + }); + } + Ok(()) + } + + fn encode_classic_sub_band(&mut self, sub_band: &PreparedClassicSubBand) -> Result<(), Error> { + let output_span = checked_f32_span( + sub_band.width as usize, + sub_band.height as usize, + "classic J2K MetalDirect sub-band", + )?; + let buffer = match self.tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let (buffers, status_check) = + encode_prepared_classic_sub_band_to_buffer_in_encoder( + self.runtime, + self.encoder, + sub_band, + &output.buffer, + self.scratch_buffers, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + let buffer = output.buffer.clone(); + self.scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => self.decode_and_upload_cpu(|counters| { + decode_prepared_classic_sub_band_on_cpu_profile(sub_band, counters) + })?, + }; + self.bands.push(DirectBandSlice { + band_id: sub_band.band_id, + buffer, + offset_bytes: 0, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + Ok(()) + } + + fn encode_ht_sub_band(&mut self, sub_band: &PreparedHtSubBand) -> Result<(), Error> { + let output_span = checked_f32_span( + sub_band.width as usize, + sub_band.height as usize, + "HTJ2K MetalDirect sub-band", + )?; + let buffer = match self.tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let (buffers, status_check) = encode_prepared_ht_sub_band_to_buffer_in_encoder( + self.runtime, + self.encoder, + sub_band, + &output.buffer, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + let buffer = output.buffer.clone(); + self.scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => self.decode_and_upload_cpu(|counters| { + decode_prepared_ht_sub_band_on_cpu_profile(sub_band, counters) + })?, + }; + self.bands.push(DirectBandSlice { + band_id: sub_band.band_id, + buffer, + offset_bytes: 0, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + Ok(()) + } + + fn encode_idwt(&mut self, idwt: &PreparedDirectIdwt) -> Result<(), Error> { + let ll = lookup_direct_band_slice_entry(&self.bands, idwt.step.ll_band_id, idwt.step.ll)?; + let hl = lookup_direct_band_slice_entry(&self.bands, idwt.step.hl_band_id, idwt.step.hl)?; + let lh = lookup_direct_band_slice_entry(&self.bands, idwt.step.lh_band_id, idwt.step.lh)?; + let hh = lookup_direct_band_slice_entry(&self.bands, idwt.step.hh_band_id, idwt.step.hh)?; + let params = prepared_idwt_params(idwt, idwt_input_windows_from_slices(&ll, &hl, &lh, &hh)); + let output = take_f32_scratch_buffer(self.runtime, prepared_idwt_output_len(idwt)?)?; + let encode_started = self.profile_stages.then(Instant::now); + let dispatch = SingleIdwtDispatch { + runtime: self.runtime, + sub_bands: IdwtSubBandBuffers { + ll: &ll.buffer, + ll_offset: ll.offset_bytes, + hl: &hl.buffer, + hl_offset: hl.offset_bytes, + lh: &lh.buffer, + lh_offset: lh.offset_bytes, + hh: &hh.buffer, + hh_offset: hh.offset_bytes, + }, + params, + decoded: &output.buffer, + decoded_offset: 0, + }; + match idwt.step.transform { + J2kWaveletTransform::Reversible53 => { + dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( + self.encoder, + dispatch, + ); + } + J2kWaveletTransform::Irreversible97 => { + self.status_checks.push( + dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets( + self.encoder, + dispatch, + )?, + ); + } + } + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + if let Some(started) = encode_started { + self.stage_timings.metal_idwt_encode += elapsed_us(started); + } + self.bands.push(DirectBandSlice { + band_id: idwt.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: idwt.output_window, + }); + self.scratch_buffers.push(output); + Ok(()) + } + + fn encode_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { + let (input, input_offset) = + lookup_direct_band_slice(&self.bands, store.input_band_id, store.input_rect)?; + let output_span = checked_f32_span( + store.output_width as usize, + store.output_height as usize, + "J2K MetalDirect stored component plane", + )?; + let dimensions = (store.output_width, store.output_height); + let output = self.final_plane.buffer_for_store( + self.runtime, + dimensions, + output_span.elements, + output_span.bytes, + self.scratch_buffers, + )?; + let encode_started = self.profile_stages.then(Instant::now); + dispatch_store_component_buffer_in_encoder_with_offsets( + self.runtime, + self.encoder, + &input, + input_offset, + &output, + 0, + J2kStoreParams { + input_width: store.input_rect.width(), + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + }, + ); + if let Some(started) = encode_started { + self.stage_timings.metal_store_encode += elapsed_us(started); + } + // Every referenced tile owns an independent coefficient graph. The + // Store is the tile boundary: subsequent tiles write disjoint ranges + // of the same final plane and no longer retain this tile's band views. + self.bands.clear(); + Ok(()) + } + + fn encode_step(&mut self, step: &PreparedDirectGrayscaleStep) -> Result<(), Error> { + match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + self.encode_classic_sub_band(sub_band) + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => self.encode_ht_sub_band(sub_band), + PreparedDirectGrayscaleStep::Idwt(idwt) => self.encode_idwt(idwt), + PreparedDirectGrayscaleStep::Store(store) => self.encode_store(store), + } + } + + fn finish(self) -> Result { + self.final_plane.finish() + } +} + +pub(in crate::compute) fn encode_prepared_direct_component_plane_in_encoder( + request: DirectComponentPlaneRequest<'_>, + encoder: &ComputeCommandEncoderRef, +) -> Result { + let DirectComponentPlaneRequest { + runtime, + command_buffer: _, + plan, + tier1_mode, + stage_timings, + retained_buffers, + status_checks, + scratch_buffers, + } = request; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K MetalDirect component band metadata", + ); + let bands = budget.try_vec(plan.steps.len(), "J2K MetalDirect component band metadata")?; + let mut execution = ComponentPlaneExecution { + runtime, + encoder, + tier1_mode, + profile_stages: metal_profile_stages_enabled(), + stage_timings, + retained_buffers, + status_checks, + scratch_buffers, + bands, + final_plane: FinalComponentPlane::empty(), + }; + let mut step_idx = 0; + while step_idx < plan.steps.len() { + if let Some(group) = plan.classic_group_starting_at(step_idx) { + execution.encode_classic_group(group)?; + step_idx = group.end_step; + continue; + } + if let Some(group) = plan.ht_group_starting_at(step_idx) { + execution.encode_ht_group(group)?; + step_idx = group.end_step; + continue; + } + execution.encode_step(&plan.steps[step_idx])?; + step_idx += 1; + } + execution.finish() +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution/final_plane.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution/final_plane.rs new file mode 100644 index 00000000..3c4d51a9 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution/final_plane.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use metal::Buffer; + +use crate::compute::{ + direct_scratch::{take_f32_scratch_buffer, DirectScratchBuffer}, + MetalRuntime, +}; +use crate::Error; + +pub(super) struct FinalComponentPlane { + retained: Option, +} + +struct RetainedComponentPlane { + buffer: Buffer, + dimensions: (u32, u32), + len: usize, +} + +impl FinalComponentPlane { + pub(super) const fn empty() -> Self { + Self { retained: None } + } + + pub(super) fn buffer_for_store( + &mut self, + runtime: &MetalRuntime, + dimensions: (u32, u32), + len: usize, + bytes: usize, + scratch_buffers: &mut Vec, + ) -> Result { + if let Some(retained) = &self.retained { + validate_later_component_store(retained.dimensions, retained.len, dimensions, len)?; + if retained.buffer.length() < bytes as u64 { + return Err(Error::MetalStateInvariant { + state: "J2K MetalDirect component tile store", + reason: "retained final component plane is smaller than the validated store", + }); + } + return Ok(retained.buffer.clone()); + } + + let output = take_f32_scratch_buffer(runtime, len)?; + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + self.retained = Some(RetainedComponentPlane { + buffer: buffer.clone(), + dimensions, + len, + }); + Ok(buffer) + } + + pub(super) fn finish(self) -> Result { + self.retained + .map(|retained| retained.buffer) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect component plan did not produce a stored plane" + .to_string(), + }) + } +} + +fn validate_later_component_store( + retained_dimensions: (u32, u32), + retained_len: usize, + dimensions: (u32, u32), + len: usize, +) -> Result<(), Error> { + if retained_dimensions != dimensions || retained_len != len { + return Err(Error::MetalStateInvariant { + state: "J2K MetalDirect component tile store", + reason: "later tile store changed the final component plane shape", + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::validate_later_component_store; + use crate::Error; + + #[test] + fn later_component_store_requires_exact_final_plane_geometry() { + assert!(validate_later_component_store((4, 2), 8, (4, 2), 8).is_ok()); + + let changed_shape = validate_later_component_store((4, 2), 8, (2, 4), 8) + .expect_err("the same element count with different dimensions must be rejected"); + assert!(matches!( + changed_shape, + Error::MetalStateInvariant { + state: "J2K MetalDirect component tile store", + reason: "later tile store changed the final component plane shape", + } + )); + + let changed_len = validate_later_component_store((4, 2), 7, (4, 2), 8) + .expect_err("matching dimensions with a different exact length must be rejected"); + assert!(matches!( + changed_len, + Error::MetalStateInvariant { + state: "J2K MetalDirect component tile store", + reason: "later tile store changed the final component plane shape", + } + )); + + let smaller_plane = validate_later_component_store((4, 2), 8, (2, 2), 4) + .expect_err("a smaller later store must not reuse the first final plane"); + assert!(matches!( + smaller_plane, + Error::MetalStateInvariant { + state: "J2K MetalDirect component tile store", + reason: "later tile store changed the final component plane shape", + } + )); + } +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/destination.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/destination.rs new file mode 100644 index 00000000..005e3e68 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/destination.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + allocate_direct_execution_metadata, new_command_buffer, new_compute_command_encoder, Arc, + Error, MetalRuntime, PixelFormat, PreparedDirectGrayscalePlan, +}; +use j2k_metal_support::MetalImageDestination; + +mod group_encode; +mod submission; + +use self::group_encode::GrayscaleGroupEncoder; +pub(super) use self::submission::commit_direct_destination; +pub(crate) use self::submission::{DirectDestinationConsumerOrdering, SubmittedDirectDestination}; + +pub(crate) fn submit_prepared_direct_grayscale_plan_batch_into_group( + runtime: Arc, + plans: &[Arc], + fmt: PixelFormat, + destination: &MetalImageDestination, + source_indices: Option<&[usize]>, + consumer_ordering: DirectDestinationConsumerOrdering, +) -> Result { + let Some(first) = plans.first() else { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal external group destination requires at least one image", + }); + }; + destination + .validate_device(&runtime.device) + .and_then(|()| destination.validate_batch(first.dimensions, fmt, plans.len())) + .map_err(|source| { + crate::error::metal_kernel_support_error( + "J2K Metal direct grayscale group destination validation failed", + source, + ) + })?; + if plans.iter().any(|plan| plan.dimensions != first.dimensions) { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal external group destination requires homogeneous image dimensions", + }); + } + if source_indices.is_some_and(|indices| indices.len() != plans.len()) { + return Err(Error::MetalStateInvariant { + state: "J2K submitted grayscale source attribution", + reason: "source index count does not match prepared plan count", + }); + } + + let step_count = crate::batch_allocation::checked_count_sum( + plans.iter().map(|plan| plan.steps.len()), + "J2K Metal submitted direct grayscale destination batch step metadata", + )?; + let mut metadata = allocate_direct_execution_metadata( + step_count, + 0, + crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal submitted direct grayscale destination batch execution resources", + ), + )?; + let command_buffer = new_command_buffer(&runtime.queue)?; + let compute_encoder = new_compute_command_encoder(&command_buffer)?; + let result = GrayscaleGroupEncoder { + runtime: &runtime, + command_buffer: &command_buffer, + compute_encoder: &compute_encoder, + plans, + fmt, + destination, + source_indices, + metadata: &mut metadata, + } + .encode(); + compute_encoder.end_encoding(); + result?; + commit_direct_destination(runtime, command_buffer, metadata, consumer_ordering) +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/destination/group_encode.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/destination/group_encode.rs new file mode 100644 index 00000000..dfcaad28 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/destination/group_encode.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{mem::size_of, sync::Arc}; + +use super::super::{ + encode_prepared_direct_grayscale_plan_into_in_encoder, + encode_stacked_direct_component_plane_batch, j2k_scalar_pack_params, + supports_stacked_direct_component_plane_batch, DirectColorBatchCommandBuffers, + DirectExecutionMetadata, DirectGrayscaleDestinationExecutionRequest, DirectHybridStageTimings, + DirectTier1Mode, Error, MetalRuntime, PixelFormat, PreparedDirectGrayscalePlan, + StackedDirectComponentPlaneBatchRequest, +}; +use crate::compute::abi::J2kRepeatedGrayStoreParams; +use j2k_metal_support::{dispatch_3d_pipeline, MetalImageDestination}; + +use super::super::destination_index_validation::validate_stacked_grayscale_destination_indices; + +pub(super) struct GrayscaleGroupEncoder<'a> { + pub(super) runtime: &'a MetalRuntime, + pub(super) command_buffer: &'a metal::CommandBufferRef, + pub(super) compute_encoder: &'a metal::ComputeCommandEncoderRef, + pub(super) plans: &'a [Arc], + pub(super) fmt: PixelFormat, + pub(super) destination: &'a MetalImageDestination, + pub(super) source_indices: Option<&'a [usize]>, + pub(super) metadata: &'a mut DirectExecutionMetadata, +} + +impl GrayscaleGroupEncoder<'_> { + pub(super) fn encode(&mut self) -> Result<(), Error> { + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal submitted stacked grayscale plan references", + ); + let mut refs = budget.try_vec( + self.plans.len(), + "J2K Metal submitted stacked grayscale plan reference slots", + )?; + refs.extend(self.plans.iter().map(Arc::as_ref)); + if self.plans.len() > 1 && supports_stacked_direct_component_plane_batch(&refs) { + self.encode_stacked(&refs) + } else { + self.encode_individually() + } + } + + fn encode_stacked(&mut self, plans: &[&PreparedDirectGrayscalePlan]) -> Result<(), Error> { + let first = &self.plans[0]; + let status_start = self.metadata.status_checks.len(); + let mut stage_timings = DirectHybridStageTimings::default(); + let stacked = + encode_stacked_direct_component_plane_batch(StackedDirectComponentPlaneBatchRequest { + runtime: self.runtime, + command_buffers: DirectColorBatchCommandBuffers::single(self.command_buffer), + compute_encoder: Some(self.compute_encoder), + plans, + component_idx: 0, + flattened_cpu_tier1_cache: None, + tier1_mode: DirectTier1Mode::Metal, + stage_timings: &mut stage_timings, + retained_buffers: &mut self.metadata.retained_buffers, + status_checks: &mut self.metadata.status_checks, + scratch_buffers: &mut self.metadata.scratch_buffers, + })?; + if stacked.dimensions != first.dimensions || stacked.count != self.plans.len() { + return Err(Error::MetalStateInvariant { + state: "J2K Metal stacked grayscale destination", + reason: "stacked component output does not match prepared group", + }); + } + encode_stacked_grayscale_destination( + self.runtime, + self.compute_encoder, + &stacked.buffer, + first, + self.fmt, + self.plans.len(), + self.destination, + )?; + if let Some(sources) = self.source_indices { + for status in &mut self.metadata.status_checks[status_start..] { + status.remap_ht_sources(sources)?; + } + } + Ok(()) + } + + fn encode_individually(&mut self) -> Result<(), Error> { + for (index, plan) in self.plans.iter().enumerate() { + let status_start = self.metadata.status_checks.len(); + encode_prepared_direct_grayscale_plan_into_in_encoder( + DirectGrayscaleDestinationExecutionRequest { + runtime: self.runtime, + command_buffer: self.command_buffer, + plan, + fmt: self.fmt, + destination: self.destination, + destination_item_index: index, + retained_buffers: &mut self.metadata.retained_buffers, + status_checks: &mut self.metadata.status_checks, + scratch_buffers: &mut self.metadata.scratch_buffers, + }, + self.compute_encoder, + )?; + let source_index = self.source_indices.map_or(index, |indices| indices[index]); + for status in &mut self.metadata.status_checks[status_start..] { + status.remap_ht_source(source_index)?; + } + } + Ok(()) + } +} + +fn encode_stacked_grayscale_destination( + runtime: &MetalRuntime, + encoder: &metal::ComputeCommandEncoderRef, + plane: &metal::Buffer, + plan: &PreparedDirectGrayscalePlan, + fmt: PixelFormat, + count: usize, + destination: &MetalImageDestination, +) -> Result<(), Error> { + let layout = destination.layout(); + let bytes_per_sample = fmt.bytes_per_sample(); + let tight_row_bytes = usize::try_from(plan.dimensions.0) + .ok() + .and_then(|width| width.checked_mul(bytes_per_sample)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal stacked grayscale row size overflow".to_string(), + })?; + let tight_image_bytes = tight_row_bytes + .checked_mul(plan.dimensions.1 as usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal stacked grayscale image size overflow".to_string(), + })?; + if layout.pitch_bytes() != tight_row_bytes || layout.image_stride_bytes() != tight_image_bytes { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal stacked grayscale destination must be one dense contiguous group", + }); + } + validate_stacked_grayscale_destination_indices(plan.dimensions, count)?; + let scale = j2k_scalar_pack_params(u32::from(plan.bit_depth)); + let max_value = if fmt == PixelFormat::GrayI16 { + let signed_bits = u32::from(plan.bit_depth).clamp(1, 16); + f32::from( + u16::try_from((1_u32 << (signed_bits - 1)) - 1) + .expect("signed bit depth is clamped to 16 bits"), + ) + } else { + scale.max_value + }; + let params = J2kRepeatedGrayStoreParams { + input_width: plan.dimensions.0, + input_height: plan.dimensions.1, + source_x: 0, + source_y: 0, + copy_width: plan.dimensions.0, + copy_height: plan.dimensions.1, + output_width: plan.dimensions.0, + output_height: plan.dimensions.1, + output_x: 0, + output_y: 0, + addend: 0.0, + batch_count: u32::try_from(count).map_err(|_| Error::MetalKernel { + message: "J2K Metal stacked grayscale batch count exceeds u32".to_string(), + })?, + max_value, + u8_scale: scale.u8_scale, + u16_scale: scale.u16_scale, + }; + let pipeline = match fmt { + PixelFormat::Gray8 => &runtime.store_component_repeated_gray_u8, + PixelFormat::Gray16 => &runtime.store_component_repeated_gray_u16, + PixelFormat::GrayI16 => &runtime.store_component_repeated_gray_i16, + _ => { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal stacked grayscale destination supports Gray8/Gray16/GrayI16", + }); + } + }; + encoder.memory_barrier_with_resources(&[plane]); + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(plane), 0); + // SAFETY: the checked destination owns this exact dense group range until + // the submitted command buffer has completed. + encoder.set_buffer( + 1, + Some(unsafe { destination.raw_buffer() }), + u64::try_from(layout.byte_offset()).map_err(|_| Error::MetalKernel { + message: "J2K Metal stacked grayscale destination offset exceeds u64".to_string(), + })?, + ); + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_3d_pipeline( + encoder, + pipeline, + (params.copy_width, params.copy_height, params.batch_count), + ); + Ok(()) +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/destination/submission.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/destination/submission.rs new file mode 100644 index 00000000..7a04f2b0 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/destination/submission.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[cfg(test)] +use super::super::DirectStatusCheck; +use super::super::{ + new_command_buffer, recycle_scratch_buffers, retire_direct_status_checks, + wait_for_completion_metal, Arc, CommandBuffer, DirectExecutionMetadata, + DirectStatusRetirementMode, Error, MetalRuntime, +}; +use metal::{foreign_types::ForeignType, CommandQueue, CommandQueueRef, Event, SharedEvent}; + +pub(crate) enum DirectDestinationConsumerOrdering { + Deferred, + HostCompletionOnly, + Known { + consumer_queue: CommandQueue, + timeline: Arc>, + }, +} + +enum DirectDestinationCompletionDependency { + Deferred(SharedEvent), + Known { _event: Event }, +} + +pub(crate) struct SubmittedDirectDestination { + pub(in crate::compute::direct_grayscale_execute) runtime: Arc, + pub(in crate::compute::direct_grayscale_execute) command_buffer: Option, + pub(in crate::compute::direct_grayscale_execute) metadata: Option, + completion_dependency: Option, + pub(in crate::compute::direct_grayscale_execute) consumer_waits: Vec, + #[cfg(test)] + known_consumer_event_ptr: Option, + #[cfg(test)] + known_consumer_value: Option, +} + +impl SubmittedDirectDestination { + pub(crate) fn enqueue_consumer_wait( + &mut self, + consumer_queue: &CommandQueueRef, + ) -> Result<(), Error> { + let producer_registry_id = self.runtime.device.registry_id(); + let consumer_registry_id = consumer_queue.device().registry_id(); + if producer_registry_id != consumer_registry_id { + return Err(crate::error::metal_kernel_support_error( + "J2K Metal consumer queue belongs to a different device", + j2k_metal_support::MetalSupportError::MetalImageDeviceMismatch { + image_registry_id: producer_registry_id, + requested_registry_id: consumer_registry_id, + }, + )); + } + crate::batch_allocation::try_reserve_for_push( + &mut self.consumer_waits, + "J2K Metal consumer queue completion waits", + )?; + let DirectDestinationCompletionDependency::Deferred(completion_event) = self + .completion_dependency + .as_ref() + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal direct destination consumer ordering", + reason: "known-queue submission has no deferred consumer event bridge", + })? + else { + return Err(Error::MetalStateInvariant { + state: "J2K Metal direct destination consumer ordering", + reason: "known consumer dependency was already registered at submission", + }); + }; + let wait_command = new_command_buffer(consumer_queue)?; + wait_command.encode_wait_for_event(completion_event, 1); + wait_command.commit(); + #[cfg(test)] + crate::compute::test_counters::record_direct_destination_event_wait(); + self.consumer_waits.push(wait_command); + Ok(()) + } + + #[cfg(test)] + pub(crate) fn ordering_diagnostics_for_test(&self) -> (bool, bool, usize) { + let has_event = self.completion_dependency.is_some(); + let has_signal = matches!( + self.completion_dependency, + Some( + DirectDestinationCompletionDependency::Deferred(_) + | DirectDestinationCompletionDependency::Known { .. } + ) + ); + (has_event, has_signal, self.consumer_waits.len()) + } + + #[cfg(test)] + pub(crate) fn known_consumer_timeline_for_test(&self) -> Option<(usize, u64)> { + self.known_consumer_event_ptr.zip(self.known_consumer_value) + } + + #[cfg(test)] + pub(crate) fn in_flight_owner_ptrs_for_test(&self) -> (Vec, Vec) { + let Some(metadata) = self.metadata.as_ref() else { + return (Vec::new(), Vec::new()); + }; + let statuses = metadata + .status_checks + .iter() + .map(|status| match status { + DirectStatusCheck::Classic { buffer, .. } + | DirectStatusCheck::Ht { buffer, .. } + | DirectStatusCheck::Idwt(buffer) + | DirectStatusCheck::Mct(buffer) => buffer.as_ptr() as usize, + }) + .collect(); + let scratch = metadata + .scratch_buffers + .iter() + .map(|owner| owner.buffer.buffer().as_ptr() as usize) + .collect(); + (statuses, scratch) + } + + pub(crate) fn wait(mut self) -> Result<(), Error> { + self.finish() + } + + fn finish(&mut self) -> Result<(), Error> { + let Some(command_buffer) = self.command_buffer.take() else { + return Ok(()); + }; + let completion = wait_for_completion_metal(&command_buffer); + let metadata = self.metadata.take().ok_or(Error::MetalStateInvariant { + state: "J2K Metal direct destination submission", + reason: "committed command buffer lost its retained execution resources", + })?; + let DirectExecutionMetadata { + retained_buffers, + status_checks, + scratch_buffers, + } = metadata; + let status_retirement = retire_direct_status_checks( + &self.runtime, + status_checks, + if completion.is_ok() { + DirectStatusRetirementMode::Validate + } else { + DirectStatusRetirementMode::RecycleWithoutRead + }, + ); + drop(retained_buffers); + let scratch_retirement = recycle_scratch_buffers(&self.runtime, scratch_buffers); + completion.and(status_retirement).and(scratch_retirement) + } +} + +impl Drop for SubmittedDirectDestination { + fn drop(&mut self) { + let _ = self.finish(); + } +} + +pub(in crate::compute::direct_grayscale_execute) fn commit_direct_destination( + runtime: Arc, + command_buffer: CommandBuffer, + metadata: DirectExecutionMetadata, + consumer_ordering: DirectDestinationConsumerOrdering, +) -> Result { + let mut consumer_waits = Vec::new(); + #[cfg(test)] + let mut known_consumer_event_ptr = None; + #[cfg(test)] + let mut known_consumer_value = None; + let completion_dependency = match consumer_ordering { + DirectDestinationConsumerOrdering::Deferred => { + #[cfg(test)] + crate::compute::test_counters::record_direct_destination_event_allocation(); + let event = runtime.device.new_shared_event(); + command_buffer.encode_signal_event(&event, 1); + #[cfg(test)] + crate::compute::test_counters::record_direct_destination_event_signal(); + command_buffer.commit(); + Some(DirectDestinationCompletionDependency::Deferred(event)) + } + DirectDestinationConsumerOrdering::HostCompletionOnly => { + command_buffer.commit(); + None + } + DirectDestinationConsumerOrdering::Known { + consumer_queue, + timeline: _, + } if consumer_queue.as_ptr() == runtime.queue.as_ptr() => { + command_buffer.commit(); + None + } + DirectDestinationConsumerOrdering::Known { + consumer_queue, + timeline, + } => { + let producer_registry_id = runtime.device.registry_id(); + let consumer_registry_id = consumer_queue.device().registry_id(); + if producer_registry_id != consumer_registry_id { + return Err(crate::error::metal_kernel_support_error( + "J2K Metal consumer queue belongs to a different device", + j2k_metal_support::MetalSupportError::MetalImageDeviceMismatch { + image_registry_id: producer_registry_id, + requested_registry_id: consumer_registry_id, + }, + )); + } + crate::batch_allocation::try_reserve_for_push( + &mut consumer_waits, + "J2K Metal known consumer queue completion wait", + )?; + let wait_command = new_command_buffer(&consumer_queue)?; + let mut timeline = timeline.lock().map_err(|_| Error::MetalStatePoisoned { + state: "J2K Metal consumer event timeline", + })?; + let value = timeline + .next_value + .checked_add(1) + .ok_or(Error::MetalStateInvariant { + state: "J2K Metal consumer event timeline", + reason: "event timeline value overflowed", + })?; + let event = timeline + .event + .get_or_insert_with(|| { + #[cfg(test)] + crate::compute::test_counters::record_direct_destination_event_allocation(); + runtime.device.new_event() + }) + .clone(); + command_buffer.encode_signal_event(&event, value); + #[cfg(test)] + crate::compute::test_counters::record_direct_destination_event_signal(); + wait_command.encode_wait_for_event(&event, value); + timeline.next_value = value; + command_buffer.commit(); + wait_command.commit(); + #[cfg(test)] + crate::compute::test_counters::record_direct_destination_event_wait(); + drop(timeline); + consumer_waits.push(wait_command); + #[cfg(test)] + { + known_consumer_event_ptr = Some(event.as_ptr() as usize); + known_consumer_value = Some(value); + } + Some(DirectDestinationCompletionDependency::Known { _event: event }) + } + }; + Ok(SubmittedDirectDestination { + runtime, + command_buffer: Some(command_buffer), + metadata: Some(metadata), + completion_dependency, + consumer_waits, + #[cfg(test)] + known_consumer_event_ptr, + #[cfg(test)] + known_consumer_value, + }) +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/destination_index_validation.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/destination_index_validation.rs new file mode 100644 index 00000000..b0c7cd40 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/destination_index_validation.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::BatchLayout; + +use super::Error; + +pub(super) fn validate_stacked_grayscale_destination_indices( + dimensions: (u32, u32), + count: usize, +) -> Result<(), Error> { + let plane_elements = usize::try_from(dimensions.0) + .ok() + .and_then(|width| width.checked_mul(dimensions.1 as usize)) + .ok_or_else(index_span_overflow)?; + validate_stacked_destination_index_span(plane_elements, count) +} + +pub(super) fn validate_stacked_color_destination_indices( + dimensions: (u32, u32), + channels: usize, + layout: BatchLayout, + count: usize, + broadcast_planes: bool, +) -> Result<(), Error> { + if !matches!(layout, BatchLayout::Nchw | BatchLayout::Nhwc) { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal exact color destination received an unknown batch layout", + }); + } + let plane_elements = usize::try_from(dimensions.0) + .ok() + .and_then(|width| width.checked_mul(dimensions.1 as usize)) + .ok_or_else(index_span_overflow)?; + let output_elements = plane_elements + .checked_mul(channels) + .ok_or_else(index_span_overflow)?; + let input_count = if broadcast_planes { 1 } else { count }; + validate_stacked_destination_index_span(plane_elements, input_count)?; + validate_stacked_destination_index_span(output_elements, count) +} + +fn validate_stacked_destination_index_span( + elements_per_image: usize, + image_count: usize, +) -> Result<(), Error> { + let aggregate_elements = elements_per_image + .checked_mul(image_count) + .ok_or_else(index_span_overflow)?; + u32::try_from(aggregate_elements).map_err(|_| index_span_overflow())?; + Ok(()) +} + +fn index_span_overflow() -> Error { + Error::MetalKernel { + message: "J2K Metal stacked destination element span exceeds u32 shader indexing" + .to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aggregate_span_accepts_u32_max_and_rejects_one_over() { + assert!(validate_stacked_destination_index_span(u32::MAX as usize, 1).is_ok()); + assert!(validate_stacked_destination_index_span(65_536, 65_536).is_err()); + } + + #[test] + fn broadcast_planes_count_one_input_plane_only() { + let plane_elements = 65_536; + let batch_count = 65_536; + assert!(validate_stacked_destination_index_span(plane_elements, 1).is_ok()); + assert!(validate_stacked_destination_index_span(plane_elements, batch_count).is_err()); + } + + #[test] + fn grayscale_plane_len_times_count_must_fit_shader_indexing() { + assert!(validate_stacked_grayscale_destination_indices((u32::MAX, 1), 1).is_ok()); + assert!(validate_stacked_grayscale_destination_indices((65_536, 1), 65_536).is_err()); + } + + #[test] + fn color_channel_spans_are_checked_for_both_tensor_layouts() { + let dimensions = (1_u32 << 29, 1); + for layout in [BatchLayout::Nchw, BatchLayout::Nhwc] { + assert!( + validate_stacked_color_destination_indices(dimensions, 3, layout, 2, false,) + .is_ok() + ); + assert!( + validate_stacked_color_destination_indices(dimensions, 3, layout, 3, false,) + .is_err() + ); + assert!( + validate_stacked_color_destination_indices(dimensions, 4, layout, 1, false,) + .is_ok() + ); + assert!( + validate_stacked_color_destination_indices(dimensions, 4, layout, 2, false,) + .is_err() + ); + } + } +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/grayscale_batch.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/grayscale_batch.rs new file mode 100644 index 00000000..b793f42d --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/grayscale_batch.rs @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{ + allocate_direct_execution_metadata, commit_and_wait_metal, + encode_prepared_direct_grayscale_plan_in_command_buffer, + encode_repeated_direct_grayscale_plan_in_command_buffer, + encode_repeated_gray_plane_to_surfaces_in_command_buffer, + encode_stacked_direct_component_plane_batch, new_command_buffer, recycle_scratch_buffers, + retire_direct_status_checks, supports_stacked_direct_component_plane_batch, with_runtime, Arc, + DirectColorBatchCommandBuffers, DirectExecutionMetadata, DirectHybridStageTimings, + DirectStatusRetirementMode, DirectTier1Mode, Error, PixelFormat, PreparedDirectGrayscalePlan, + RepeatedDirectGrayscalePlanRequest, StackedDirectComponentPlaneBatchRequest, Surface, +}; + +pub(crate) fn execute_repeated_prepared_direct_grayscale_plan( + plan: &PreparedDirectGrayscalePlan, + fmt: PixelFormat, + count: usize, +) -> Result, Error> { + with_runtime(|runtime| { + let DirectExecutionMetadata { + mut retained_buffers, + mut status_checks, + mut scratch_buffers, + } = allocate_direct_execution_metadata( + plan.steps.len(), + 0, + crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal repeated direct execution resources", + ), + )?; + let command_buffer = new_command_buffer(&runtime.queue)?; + let surfaces = encode_repeated_direct_grayscale_plan_in_command_buffer( + RepeatedDirectGrayscalePlanRequest { + runtime, + command_buffer: &command_buffer, + plan, + fmt, + count, + retained_buffers: &mut retained_buffers, + status_checks: &mut status_checks, + scratch_buffers: &mut scratch_buffers, + }, + )?; + let completion = commit_and_wait_metal(&command_buffer); + let status_retirement = retire_direct_status_checks( + runtime, + status_checks, + if completion.is_ok() { + DirectStatusRetirementMode::Validate + } else { + DirectStatusRetirementMode::RecycleWithoutRead + }, + ); + drop(retained_buffers); + let scratch_retirement = recycle_scratch_buffers(runtime, scratch_buffers); + completion.and(status_retirement).and(scratch_retirement)?; + Ok(surfaces) + }) +} + +pub(crate) fn execute_prepared_direct_grayscale_plan_batch( + plans: &[Arc], + fmt: PixelFormat, +) -> Result, Error> { + if plans.is_empty() { + return Ok(Vec::new()); + } + + with_runtime(|runtime| { + let step_count = crate::batch_allocation::checked_count_sum( + plans.iter().map(|plan| plan.steps.len()), + "J2K Metal direct grayscale batch step metadata", + )?; + let DirectExecutionMetadata { + mut retained_buffers, + mut status_checks, + mut scratch_buffers, + } = allocate_direct_execution_metadata( + step_count, + 0, + crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal direct grayscale batch execution resources", + ), + )?; + let command_buffer = new_command_buffer(&runtime.queue)?; + let mut stage_timings = DirectHybridStageTimings::default(); + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K Metal direct grayscale batch execution", + ); + budget.preflight(&[ + crate::batch_allocation::BatchMetadataRequest::of::(plans.len()), + crate::batch_allocation::BatchMetadataRequest::of::<&PreparedDirectGrayscalePlan>( + plans.len(), + ), + ])?; + let mut surfaces = budget.try_vec(plans.len(), "J2K Metal direct grayscale surfaces")?; + let mut component_plan_refs = budget.try_vec( + plans.len(), + "J2K Metal direct grayscale component plan references", + )?; + component_plan_refs.extend(plans.iter().map(Arc::as_ref)); + if plans.len() > 1 && supports_stacked_direct_component_plane_batch(&component_plan_refs) { + let stacked_plane = encode_stacked_direct_component_plane_batch( + StackedDirectComponentPlaneBatchRequest { + runtime, + command_buffers: DirectColorBatchCommandBuffers::single(&command_buffer), + compute_encoder: None, + plans: &component_plan_refs, + component_idx: 0, + flattened_cpu_tier1_cache: None, + tier1_mode: DirectTier1Mode::Metal, + stage_timings: &mut stage_timings, + retained_buffers: &mut retained_buffers, + status_checks: &mut status_checks, + scratch_buffers: &mut scratch_buffers, + }, + )?; + let first = plans.first().expect("plans is not empty"); + if stacked_plane.dimensions == first.dimensions && stacked_plane.count == plans.len() { + surfaces = encode_repeated_gray_plane_to_surfaces_in_command_buffer( + runtime, + &command_buffer, + &stacked_plane.buffer, + first.dimensions, + first.bit_depth, + fmt, + plans.len(), + )?; + } + } + + if surfaces.is_empty() { + for plan in plans { + surfaces.push(encode_prepared_direct_grayscale_plan_in_command_buffer( + runtime, + &command_buffer, + plan, + fmt, + &mut retained_buffers, + &mut status_checks, + &mut scratch_buffers, + )?); + } + } + + let completion = commit_and_wait_metal(&command_buffer); + let status_retirement = retire_direct_status_checks( + runtime, + status_checks, + if completion.is_ok() { + DirectStatusRetirementMode::Validate + } else { + DirectStatusRetirementMode::RecycleWithoutRead + }, + ); + drop(retained_buffers); + let scratch_retirement = recycle_scratch_buffers(runtime, scratch_buffers); + completion.and(status_retirement).and(scratch_retirement)?; + Ok(surfaces) + }) +} diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/single.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/single.rs index 5c7aea3f..cef9ef05 100644 --- a/crates/j2k-metal/src/compute/direct_grayscale_execute/single.rs +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/single.rs @@ -1,267 +1,39 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use j2k_native::J2kDirectStoreStep; -use metal::ComputeCommandEncoderRef; +use j2k_metal_support::MetalImageDestination; use super::{ - dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets, - dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets, - dispatch_store_component_buffer_in_encoder_with_offsets, - encode_gray_plane_to_surface_in_encoder, encode_gray_store_to_surface_in_encoder, - encode_prepared_classic_sub_band_group_to_buffer_in_encoder, - encode_prepared_classic_sub_band_to_buffer_in_encoder, - encode_prepared_ht_sub_band_group_to_buffer_in_encoder, - encode_prepared_ht_sub_band_to_buffer_in_encoder, idwt_input_windows_from_slices, - j2k_scalar_pack_params, lookup_direct_band_slice, lookup_direct_band_slice_entry, - new_compute_command_encoder, prepared_idwt_output_len, prepared_idwt_params, size_of, - take_f32_scratch_buffer, BandRequiredRegion, Buffer, CommandBufferRef, DirectBandSlice, - DirectScratchBuffer, DirectStatusCheck, Error, IdwtSubBandBuffers, J2kGrayStoreParams, - J2kStoreParams, J2kWaveletTransform, MetalRuntime, PixelFormat, PreparedDirectGrayscalePlan, - PreparedDirectGrayscaleStep, SingleIdwtDispatch, Surface, -}; -use crate::compute::{ - PreparedClassicSubBand, PreparedClassicSubBandGroup, PreparedDirectIdwt, PreparedHtSubBand, - PreparedHtSubBandGroup, + new_compute_command_encoder, Buffer, CommandBufferRef, DirectScratchBuffer, DirectStatusCheck, + Error, MetalRuntime, PixelFormat, PreparedDirectGrayscalePlan, Surface, }; -struct SingleGrayscaleExecution<'a> { +mod execution; + +use self::execution::SingleGrayscaleExecution; + +#[cfg(target_os = "macos")] +struct GrayscalePlanExecutionRequest<'a> { runtime: &'a MetalRuntime, - encoder: &'a ComputeCommandEncoderRef, + command_buffer: &'a CommandBufferRef, + plan: &'a PreparedDirectGrayscalePlan, fmt: PixelFormat, - dimensions: (u32, u32), - bit_depth: u8, + destination: Option<(&'a MetalImageDestination, usize)>, retained_buffers: &'a mut Vec, status_checks: &'a mut Vec, scratch_buffers: &'a mut Vec, - bands: Vec, - final_surface: Option, } -impl SingleGrayscaleExecution<'_> { - fn encode_classic_group(&mut self, group: &PreparedClassicSubBandGroup) -> Result<(), Error> { - let output = take_f32_scratch_buffer(self.runtime, group.total_coefficients)?; - let (buffers, status_check) = encode_prepared_classic_sub_band_group_to_buffer_in_encoder( - self.runtime, - self.encoder, - group, - &output.buffer, - self.scratch_buffers, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - for member in &group.members { - self.bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, - }); - } - self.scratch_buffers.push(output); - Ok(()) - } - - fn encode_ht_group(&mut self, group: &PreparedHtSubBandGroup) -> Result<(), Error> { - let output = take_f32_scratch_buffer(self.runtime, group.total_coefficients)?; - let (buffers, status_check) = encode_prepared_ht_sub_band_group_to_buffer_in_encoder( - self.runtime, - self.encoder, - group, - &output.buffer, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - for member in &group.members { - self.bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, - }); - } - self.scratch_buffers.push(output); - Ok(()) - } - - fn encode_classic_sub_band(&mut self, sub_band: &PreparedClassicSubBand) -> Result<(), Error> { - let output = take_f32_scratch_buffer( - self.runtime, - sub_band.width as usize * sub_band.height as usize, - )?; - let (buffers, status_check) = encode_prepared_classic_sub_band_to_buffer_in_encoder( - self.runtime, - self.encoder, - sub_band, - &output.buffer, - self.scratch_buffers, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - self.bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - self.scratch_buffers.push(output); - Ok(()) - } - - fn encode_ht_sub_band(&mut self, sub_band: &PreparedHtSubBand) -> Result<(), Error> { - let output = take_f32_scratch_buffer( - self.runtime, - sub_band.width as usize * sub_band.height as usize, - )?; - let (buffers, status_check) = encode_prepared_ht_sub_band_to_buffer_in_encoder( - self.runtime, - self.encoder, - sub_band, - &output.buffer, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - self.bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - self.scratch_buffers.push(output); - Ok(()) - } - - fn encode_idwt(&mut self, idwt: &PreparedDirectIdwt) -> Result<(), Error> { - let ll = lookup_direct_band_slice_entry(&self.bands, idwt.step.ll_band_id, idwt.step.ll)?; - let hl = lookup_direct_band_slice_entry(&self.bands, idwt.step.hl_band_id, idwt.step.hl)?; - let lh = lookup_direct_band_slice_entry(&self.bands, idwt.step.lh_band_id, idwt.step.lh)?; - let hh = lookup_direct_band_slice_entry(&self.bands, idwt.step.hh_band_id, idwt.step.hh)?; - let params = prepared_idwt_params(idwt, idwt_input_windows_from_slices(&ll, &hl, &lh, &hh)); - let output = take_f32_scratch_buffer(self.runtime, prepared_idwt_output_len(idwt))?; - let dispatch = SingleIdwtDispatch { - runtime: self.runtime, - sub_bands: IdwtSubBandBuffers { - ll: &ll.buffer, - ll_offset: ll.offset_bytes, - hl: &hl.buffer, - hl_offset: hl.offset_bytes, - lh: &lh.buffer, - lh_offset: lh.offset_bytes, - hh: &hh.buffer, - hh_offset: hh.offset_bytes, - }, - params, - decoded: &output.buffer, - decoded_offset: 0, - }; - match idwt.step.transform { - J2kWaveletTransform::Reversible53 => { - dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( - self.encoder, - dispatch, - ); - } - J2kWaveletTransform::Irreversible97 => { - self.status_checks.push( - dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets( - self.encoder, - dispatch, - )?, - ); - } - } - self.bands.push(DirectBandSlice { - band_id: idwt.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: idwt.output_window, - }); - self.scratch_buffers.push(output); - Ok(()) - } - - fn encode_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { - let (input, input_offset) = - lookup_direct_band_slice(&self.bands, store.input_band_id, store.input_rect)?; - if matches!(self.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - let scale = j2k_scalar_pack_params(u32::from(self.bit_depth)); - self.final_surface = Some(encode_gray_store_to_surface_in_encoder( - self.runtime, - self.encoder, - &input, - input_offset, - J2kGrayStoreParams { - input_width: store.input_rect.width(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - max_value: scale.max_value, - u8_scale: scale.u8_scale, - u16_scale: scale.u16_scale, - }, - self.dimensions, - self.fmt, - )?); - } else { - let output = take_f32_scratch_buffer( - self.runtime, - store.output_width as usize * store.output_height as usize, - )?; - let params = J2kStoreParams { - input_width: store.input_rect.width(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - }; - dispatch_store_component_buffer_in_encoder_with_offsets( - self.runtime, - self.encoder, - &input, - input_offset, - &output.buffer, - 0, - params, - ); - self.retained_buffers.push(output.buffer.clone()); - self.final_surface = Some(encode_gray_plane_to_surface_in_encoder( - self.runtime, - self.encoder, - &output.buffer, - self.dimensions, - self.bit_depth, - self.fmt, - )?); - self.scratch_buffers.push(output); - } - Ok(()) - } - - fn encode_step(&mut self, step: &PreparedDirectGrayscaleStep) -> Result<(), Error> { - match step { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - self.encode_classic_sub_band(sub_band) - } - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => self.encode_ht_sub_band(sub_band), - PreparedDirectGrayscaleStep::Idwt(idwt) => self.encode_idwt(idwt), - PreparedDirectGrayscaleStep::Store(store) => self.encode_store(store), - } - } - - fn finish(self) -> Result { - self.final_surface.ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect prepared grayscale plan did not produce a final stored plane" - .to_string(), - }) - } +#[cfg(target_os = "macos")] +pub(in crate::compute) struct DirectGrayscaleDestinationExecutionRequest<'a> { + pub(in crate::compute) runtime: &'a MetalRuntime, + pub(in crate::compute) command_buffer: &'a CommandBufferRef, + pub(in crate::compute) plan: &'a PreparedDirectGrayscalePlan, + pub(in crate::compute) fmt: PixelFormat, + pub(in crate::compute) destination: &'a MetalImageDestination, + pub(in crate::compute) destination_item_index: usize, + pub(in crate::compute) retained_buffers: &'a mut Vec, + pub(in crate::compute) status_checks: &'a mut Vec, + pub(in crate::compute) scratch_buffers: &'a mut Vec, } #[cfg(target_os = "macos")] @@ -274,14 +46,97 @@ pub(in crate::compute) fn encode_prepared_direct_grayscale_plan_in_command_buffe status_checks: &mut Vec, scratch_buffers: &mut Vec, ) -> Result { + encode_prepared_direct_grayscale_plan_in_command_buffer_inner( + GrayscalePlanExecutionRequest { + runtime, + command_buffer, + plan, + fmt, + destination: None, + retained_buffers, + status_checks, + scratch_buffers, + }, + None, + )? + .ok_or_else(|| Error::MetalStateInvariant { + state: "J2K Metal direct grayscale execution", + reason: "surface execution completed without a surface", + }) +} + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn encode_prepared_direct_grayscale_plan_into_in_encoder( + request: DirectGrayscaleDestinationExecutionRequest<'_>, + encoder: &metal::ComputeCommandEncoderRef, +) -> Result<(), Error> { + let DirectGrayscaleDestinationExecutionRequest { + runtime, + command_buffer, + plan, + fmt, + destination, + destination_item_index, + retained_buffers, + status_checks, + scratch_buffers, + } = request; + let surface = encode_prepared_direct_grayscale_plan_in_command_buffer_inner( + GrayscalePlanExecutionRequest { + runtime, + command_buffer, + plan, + fmt, + destination: Some((destination, destination_item_index)), + retained_buffers, + status_checks, + scratch_buffers, + }, + Some(encoder), + )?; + if surface.is_some() { + return Err(Error::MetalStateInvariant { + state: "J2K Metal direct grayscale destination execution", + reason: "destination execution unexpectedly allocated a surface", + }); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn encode_prepared_direct_grayscale_plan_in_command_buffer_inner( + request: GrayscalePlanExecutionRequest<'_>, + existing_encoder: Option<&metal::ComputeCommandEncoderRef>, +) -> Result, Error> { + let GrayscalePlanExecutionRequest { + runtime, + command_buffer, + plan, + fmt, + destination, + retained_buffers, + status_checks, + scratch_buffers, + } = request; let mut budget = crate::batch_allocation::BatchMetadataBudget::new( "J2K MetalDirect grayscale band metadata", ); let bands = budget.try_vec(plan.steps.len(), "J2K MetalDirect grayscale band metadata")?; - let encoder = new_compute_command_encoder(command_buffer)?; + let owned_encoder = if existing_encoder.is_none() { + Some(new_compute_command_encoder(command_buffer)?) + } else { + None + }; + let encoder = existing_encoder.unwrap_or_else(|| { + owned_encoder + .as_ref() + .expect("missing owned grayscale compute encoder") + }); + let (destination, destination_item_index) = + destination.map_or((None, 0), |(destination, index)| (Some(destination), index)); let mut execution = SingleGrayscaleExecution { runtime, - encoder: &encoder, + encoder, fmt, dimensions: plan.dimensions, bit_depth: plan.bit_depth, @@ -290,6 +145,9 @@ pub(in crate::compute) fn encode_prepared_direct_grayscale_plan_in_command_buffe scratch_buffers, bands, final_surface: None, + destination, + destination_item_index, + destination_written: false, }; let result = (|| { let mut step_idx = 0; @@ -309,6 +167,8 @@ pub(in crate::compute) fn encode_prepared_direct_grayscale_plan_in_command_buffe } execution.finish() })(); - encoder.end_encoding(); + if let Some(encoder) = owned_encoder { + encoder.end_encoding(); + } result } diff --git a/crates/j2k-metal/src/compute/direct_grayscale_execute/single/execution.rs b/crates/j2k-metal/src/compute/direct_grayscale_execute/single/execution.rs new file mode 100644 index 00000000..81d7be92 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_grayscale_execute/single/execution.rs @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_metal_support::MetalImageDestination; +use j2k_native::J2kDirectStoreStep; +use metal::ComputeCommandEncoderRef; + +use super::super::{ + dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets, + dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets, + dispatch_store_component_buffer_in_encoder_with_offsets, + encode_gray_plane_to_surface_in_encoder, encode_gray_store_to_destination_in_encoder, + encode_gray_store_to_surface_in_encoder, + encode_prepared_classic_sub_band_group_to_buffer_in_encoder, + encode_prepared_classic_sub_band_to_buffer_in_encoder, + encode_prepared_ht_sub_band_group_to_buffer_in_encoder, + encode_prepared_ht_sub_band_to_buffer_in_encoder, idwt_input_windows_from_slices, + j2k_scalar_pack_params, lookup_direct_band_slice, lookup_direct_band_slice_entry, + prepared_idwt_output_len, prepared_idwt_params, take_f32_scratch_buffer, BandRequiredRegion, + Buffer, DirectBandSlice, DirectScratchBuffer, DirectStatusCheck, Error, + GrayStoreDestinationRequest, IdwtSubBandBuffers, J2kGrayStoreParams, J2kStoreParams, + J2kWaveletTransform, MetalRuntime, PixelFormat, PreparedDirectGrayscaleStep, + SingleIdwtDispatch, Surface, +}; +use crate::compute::{ + direct_roi::checked_f32_span, PreparedClassicSubBand, PreparedClassicSubBandGroup, + PreparedDirectIdwt, PreparedHtSubBand, PreparedHtSubBandGroup, +}; + +pub(super) struct SingleGrayscaleExecution<'a> { + pub(super) runtime: &'a MetalRuntime, + pub(super) encoder: &'a ComputeCommandEncoderRef, + pub(super) fmt: PixelFormat, + pub(super) dimensions: (u32, u32), + pub(super) bit_depth: u8, + pub(super) retained_buffers: &'a mut Vec, + pub(super) status_checks: &'a mut Vec, + pub(super) scratch_buffers: &'a mut Vec, + pub(super) bands: Vec, + pub(super) final_surface: Option, + pub(super) destination: Option<&'a MetalImageDestination>, + pub(super) destination_item_index: usize, + pub(super) destination_written: bool, +} + +impl SingleGrayscaleExecution<'_> { + pub(super) fn encode_classic_group( + &mut self, + group: &PreparedClassicSubBandGroup, + ) -> Result<(), Error> { + let output_span = checked_f32_span( + group.total_coefficients, + 1, + "classic J2K MetalDirect single grouped coefficients", + )?; + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let (buffers, status_check) = encode_prepared_classic_sub_band_group_to_buffer_in_encoder( + self.runtime, + self.encoder, + group, + &output.buffer, + self.scratch_buffers, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + for member in &group.members { + let offset_bytes = checked_f32_span( + member.offset_elements, + 1, + "classic J2K MetalDirect single grouped member offset", + )? + .bytes; + self.bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: output.buffer.clone(), + offset_bytes, + window: member.window, + }); + } + self.scratch_buffers.push(output); + Ok(()) + } + + pub(super) fn encode_ht_group(&mut self, group: &PreparedHtSubBandGroup) -> Result<(), Error> { + let output_span = checked_f32_span( + group.total_coefficients, + 1, + "HTJ2K MetalDirect single grouped coefficients", + )?; + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let (buffers, status_check) = encode_prepared_ht_sub_band_group_to_buffer_in_encoder( + self.runtime, + self.encoder, + group, + &output.buffer, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + for member in &group.members { + let offset_bytes = checked_f32_span( + member.offset_elements, + 1, + "HTJ2K MetalDirect single grouped member offset", + )? + .bytes; + self.bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: output.buffer.clone(), + offset_bytes, + window: member.window, + }); + } + self.scratch_buffers.push(output); + Ok(()) + } + + fn encode_classic_sub_band(&mut self, sub_band: &PreparedClassicSubBand) -> Result<(), Error> { + let output_span = checked_f32_span( + sub_band.width as usize, + sub_band.height as usize, + "classic J2K MetalDirect single sub-band", + )?; + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let (buffers, status_check) = encode_prepared_classic_sub_band_to_buffer_in_encoder( + self.runtime, + self.encoder, + sub_band, + &output.buffer, + self.scratch_buffers, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + self.bands.push(DirectBandSlice { + band_id: sub_band.band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + self.scratch_buffers.push(output); + Ok(()) + } + + fn encode_ht_sub_band(&mut self, sub_band: &PreparedHtSubBand) -> Result<(), Error> { + let output_span = checked_f32_span( + sub_band.width as usize, + sub_band.height as usize, + "HTJ2K MetalDirect single sub-band", + )?; + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let (buffers, status_check) = encode_prepared_ht_sub_band_to_buffer_in_encoder( + self.runtime, + self.encoder, + sub_band, + &output.buffer, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + self.bands.push(DirectBandSlice { + band_id: sub_band.band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + self.scratch_buffers.push(output); + Ok(()) + } + + fn encode_idwt(&mut self, idwt: &PreparedDirectIdwt) -> Result<(), Error> { + let ll = lookup_direct_band_slice_entry(&self.bands, idwt.step.ll_band_id, idwt.step.ll)?; + let hl = lookup_direct_band_slice_entry(&self.bands, idwt.step.hl_band_id, idwt.step.hl)?; + let lh = lookup_direct_band_slice_entry(&self.bands, idwt.step.lh_band_id, idwt.step.lh)?; + let hh = lookup_direct_band_slice_entry(&self.bands, idwt.step.hh_band_id, idwt.step.hh)?; + let params = prepared_idwt_params(idwt, idwt_input_windows_from_slices(&ll, &hl, &lh, &hh)); + let output = take_f32_scratch_buffer(self.runtime, prepared_idwt_output_len(idwt)?)?; + let dispatch = SingleIdwtDispatch { + runtime: self.runtime, + sub_bands: IdwtSubBandBuffers { + ll: &ll.buffer, + ll_offset: ll.offset_bytes, + hl: &hl.buffer, + hl_offset: hl.offset_bytes, + lh: &lh.buffer, + lh_offset: lh.offset_bytes, + hh: &hh.buffer, + hh_offset: hh.offset_bytes, + }, + params, + decoded: &output.buffer, + decoded_offset: 0, + }; + match idwt.step.transform { + J2kWaveletTransform::Reversible53 => { + dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( + self.encoder, + dispatch, + ); + } + J2kWaveletTransform::Irreversible97 => { + self.status_checks.push( + dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets( + self.encoder, + dispatch, + )?, + ); + } + } + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + self.bands.push(DirectBandSlice { + band_id: idwt.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: idwt.output_window, + }); + self.scratch_buffers.push(output); + Ok(()) + } + + fn encode_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { + let (input, input_offset) = + lookup_direct_band_slice(&self.bands, store.input_band_id, store.input_rect)?; + if matches!( + self.fmt, + PixelFormat::Gray8 | PixelFormat::Gray16 | PixelFormat::GrayI16 + ) { + let scale = j2k_scalar_pack_params(u32::from(self.bit_depth)); + let max_value = if self.fmt == PixelFormat::GrayI16 { + let signed_bits = u32::from(self.bit_depth).clamp(1, 16); + let positive_max = u16::try_from((1_u32 << (signed_bits - 1)) - 1) + .expect("signed bit depth is clamped to 16 bits"); + f32::from(positive_max) + } else { + scale.max_value + }; + let params = J2kGrayStoreParams { + input_width: store.input_rect.width(), + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_stride: store.output_width, + output_item_offset: 0, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + max_value, + u8_scale: scale.u8_scale, + u16_scale: scale.u16_scale, + }; + if let Some(destination) = self.destination { + encode_gray_store_to_destination_in_encoder(GrayStoreDestinationRequest { + runtime: self.runtime, + encoder: self.encoder, + input: &input, + input_offset_bytes: input_offset, + params, + dims: self.dimensions, + fmt: self.fmt, + destination, + destination_item_index: self.destination_item_index, + })?; + self.destination_written = true; + } else { + self.final_surface = Some(encode_gray_store_to_surface_in_encoder( + self.runtime, + self.encoder, + &input, + input_offset, + params, + self.dimensions, + self.fmt, + )?); + } + } else { + let output_span = checked_f32_span( + store.output_width as usize, + store.output_height as usize, + "J2K MetalDirect single stored component plane", + )?; + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let params = J2kStoreParams { + input_width: store.input_rect.width(), + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + }; + dispatch_store_component_buffer_in_encoder_with_offsets( + self.runtime, + self.encoder, + &input, + input_offset, + &output.buffer, + 0, + params, + ); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + self.retained_buffers.push(output.buffer.clone()); + self.final_surface = Some(encode_gray_plane_to_surface_in_encoder( + self.runtime, + self.encoder, + &output.buffer, + self.dimensions, + self.bit_depth, + self.fmt, + )?); + self.scratch_buffers.push(output); + } + Ok(()) + } + + pub(super) fn encode_step(&mut self, step: &PreparedDirectGrayscaleStep) -> Result<(), Error> { + match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + self.encode_classic_sub_band(sub_band) + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => self.encode_ht_sub_band(sub_band), + PreparedDirectGrayscaleStep::Idwt(idwt) => self.encode_idwt(idwt), + PreparedDirectGrayscaleStep::Store(store) => self.encode_store(store), + } + } + + pub(super) fn finish(self) -> Result, Error> { + if self.destination.is_some() { + return self + .destination_written + .then_some(None) + .ok_or_else(|| Error::MetalKernel { + message: + "J2K MetalDirect prepared grayscale plan did not write its destination" + .to_string(), + }); + } + self.final_surface + .map(Some) + .ok_or_else(|| Error::MetalKernel { + message: + "J2K MetalDirect prepared grayscale plan did not produce a final stored plane" + .to_string(), + }) + } +} diff --git a/crates/j2k-metal/src/compute/direct_plan_support.rs b/crates/j2k-metal/src/compute/direct_plan_support.rs deleted file mode 100644 index b3ac5f9e..00000000 --- a/crates/j2k-metal/src/compute/direct_plan_support.rs +++ /dev/null @@ -1,261 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use j2k_core::PixelFormat; -use j2k_native::J2kRect; - -use super::{ - DirectTier1Mode, J2kClassicCleanupBatchJob, J2kClassicSegment, J2kDirectStoreStep, - J2kHtCleanupBatchJob, PreparedClassicSubBand, PreparedClassicSubBandGroup, - PreparedDirectColorPlan, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, - PreparedDirectIdwt, PreparedHtSubBand, PreparedHtSubBandGroup, J2K_CLASSIC_MAX_HEIGHT, - J2K_CLASSIC_MAX_WIDTH, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, -}; -use crate::Error; - -#[cfg(test)] -pub(super) fn prepared_direct_color_tier1_input_count(plan: &PreparedDirectColorPlan) -> usize { - plan.component_plans - .iter() - .map(prepared_direct_component_tier1_input_count) - .sum() -} - -#[cfg(test)] -fn prepared_direct_component_tier1_input_count(plan: &PreparedDirectGrayscalePlan) -> usize { - let mut count = 0; - let mut step_idx = 0; - while step_idx < plan.steps.len() { - if let Some(group) = plan.classic_group_starting_at(step_idx) { - count += 1; - step_idx = group.end_step; - continue; - } - if let Some(group) = plan.ht_group_starting_at(step_idx) { - count += 1; - step_idx = group.end_step; - continue; - } - if matches!( - &plan.steps[step_idx], - PreparedDirectGrayscaleStep::ClassicSubBand(_) - | PreparedDirectGrayscaleStep::HtSubBand(_) - ) { - count += 1; - } - step_idx += 1; - } - count -} - -pub(super) fn prepared_direct_color_plan_supports_runtime( - plan: &PreparedDirectColorPlan, - fmt: PixelFormat, -) -> bool { - matches!( - fmt, - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 - ) && plan.component_plans.len() == 3 - && plan - .component_plans - .iter() - .all(prepared_direct_component_plan_supports_runtime) -} - -fn prepared_direct_component_plan_supports_runtime(plan: &PreparedDirectGrayscalePlan) -> bool { - plan.tier1_prepare_mode == DirectTier1Mode::Metal - && plan.steps.iter().all(|step| match step { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => sub_band - .jobs - .iter() - .all(|job| classic_prepared_job_supports_runtime(job, &sub_band.segments)), - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - sub_band.jobs.iter().all(ht_prepared_job_supports_runtime) - } - PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => true, - }) - && plan.classic_groups.iter().all(|group| { - group - .jobs - .iter() - .all(|job| classic_prepared_job_supports_runtime(job, &group.segments)) - }) - && plan - .ht_groups - .iter() - .all(|group| group.jobs.iter().all(ht_prepared_job_supports_runtime)) -} - -fn classic_prepared_job_supports_runtime( - job: &J2kClassicCleanupBatchJob, - segments: &[J2kClassicSegment], -) -> bool { - if job.width == 0 || job.height == 0 { - return true; - } - if job.width > J2K_CLASSIC_MAX_WIDTH || job.height > J2K_CLASSIC_MAX_HEIGHT { - return false; - } - if job.output_stride < job.width { - return false; - } - if job.roi_shift != 0 { - return false; - } - if job.total_bitplanes == 0 || job.total_bitplanes > 31 || job.missing_msbs >= 31 { - return false; - } - let bitplanes = job.total_bitplanes.saturating_sub(job.missing_msbs); - if bitplanes == 0 { - return false; - } - let max_coding_passes = 1 + 3 * (bitplanes - 1); - if job.number_of_coding_passes == 0 || job.number_of_coding_passes > max_coding_passes { - return false; - } - - let start = job.segment_offset as usize; - let count = job.segment_count as usize; - let Some(end) = start.checked_add(count) else { - return false; - }; - if end > segments.len() || count == 0 { - return false; - } - - let uses_bypass = (job.style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) != 0; - let mut expected_start = 0u32; - let mut expected_offset = job.coded_offset; - for segment in &segments[start..end] { - if segment.start_coding_pass != expected_start - || segment.start_coding_pass > segment.end_coding_pass - { - return false; - } - if uses_bypass { - let expected_arithmetic = - segment.start_coding_pass <= 9 || segment.start_coding_pass % 3 == 0; - if (segment.use_arithmetic != 0) != expected_arithmetic { - return false; - } - if segment.use_arithmetic == 0 { - if segment.start_coding_pass % 3 != 1 { - return false; - } - if segment - .end_coding_pass - .saturating_sub(segment.start_coding_pass) - > 2 - { - return false; - } - if (segment.start_coding_pass..segment.end_coding_pass).any(|pass| pass % 3 == 0) { - return false; - } - } - } else if segment.use_arithmetic == 0 { - return false; - } - - let Some(data_end) = segment.data_offset.checked_add(segment.data_length) else { - return false; - }; - if segment.data_offset != expected_offset - || segment.data_offset < job.coded_offset - || data_end > job.coded_offset.saturating_add(job.coded_len) - { - return false; - } - expected_offset = data_end; - expected_start = segment.end_coding_pass; - } - - expected_start == job.number_of_coding_passes - && expected_offset == job.coded_offset.saturating_add(job.coded_len) -} - -pub(super) fn classic_group_shapes_match( - first: &PreparedClassicSubBandGroup, - other: &PreparedClassicSubBandGroup, -) -> bool { - first.end_step == other.end_step - && first.total_coefficients == other.total_coefficients - && first.members.len() == other.members.len() - && first - .members - .iter() - .zip(&other.members) - .all(|(left, right)| left.offset_elements == right.offset_elements) -} - -pub(super) fn ht_group_shapes_match( - first: &PreparedHtSubBandGroup, - other: &PreparedHtSubBandGroup, -) -> bool { - first.end_step == other.end_step - && first.total_coefficients == other.total_coefficients - && first.members.len() == other.members.len() - && first - .members - .iter() - .zip(&other.members) - .all(|(left, right)| left.offset_elements == right.offset_elements) -} - -pub(super) fn classic_sub_band_shapes_match( - first: &PreparedClassicSubBand, - other: &PreparedClassicSubBand, -) -> bool { - first.width == other.width && first.height == other.height -} - -pub(super) fn ht_sub_band_shapes_match( - first: &PreparedHtSubBand, - other: &PreparedHtSubBand, -) -> bool { - first.width == other.width && first.height == other.height -} - -fn rect_shapes_match(first: J2kRect, other: J2kRect) -> bool { - first.x0 == other.x0 && first.y0 == other.y0 && first.x1 == other.x1 && first.y1 == other.y1 -} - -pub(super) fn idwt_shapes_match(first: &PreparedDirectIdwt, other: &PreparedDirectIdwt) -> bool { - first.step.transform == other.step.transform - && rect_shapes_match(first.step.rect, other.step.rect) - && first.output_window.x0 == other.output_window.x0 - && first.output_window.y0 == other.output_window.y0 - && first.output_window.x1 == other.output_window.x1 - && first.output_window.y1 == other.output_window.y1 - && rect_shapes_match(first.step.ll, other.step.ll) - && rect_shapes_match(first.step.hl, other.step.hl) - && rect_shapes_match(first.step.lh, other.step.lh) - && rect_shapes_match(first.step.hh, other.step.hh) -} - -pub(super) fn store_shapes_match(first: &J2kDirectStoreStep, other: &J2kDirectStoreStep) -> bool { - rect_shapes_match(first.input_rect, other.input_rect) - && first.source_x == other.source_x - && first.source_y == other.source_y - && first.copy_width == other.copy_width - && first.copy_height == other.copy_height - && first.output_width == other.output_width - && first.output_height == other.output_height - && first.output_x == other.output_x - && first.output_y == other.output_y - && first.addend.to_bits() == other.addend.to_bits() -} - -pub(super) fn direct_preflight_invariant(message: &'static str) -> Error { - Error::MetalKernel { - message: format!("internal J2K Metal direct preflight error: {message}"), - } -} - -fn ht_prepared_job_supports_runtime(job: &J2kHtCleanupBatchJob) -> bool { - if job.width == 0 || job.height == 0 { - return true; - } - job.roi_shift == 0 - && job.output_stride >= job.width - && crate::ht::supports_metal_ht_geometry(job.width, job.height) -} diff --git a/crates/j2k-metal/src/compute/direct_plan_types.rs b/crates/j2k-metal/src/compute/direct_plan_types.rs index 7f40db34..75ebb1ca 100644 --- a/crates/j2k-metal/src/compute/direct_plan_types.rs +++ b/crates/j2k-metal/src/compute/direct_plan_types.rs @@ -1,17 +1,15 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::sync::Arc; +use std::{borrow::Cow, sync::Arc}; use j2k_native::{ - J2kDirectBandId, J2kDirectIdwtStep, J2kDirectStoreStep, J2kRequiredBandRegion, - J2kWaveletTransform, + HtCodeBlockPayloadRanges, J2kDirectBandId, J2kDirectIdwtStep, J2kDirectStoreStep, + J2kRequiredBandRegion, J2kWaveletTransform, }; use metal::Buffer; -use super::{ - CpuTier1CoefficientCache, DirectTier1Mode, J2kClassicCleanupBatchJob, J2kClassicSegment, - J2kHtCleanupBatchJob, -}; +use super::abi::{J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob}; +use super::{CpuTier1CoefficientCache, DirectTier1Mode}; mod allocation; @@ -28,6 +26,8 @@ pub(crate) struct PreparedDirectGrayscalePlan { pub(crate) struct PreparedDirectColorPlan { pub(super) dimensions: (u32, u32), pub(super) bit_depths: [u8; 3], + pub(super) alpha_bit_depth: Option, + pub(super) signed: bool, pub(super) mct: bool, pub(super) transform: J2kWaveletTransform, pub(super) component_plans: Vec, @@ -82,25 +82,101 @@ pub(super) struct PreparedHtSubBand { pub(super) band_id: J2kDirectBandId, pub(super) width: u32, pub(super) height: u32, - pub(super) coded_data: Vec, - pub(super) coded_buffer: Option, + pub(super) payload_source: PreparedHtPayloadSource, pub(super) jobs: Vec, - pub(super) jobs_buffer: Option, + pub(super) execution_owner: Arc, } -pub(super) struct HtCodedArena { - pub(super) data: Vec, - pub(super) buffer: Buffer, +pub(super) struct PreparedHtExecutionOwner; + +pub(super) enum PreparedHtPayloadSource { + Contiguous(Vec), + Referenced { + input: Arc<[u8]>, + ranges: Vec, + }, +} + +impl PreparedHtPayloadSource { + #[cfg(test)] + pub(super) fn owned_payload_capacity(&self) -> usize { + match self { + Self::Contiguous(data) => data.capacity(), + Self::Referenced { .. } => 0, + } + } + + pub(super) fn contiguous(&self) -> Option<&[u8]> { + match self { + Self::Contiguous(data) => Some(data), + Self::Referenced { .. } => None, + } + } + + pub(super) fn contiguous_mut(&mut self) -> Option<&mut Vec> { + match self { + Self::Contiguous(data) => Some(data), + Self::Referenced { .. } => None, + } + } + + pub(super) fn materialize_for_cpu(&self) -> Result, crate::Error> { + let Self::Referenced { input, ranges } = self else { + return Ok(Cow::Borrowed(self.contiguous().ok_or( + crate::Error::MetalStateInvariant { + state: "HTJ2K Metal prepared payload source", + reason: "contiguous payload source could not expose its bytes", + }, + )?)); + }; + let payload_len = crate::batch_allocation::checked_count_sum( + ranges.iter().flat_map(|payload| { + core::iter::once(payload.cleanup.length) + .chain(payload.refinement.map(|range| range.length)) + }), + "HTJ2K Metal CPU fallback referenced payload", + )?; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "HTJ2K Metal CPU fallback referenced payload", + ); + let mut materialized = + budget.try_vec(payload_len, "HTJ2K Metal CPU fallback referenced payload")?; + for payload in ranges { + append_referenced_payload_range(&mut materialized, input, payload.cleanup)?; + if let Some(refinement) = payload.refinement { + append_referenced_payload_range(&mut materialized, input, refinement)?; + } + } + Ok(Cow::Owned(materialized)) + } +} + +fn append_referenced_payload_range( + destination: &mut Vec, + input: &[u8], + range: j2k_native::J2kCodestreamRange, +) -> Result<(), crate::Error> { + let end = range.end().ok_or_else(|| crate::Error::MetalKernel { + message: "HTJ2K Metal CPU fallback referenced payload range overflow".to_string(), + })?; + let bytes = input + .get(range.offset..end) + .ok_or_else(|| crate::Error::MetalKernel { + message: "HTJ2K Metal CPU fallback referenced payload range exceeds retained input" + .to_string(), + })?; + destination.extend_from_slice(bytes); + Ok(()) } pub(super) struct PreparedHtSubBandGroup { pub(super) start_step: usize, pub(super) end_step: usize, pub(super) total_coefficients: usize, - pub(super) coded_arena: HtCodedArena, + pub(super) payload_source: PreparedHtPayloadSource, pub(super) jobs: Vec, - pub(super) jobs_buffer: Buffer, pub(super) members: Vec, + pub(super) execution_owner: Arc, } pub(super) struct PreparedHtSubBandGroupMember { diff --git a/crates/j2k-metal/src/compute/direct_plan_types/allocation.rs b/crates/j2k-metal/src/compute/direct_plan_types/allocation.rs index 6d7665a4..e9c8703d 100644 --- a/crates/j2k-metal/src/compute/direct_plan_types/allocation.rs +++ b/crates/j2k-metal/src/compute/direct_plan_types/allocation.rs @@ -7,11 +7,11 @@ use core::mem::size_of; use metal::BufferRef; use super::{ - HtCodedArena, PreparedClassicSubBand, PreparedClassicSubBandGroup, PreparedDirectColorPlan, - PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, PreparedHtSubBand, - PreparedHtSubBandGroup, + PreparedClassicSubBand, PreparedClassicSubBandGroup, PreparedDirectColorPlan, + PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, PreparedHtExecutionOwner, + PreparedHtPayloadSource, PreparedHtSubBand, PreparedHtSubBandGroup, }; -use crate::compute::{J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob}; +use crate::compute::abi::{J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob}; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub(crate) struct PreparedPlanRetainedBytes { @@ -48,16 +48,6 @@ impl PreparedPlanRetainedBytes { .ok_or("prepared-plan aggregate device byte overflow")?; Ok(()) } - - fn include_optional_buffer( - &mut self, - buffer: Option<&metal::Buffer>, - ) -> Result<(), &'static str> { - if let Some(buffer) = buffer { - self.include_buffer(buffer)?; - } - Ok(()) - } } impl PreparedDirectGrayscalePlan { @@ -152,27 +142,33 @@ fn include_ht_sub_band( retained: &mut PreparedPlanRetainedBytes, sub_band: &PreparedHtSubBand, ) -> Result<(), &'static str> { - retained.include_host_capacity::(sub_band.coded_data.capacity())?; + include_ht_payload_source(retained, &sub_band.payload_source)?; retained.include_host_capacity::(sub_band.jobs.capacity())?; - retained.include_optional_buffer(sub_band.coded_buffer.as_ref())?; - retained.include_optional_buffer(sub_band.jobs_buffer.as_ref()) + retained.include_host_bytes(size_of::() + 2 * size_of::()) } fn include_ht_group( retained: &mut PreparedPlanRetainedBytes, group: &PreparedHtSubBandGroup, ) -> Result<(), &'static str> { - include_ht_coded_arena(retained, &group.coded_arena)?; + include_ht_payload_source(retained, &group.payload_source)?; retained.include_host_capacity::(group.jobs.capacity())?; retained .include_host_capacity::(group.members.capacity())?; - retained.include_buffer(&group.jobs_buffer) + retained.include_host_bytes(size_of::() + 2 * size_of::()) } -fn include_ht_coded_arena( +fn include_ht_payload_source( retained: &mut PreparedPlanRetainedBytes, - arena: &HtCodedArena, + payload_source: &PreparedHtPayloadSource, ) -> Result<(), &'static str> { - retained.include_host_capacity::(arena.data.capacity())?; - retained.include_buffer(&arena.buffer) + match payload_source { + PreparedHtPayloadSource::Contiguous(data) => { + retained.include_host_capacity::(data.capacity()) + } + PreparedHtPayloadSource::Referenced { ranges, .. } => { + retained + .include_host_capacity::(ranges.capacity()) + } + } } diff --git a/crates/j2k-metal/src/compute/direct_plan_validation.rs b/crates/j2k-metal/src/compute/direct_plan_validation.rs new file mode 100644 index 00000000..d95b4062 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_plan_validation.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +mod runtime; +mod shape; + +pub(super) use self::runtime::prepared_direct_color_plan_supports_runtime; +pub(super) use self::shape::{ + classic_group_shapes_match, classic_sub_band_shapes_match, ht_group_shapes_match, + ht_sub_band_shapes_match, idwt_shapes_match, store_shapes_match, +}; + +use crate::Error; + +pub(super) fn direct_preflight_invariant(message: &'static str) -> Error { + Error::MetalKernel { + message: format!("internal J2K Metal direct preflight error: {message}"), + } +} diff --git a/crates/j2k-metal/src/compute/direct_plan_validation/runtime.rs b/crates/j2k-metal/src/compute/direct_plan_validation/runtime.rs new file mode 100644 index 00000000..11f7a8f9 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_plan_validation/runtime.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_core::PixelFormat; + +use super::super::abi::{ + J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob, J2K_CLASSIC_MAX_HEIGHT, + J2K_CLASSIC_MAX_WIDTH, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, +}; +use super::super::{ + DirectTier1Mode, PreparedDirectColorPlan, PreparedDirectGrayscalePlan, + PreparedDirectGrayscaleStep, +}; + +pub(in crate::compute) fn prepared_direct_color_plan_supports_runtime( + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, +) -> bool { + let supported_format = matches!( + fmt, + PixelFormat::Rgb8 + | PixelFormat::Rgb16 + | PixelFormat::RgbI16 + | PixelFormat::Rgba8 + | PixelFormat::Rgba16 + | PixelFormat::RgbaI16 + ); + supported_format + && plan.component_plans.len() == fmt.channels() + && plan + .component_plans + .iter() + .all(prepared_direct_component_plan_supports_runtime) +} + +fn prepared_direct_component_plan_supports_runtime(plan: &PreparedDirectGrayscalePlan) -> bool { + plan.tier1_prepare_mode == DirectTier1Mode::Metal + && plan.steps.iter().all(|step| match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => sub_band + .jobs + .iter() + .all(|job| classic_prepared_job_supports_runtime(job, &sub_band.segments)), + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + sub_band.jobs.iter().all(ht_prepared_job_supports_runtime) + } + PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => true, + }) + && plan.classic_groups.iter().all(|group| { + group + .jobs + .iter() + .all(|job| classic_prepared_job_supports_runtime(job, &group.segments)) + }) + && plan + .ht_groups + .iter() + .all(|group| group.jobs.iter().all(ht_prepared_job_supports_runtime)) +} + +fn classic_prepared_job_supports_runtime( + job: &J2kClassicCleanupBatchJob, + segments: &[J2kClassicSegment], +) -> bool { + if job.width == 0 || job.height == 0 { + return true; + } + if job.width > J2K_CLASSIC_MAX_WIDTH || job.height > J2K_CLASSIC_MAX_HEIGHT { + return false; + } + if job.output_stride < job.width || job.roi_shift != 0 { + return false; + } + if job.total_bitplanes == 0 || job.total_bitplanes > 31 || job.missing_msbs >= 31 { + return false; + } + let bitplanes = job.total_bitplanes.saturating_sub(job.missing_msbs); + if bitplanes == 0 { + return false; + } + + let start = job.segment_offset as usize; + let count = job.segment_count as usize; + let Some(end) = start.checked_add(count) else { + return false; + }; + if end > segments.len() { + return false; + } + let job_segments = &segments[start..end]; + if job.coded_len == 0 || job.number_of_coding_passes == 0 { + return job.coded_len == 0 + && job.number_of_coding_passes == 0 + && job_segments.iter().all(|segment| { + segment.data_offset == job.coded_offset + && segment.data_length == 0 + && segment.start_coding_pass == 0 + && segment.end_coding_pass == 0 + }); + } + + let max_coding_passes = 1 + 3 * (bitplanes - 1); + if job.number_of_coding_passes > max_coding_passes || count == 0 { + return false; + } + + let uses_bypass = (job.style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) != 0; + let mut expected_start = 0u32; + let mut expected_offset = job.coded_offset; + for segment in job_segments { + if segment.start_coding_pass != expected_start + || segment.start_coding_pass > segment.end_coding_pass + { + return false; + } + if uses_bypass { + let expected_arithmetic = + segment.start_coding_pass <= 9 || segment.start_coding_pass % 3 == 0; + if (segment.use_arithmetic != 0) != expected_arithmetic { + return false; + } + if segment.use_arithmetic == 0 { + if segment.start_coding_pass % 3 != 1 { + return false; + } + if segment + .end_coding_pass + .saturating_sub(segment.start_coding_pass) + > 2 + { + return false; + } + if (segment.start_coding_pass..segment.end_coding_pass).any(|pass| pass % 3 == 0) { + return false; + } + } + } else if segment.use_arithmetic == 0 { + return false; + } + + let Some(data_end) = segment.data_offset.checked_add(segment.data_length) else { + return false; + }; + if segment.data_offset != expected_offset + || segment.data_offset < job.coded_offset + || data_end > job.coded_offset.saturating_add(job.coded_len) + { + return false; + } + expected_offset = data_end; + expected_start = segment.end_coding_pass; + } + + expected_start == job.number_of_coding_passes + && expected_offset == job.coded_offset.saturating_add(job.coded_len) +} + +fn ht_prepared_job_supports_runtime(job: &J2kHtCleanupBatchJob) -> bool { + if job.width == 0 || job.height == 0 { + return true; + } + job.roi_shift == 0 + && job.output_stride >= job.width + && crate::ht::supports_metal_ht_geometry(job.width, job.height) +} + +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::*; + + fn valid_classic_job() -> J2kClassicCleanupBatchJob { + J2kClassicCleanupBatchJob { + coded_offset: 7, + coded_len: 3, + segment_offset: 0, + segment_count: 1, + width: 1, + height: 1, + output_stride: 1, + output_offset: 0, + missing_msbs: 0, + total_bitplanes: 1, + roi_shift: 0, + number_of_coding_passes: 1, + sub_band_type: 0, + style_flags: 0, + strict: 1, + dequantization_step: 1.0, + } + } + + fn valid_classic_segment() -> J2kClassicSegment { + J2kClassicSegment { + data_offset: 7, + data_length: 3, + start_coding_pass: 0, + end_coding_pass: 1, + use_arithmetic: 1, + } + } + + #[test] + fn classic_runtime_preflight_rejects_unimplemented_roi_shift_and_inconsistent_empty_job() { + let segment = valid_classic_segment(); + let mut job = valid_classic_job(); + assert!(classic_prepared_job_supports_runtime(&job, &[segment])); + + job.roi_shift = 1; + assert!(!classic_prepared_job_supports_runtime(&job, &[segment])); + + job.roi_shift = 0; + job.number_of_coding_passes = 0; + assert!(!classic_prepared_job_supports_runtime(&job, &[segment])); + + job.coded_len = 0; + let empty_segment = J2kClassicSegment { + data_offset: job.coded_offset, + data_length: 0, + start_coding_pass: 0, + end_coding_pass: 0, + use_arithmetic: 1, + }; + assert!(classic_prepared_job_supports_runtime( + &job, + &[empty_segment] + )); + } +} diff --git a/crates/j2k-metal/src/compute/direct_plan_validation/shape.rs b/crates/j2k-metal/src/compute/direct_plan_validation/shape.rs new file mode 100644 index 00000000..fd77c519 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_plan_validation/shape.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_native::J2kRect; + +use super::super::{ + J2kDirectStoreStep, PreparedClassicSubBand, PreparedClassicSubBandGroup, PreparedDirectIdwt, + PreparedHtSubBand, PreparedHtSubBandGroup, +}; + +pub(in crate::compute) fn classic_group_shapes_match( + first: &PreparedClassicSubBandGroup, + other: &PreparedClassicSubBandGroup, +) -> bool { + first.end_step == other.end_step + && first.total_coefficients == other.total_coefficients + && first.members.len() == other.members.len() + && first + .members + .iter() + .zip(&other.members) + .all(|(left, right)| left.offset_elements == right.offset_elements) +} + +pub(in crate::compute) fn ht_group_shapes_match( + first: &PreparedHtSubBandGroup, + other: &PreparedHtSubBandGroup, +) -> bool { + first.end_step == other.end_step + && first.total_coefficients == other.total_coefficients + && first.members.len() == other.members.len() + && first + .members + .iter() + .zip(&other.members) + .all(|(left, right)| left.offset_elements == right.offset_elements) +} + +pub(in crate::compute) fn classic_sub_band_shapes_match( + first: &PreparedClassicSubBand, + other: &PreparedClassicSubBand, +) -> bool { + first.width == other.width && first.height == other.height +} + +pub(in crate::compute) fn ht_sub_band_shapes_match( + first: &PreparedHtSubBand, + other: &PreparedHtSubBand, +) -> bool { + first.width == other.width && first.height == other.height +} + +fn rect_shapes_match(first: J2kRect, other: J2kRect) -> bool { + first.x0 == other.x0 && first.y0 == other.y0 && first.x1 == other.x1 && first.y1 == other.y1 +} + +pub(in crate::compute) fn idwt_shapes_match( + first: &PreparedDirectIdwt, + other: &PreparedDirectIdwt, +) -> bool { + first.step.transform == other.step.transform + && rect_shapes_match(first.step.rect, other.step.rect) + && first.output_window.x0 == other.output_window.x0 + && first.output_window.y0 == other.output_window.y0 + && first.output_window.x1 == other.output_window.x1 + && first.output_window.y1 == other.output_window.y1 + && rect_shapes_match(first.step.ll, other.step.ll) + && rect_shapes_match(first.step.hl, other.step.hl) + && rect_shapes_match(first.step.lh, other.step.lh) + && rect_shapes_match(first.step.hh, other.step.hh) +} + +pub(in crate::compute) fn store_shapes_match( + first: &J2kDirectStoreStep, + other: &J2kDirectStoreStep, +) -> bool { + rect_shapes_match(first.input_rect, other.input_rect) + && first.source_x == other.source_x + && first.source_y == other.source_y + && first.copy_width == other.copy_width + && first.copy_height == other.copy_height + && first.output_width == other.output_width + && first.output_height == other.output_height + && first.output_x == other.output_x + && first.output_y == other.output_y + && first.addend.to_bits() == other.addend.to_bits() +} diff --git a/crates/j2k-metal/src/compute/direct_plane_pack.rs b/crates/j2k-metal/src/compute/direct_plane_pack.rs index 63add0e3..3ed4018e 100644 --- a/crates/j2k-metal/src/compute/direct_plane_pack.rs +++ b/crates/j2k-metal/src/compute/direct_plane_pack.rs @@ -1,14 +1,22 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use std::{mem::size_of, sync::Arc}; + +use j2k_metal_support::{dispatch_2d_pipeline, dispatch_3d_pipeline}; + +use crate::profile_env::{ + hybrid_stage_signpost, label_compute_encoder, SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE, +}; + +use super::abi::{J2kBatchedMctRgb8PackParams, J2kMctRgb8PackParams, J2kPackParams}; +use super::direct_surface_pack::checked_metal_surface_len; use super::{ - checked_metal_surface_len, commit_and_wait_metal, copy_plane_samples, dispatch_2d_pipeline, - dispatch_3d_pipeline, hybrid_stage_signpost, j2k_pack_scale_arrays, j2k_u32_param, - label_compute_encoder, metal_profile_stages_enabled, new_blit_command_encoder, - new_command_buffer, new_compute_command_encoder, new_shared_buffer, output_shape_for, - record_hybrid_repeated_output_blit, signed_sample_bias, size_of, Arc, Buffer, CommandBufferRef, - Device, Error, J2kBatchedMctRgb8PackParams, J2kMctRgb8PackParams, J2kPackParams, - J2kWaveletTransform, MetalRuntime, NativeColorSpace, NativeDecodedComponents, PixelFormat, - PreparedDirectColorPlan, Rect, Surface, SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE, + commit_and_wait_metal, copy_plane_samples, j2k_pack_scale_arrays, j2k_u32_param, + metal_profile_stages_enabled, new_blit_command_encoder, new_command_buffer, + new_compute_command_encoder, new_shared_buffer, output_shape_for, + record_hybrid_repeated_output_blit, signed_sample_bias, Buffer, CommandBufferRef, Device, + Error, J2kWaveletTransform, MetalRuntime, NativeColorSpace, NativeDecodedComponents, + PixelFormat, PreparedDirectColorPlan, Rect, Surface, }; #[cfg(target_os = "macos")] diff --git a/crates/j2k-metal/src/compute/direct_prepare.rs b/crates/j2k-metal/src/compute/direct_prepare.rs index 7a597d47..e89bdc53 100644 --- a/crates/j2k-metal/src/compute/direct_prepare.rs +++ b/crates/j2k-metal/src/compute/direct_prepare.rs @@ -1,573 +1,47 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use std::sync::Arc; + +use super::abi::{J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob}; +use super::direct_roi::BandRequiredRegion; use super::{ - classic_style_flags, prepare_direct_tier1_input_buffer, with_runtime, Arc, BandRequiredRegion, - Buffer, CpuTier1CoefficientCache, DirectTier1Mode, Error, HtCodedArena, - J2kClassicCleanupBatchJob, J2kClassicSegment, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, - J2kHtCleanupBatchJob, PreparedClassicSubBand, PreparedClassicSubBandGroup, - PreparedClassicSubBandGroupMember, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, - PreparedDirectIdwt, PreparedHtSubBand, PreparedHtSubBandGroup, PreparedHtSubBandGroupMember, + classic_style_flags, prepare_direct_tier1_input_buffer, with_runtime, CpuTier1CoefficientCache, + DirectTier1Mode, Error, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, PreparedClassicSubBand, + PreparedClassicSubBandGroup, PreparedClassicSubBandGroupMember, PreparedDirectColorPlan, + PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, PreparedDirectIdwt, + PreparedHtPayloadSource, PreparedHtSubBand, PreparedHtSubBandGroup, + PreparedHtSubBandGroupMember, }; - -#[cfg(target_os = "macos")] -struct ClassicGroupOwners { - members: Vec, - jobs: Vec, - segments: Vec, - coded_data: Vec, -} - -#[cfg(target_os = "macos")] -fn allocate_classic_group_owners( - sub_bands: &[&PreparedClassicSubBand], -) -> Result { - let job_count = crate::batch_allocation::checked_count_sum( - sub_bands.iter().map(|sub_band| sub_band.jobs.len()), - "classic J2K MetalDirect grouped jobs", - )?; - let segment_count = crate::batch_allocation::checked_count_sum( - sub_bands.iter().map(|sub_band| sub_band.segments.len()), - "classic J2K MetalDirect grouped segment table", - )?; - let coded_len = crate::batch_allocation::checked_count_sum( - sub_bands.iter().map(|sub_band| sub_band.coded_data.len()), - "classic J2K MetalDirect grouped coded payload", - )?; - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "classic J2K MetalDirect prepared sub-band group", - ); - Ok(ClassicGroupOwners { - members: budget.try_vec(sub_bands.len(), "classic J2K MetalDirect grouped members")?, - jobs: budget.try_vec(job_count, "classic J2K MetalDirect grouped jobs")?, - segments: budget.try_vec(segment_count, "classic J2K MetalDirect grouped segments")?, - coded_data: budget.try_vec(coded_len, "classic J2K MetalDirect grouped coded payload")?, - }) -} - -#[cfg(target_os = "macos")] -pub(super) fn prepare_classic_sub_band( - job: &j2k_native::J2kOwnedSubBandPlan, - tier1_prepare_mode: DirectTier1Mode, -) -> Result { - let coded_len = crate::batch_allocation::checked_count_sum( - job.jobs.iter().map(|block| block.data.len()), - "classic J2K MetalDirect coded payload", - )?; - let segment_count = crate::batch_allocation::checked_count_sum( - job.jobs.iter().map(|block| block.segments.len()), - "classic J2K MetalDirect segment table", - )?; - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "classic J2K MetalDirect prepared sub-band", - ); - let mut jobs = budget.try_vec(job.jobs.len(), "classic J2K MetalDirect jobs")?; - let mut coded_data = budget.try_vec(coded_len, "classic J2K MetalDirect coded payload")?; - let mut segments = budget.try_vec(segment_count, "classic J2K MetalDirect segment table")?; - - for block in &job.jobs { - let coded_offset = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect coded payload exceeds u32".to_string(), - })?; - coded_data.extend_from_slice(&block.data); - let segment_offset = u32::try_from(segments.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect segment table exceeds u32".to_string(), - })?; - for segment in &block.segments { - let data_offset = coded_offset - .checked_add(segment.data_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect segment offset overflow".to_string(), - })?; - segments.push(J2kClassicSegment { - data_offset, - data_length: segment.data_length, - start_coding_pass: u32::from(segment.start_coding_pass), - end_coding_pass: u32::from(segment.end_coding_pass), - use_arithmetic: u32::from(segment.use_arithmetic), - }); - } - jobs.push(J2kClassicCleanupBatchJob { - coded_offset, - coded_len: u32::try_from(block.data.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect coded payload exceeds u32".to_string(), - })?, - segment_offset, - segment_count: u32::try_from(block.segments.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect segment count exceeds u32".to_string(), - })?, - width: block.width, - height: block.height, - output_stride: job.width, - output_offset: block - .output_y - .checked_mul(job.width) - .and_then(|row| row.checked_add(block.output_x)) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect output offset overflow".to_string(), - })?, - missing_msbs: u32::from(block.missing_bit_planes), - total_bitplanes: u32::from(block.total_bitplanes), - roi_shift: u32::from(block.roi_shift), - number_of_coding_passes: u32::from(block.number_of_coding_passes), - sub_band_type: match block.sub_band_type { - j2k_native::J2kSubBandType::LowLow => 0, - j2k_native::J2kSubBandType::HighLow => 1, - j2k_native::J2kSubBandType::LowHigh => 2, - j2k_native::J2kSubBandType::HighHigh => 3, - }, - style_flags: classic_style_flags(block.style), - strict: u32::from(block.strict), - dequantization_step: block.dequantization_step, - }); - } - - with_runtime(|runtime| { - let coded_buffer = - prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode)?; - let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode)?; - let segments_buffer = - prepare_direct_tier1_input_buffer(runtime, &segments, tier1_prepare_mode)?; - Ok(PreparedClassicSubBand { - band_id: job.band_id, - width: job.width, - height: job.height, - zero_fill: false, - coded_data, - coded_buffer, - jobs, - jobs_buffer, - segments, - segments_buffer, - }) - }) -} - -#[cfg(target_os = "macos")] -pub(super) fn prepare_sub_band_groups<'a, SubBand: 'a, Group>( - steps: &'a [PreparedDirectGrayscaleStep], - tier1_prepare_mode: DirectTier1Mode, - mut sub_band_for_step: impl FnMut(&'a PreparedDirectGrayscaleStep) -> Option<&'a SubBand>, - mut prepare_group: impl FnMut(usize, usize, &[&'a SubBand], DirectTier1Mode) -> Result, -) -> Result, Error> { - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "J2K MetalDirect prepared sub-band groups", - ); - let mut groups = budget.try_vec( - steps.len(), - "J2K MetalDirect prepared sub-band group results", - )?; - let mut sub_bands = - budget.try_vec(steps.len(), "J2K MetalDirect grouped sub-band references")?; - let mut step_idx = 0; - while step_idx < steps.len() { - let start_step = step_idx; - sub_bands.clear(); - while let Some(sub_band) = steps.get(step_idx).and_then(&mut sub_band_for_step) { - sub_bands.push(sub_band); - step_idx += 1; - } - if sub_bands.len() > 1 { - groups.push(prepare_group( - start_step, - step_idx, - &sub_bands, - tier1_prepare_mode, - )?); - } - if step_idx == start_step { - step_idx += 1; - } - } - Ok(groups) -} - #[cfg(target_os = "macos")] -pub(super) fn prepare_classic_sub_band_groups( - steps: &[PreparedDirectGrayscaleStep], - tier1_prepare_mode: DirectTier1Mode, -) -> Result, Error> { - prepare_sub_band_groups( - steps, - tier1_prepare_mode, - |step| match step { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), - _ => None, - }, - prepare_classic_sub_band_group, - ) -} - -#[cfg(target_os = "macos")] -pub(super) fn prepare_classic_sub_band_group( - start_step: usize, - end_step: usize, - sub_bands: &[&PreparedClassicSubBand], - tier1_prepare_mode: DirectTier1Mode, -) -> Result { - let ClassicGroupOwners { - mut members, - mut jobs, - mut segments, - mut coded_data, - } = allocate_classic_group_owners(sub_bands)?; - let mut output_base = 0usize; - - for sub_band in sub_bands { - members.push(PreparedClassicSubBandGroupMember { - band_id: sub_band.band_id, - offset_elements: output_base, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - - let coded_base = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect grouped coded payload exceeds u32".to_string(), - })?; - let segment_base = u32::try_from(segments.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect grouped segment table exceeds u32".to_string(), - })?; - let output_base_u32 = u32::try_from(output_base).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect grouped coefficient arena exceeds u32".to_string(), - })?; - - for segment in &sub_band.segments { - let mut grouped_segment = *segment; - grouped_segment.data_offset = - coded_base - .checked_add(segment.data_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped segment offset overflow" - .to_string(), - })?; - segments.push(grouped_segment); - } - - for job in &sub_band.jobs { - let mut grouped_job = *job; - grouped_job.coded_offset = - coded_base - .checked_add(job.coded_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped job coded offset overflow" - .to_string(), - })?; - grouped_job.segment_offset = - segment_base - .checked_add(job.segment_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped job segment offset overflow" - .to_string(), - })?; - grouped_job.output_offset = - output_base_u32 - .checked_add(job.output_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped output offset overflow" - .to_string(), - })?; - jobs.push(grouped_job); - } - - coded_data.extend_from_slice(&sub_band.coded_data); - let sub_band_len = - sub_band - .width - .checked_mul(sub_band.height) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped sub-band size overflow".to_string(), - })? as usize; - output_base = output_base - .checked_add(sub_band_len) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped coefficient arena overflow".to_string(), - })?; - } - - with_runtime(|runtime| { - let coded_buffer = - prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode)?; - let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode)?; - let segments_buffer = - prepare_direct_tier1_input_buffer(runtime, &segments, tier1_prepare_mode)?; - Ok(PreparedClassicSubBandGroup { - start_step, - end_step, - total_coefficients: output_base, - zero_fill: sub_bands.iter().any(|sub_band| sub_band.zero_fill), - coded_data, - coded_buffer, - jobs, - jobs_buffer, - segments, - segments_buffer, - members, - }) - }) -} - -#[cfg(target_os = "macos")] -pub(super) fn prepare_ht_sub_band( - job: &j2k_native::HtOwnedSubBandPlan, - _tier1_prepare_mode: DirectTier1Mode, -) -> Result { - let coded_len = crate::batch_allocation::checked_count_sum( - job.jobs.iter().map(|block| block.data.len()), - "HTJ2K MetalDirect coded payload", - )?; - let mut budget = - crate::batch_allocation::BatchMetadataBudget::new("HTJ2K MetalDirect prepared sub-band"); - let mut jobs = budget.try_vec(job.jobs.len(), "HTJ2K MetalDirect jobs")?; - let mut coded_data = budget.try_vec(coded_len, "HTJ2K MetalDirect coded payload")?; - for block in &job.jobs { - let coded_offset = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect coded payload exceeds u32".to_string(), - })?; - coded_data.extend_from_slice(&block.data); - jobs.push(J2kHtCleanupBatchJob { - coded_offset, - width: block.width, - height: block.height, - coded_len: u32::try_from(block.data.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect coded payload exceeds u32".to_string(), - })?, - cleanup_length: block.cleanup_length, - refinement_length: block.refinement_length, - missing_msbs: u32::from(block.missing_bit_planes), - num_bitplanes: u32::from(block.num_bitplanes), - roi_shift: u32::from(block.roi_shift), - number_of_coding_passes: u32::from(block.number_of_coding_passes), - output_stride: job.width, - output_offset: block - .output_y - .checked_mul(job.width) - .and_then(|row| row.checked_add(block.output_x)) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect output offset overflow".to_string(), - })?, - dequantization_step: block.dequantization_step, - stripe_causal: u32::from(block.stripe_causal), - }); - } - - Ok(PreparedHtSubBand { - band_id: job.band_id, - width: job.width, - height: job.height, - coded_data, - coded_buffer: None, - jobs, - jobs_buffer: None, - }) -} - -#[cfg(target_os = "macos")] -pub(super) fn prepare_ht_sub_band_groups( - steps: &[PreparedDirectGrayscaleStep], - tier1_prepare_mode: DirectTier1Mode, -) -> Result, Error> { - prepare_sub_band_groups( - steps, - tier1_prepare_mode, - |step| match step { - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => Some(sub_band), - _ => None, - }, - prepare_ht_sub_band_group, - ) -} - -#[cfg(target_os = "macos")] -pub(super) fn prepare_ht_sub_band_group( - start_step: usize, - end_step: usize, - sub_bands: &[&PreparedHtSubBand], - tier1_prepare_mode: DirectTier1Mode, -) -> Result { - let job_count = crate::batch_allocation::checked_count_sum( - sub_bands.iter().map(|sub_band| sub_band.jobs.len()), - "HTJ2K MetalDirect grouped jobs", - )?; - let coded_len = crate::batch_allocation::checked_count_sum( - sub_bands.iter().map(|sub_band| sub_band.coded_data.len()), - "HTJ2K MetalDirect grouped coded payload", - )?; - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "HTJ2K MetalDirect prepared sub-band group", - ); - let mut members = budget.try_vec(sub_bands.len(), "HTJ2K MetalDirect grouped members")?; - let mut jobs = budget.try_vec(job_count, "HTJ2K MetalDirect grouped jobs")?; - let mut coded_data = budget.try_vec(coded_len, "HTJ2K MetalDirect grouped coded payload")?; - let mut output_base = 0usize; - - for sub_band in sub_bands { - members.push(PreparedHtSubBandGroupMember { - band_id: sub_band.band_id, - offset_elements: output_base, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - - let coded_base = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped coded payload exceeds u32".to_string(), - })?; - let output_base_u32 = u32::try_from(output_base).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped coefficient arena exceeds u32".to_string(), - })?; - for job in &sub_band.jobs { - let mut grouped_job = *job; - grouped_job.coded_offset = - coded_base - .checked_add(job.coded_offset) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped coded offset overflow".to_string(), - })?; - grouped_job.output_offset = - output_base_u32 - .checked_add(job.output_offset) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped output offset overflow".to_string(), - })?; - jobs.push(grouped_job); - } - coded_data.extend_from_slice(&sub_band.coded_data); - let sub_band_len = - sub_band - .width - .checked_mul(sub_band.height) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped sub-band size overflow".to_string(), - })? as usize; - output_base = output_base - .checked_add(sub_band_len) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped coefficient arena overflow".to_string(), - })?; - } - - with_runtime(|runtime| { - let coded_buffer = - prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode)?; - let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode)?; - Ok(PreparedHtSubBandGroup { - start_step, - end_step, - total_coefficients: output_base, - coded_arena: HtCodedArena { - data: coded_data, - buffer: coded_buffer, - }, - jobs, - jobs_buffer, - members, - }) - }) -} - -#[cfg(target_os = "macos")] -pub(super) fn prepare_ungrouped_ht_sub_band_buffers( - steps: &mut [PreparedDirectGrayscaleStep], - groups: &[PreparedHtSubBandGroup], - tier1_prepare_mode: DirectTier1Mode, -) -> Result<(), Error> { - if tier1_prepare_mode != DirectTier1Mode::Metal { - return Ok(()); - } - - for (step_idx, step) in steps.iter_mut().enumerate() { - let PreparedDirectGrayscaleStep::HtSubBand(sub_band) = step else { - continue; - }; - if groups - .iter() - .any(|group| group.start_step <= step_idx && step_idx < group.end_step) - { - sub_band.coded_buffer = None; - sub_band.jobs_buffer = None; - continue; - } - with_runtime(|runtime| { - sub_band.coded_buffer = Some(prepare_direct_tier1_input_buffer( - runtime, - &sub_band.coded_data, - tier1_prepare_mode, - )?); - sub_band.jobs_buffer = Some(prepare_direct_tier1_input_buffer( - runtime, - &sub_band.jobs, - tier1_prepare_mode, - )?); - Ok(()) - })?; - } - - Ok(()) -} - -#[cfg(target_os = "macos")] -pub(super) fn prepared_ht_buffer<'a>( - buffer: Option<&'a Buffer>, - label: &str, -) -> Result<&'a Buffer, Error> { - buffer.ok_or_else(|| Error::MetalKernel { - message: format!("HTJ2K MetalDirect ungrouped sub-band is missing prepared {label} buffer"), - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn prepare_direct_grayscale_plan( - plan: &J2kDirectGrayscalePlan, -) -> Result { - prepare_direct_grayscale_plan_with_tier1_mode(plan, DirectTier1Mode::Metal) -} +use j2k_native::{ + HtCodeBlockPayloadRanges, J2kClassicCodeBlockPayload, J2kCodestreamRange, + J2kDirectGrayscalePlan as NativeGrayscalePlan, J2kReferencedClassicPlan, + J2kReferencedHtj2kPlan, +}; -#[cfg(target_os = "macos")] -pub(super) fn prepare_direct_grayscale_plan_for_cpu_upload( - plan: &J2kDirectGrayscalePlan, -) -> Result { - prepare_direct_grayscale_plan_with_tier1_mode(plan, DirectTier1Mode::CpuUpload) -} +mod classic; +mod color; +mod grayscale; +mod ht; +mod referenced; -#[cfg(target_os = "macos")] -pub(super) fn prepare_direct_grayscale_plan_with_tier1_mode( - plan: &J2kDirectGrayscalePlan, - tier1_prepare_mode: DirectTier1Mode, -) -> Result { - let mut budget = crate::batch_allocation::BatchMetadataBudget::new( - "J2K MetalDirect prepared grayscale plan", - ); - let mut steps = budget.try_vec(plan.steps.len(), "J2K MetalDirect prepared grayscale steps")?; - for step in &plan.steps { - match step { - J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { - steps.push(PreparedDirectGrayscaleStep::ClassicSubBand( - prepare_classic_sub_band(sub_band, tier1_prepare_mode)?, - )); - } - J2kDirectGrayscaleStep::HtSubBand(sub_band) => { - steps.push(PreparedDirectGrayscaleStep::HtSubBand(prepare_ht_sub_band( - sub_band, - tier1_prepare_mode, - )?)); - } - J2kDirectGrayscaleStep::Idwt(idwt) => { - steps.push(PreparedDirectGrayscaleStep::Idwt(PreparedDirectIdwt { - step: *idwt, - output_window: BandRequiredRegion::full(idwt.rect.width(), idwt.rect.height()), - })); - } - J2kDirectGrayscaleStep::Store(store) => { - steps.push(PreparedDirectGrayscaleStep::Store(*store)); - } - } - } - let classic_groups = prepare_classic_sub_band_groups(&steps, tier1_prepare_mode)?; - let ht_groups = prepare_ht_sub_band_groups(&steps, tier1_prepare_mode)?; - prepare_ungrouped_ht_sub_band_buffers(&mut steps, &ht_groups, tier1_prepare_mode)?; - Ok(PreparedDirectGrayscalePlan { - dimensions: plan.dimensions, - bit_depth: plan.bit_depth, - tier1_prepare_mode, - steps, - classic_groups, - ht_groups, - cpu_tier1_cache: Arc::new(CpuTier1CoefficientCache::default()), - }) -} +pub(in crate::compute) use self::classic::{ + prepare_classic_sub_band, prepare_classic_sub_band_groups, prepare_sub_band_groups, +}; +use self::classic::{prepare_referenced_classic_sub_band, ReferencedClassicPayloadCursor}; +pub(crate) use self::color::{ + prepare_referenced_classic_color_plan, prepare_referenced_classic_rgba_plan, + prepare_referenced_htj2k_color_plan, prepare_referenced_htj2k_rgba_plan, +}; +pub(in crate::compute) use self::grayscale::prepare_direct_grayscale_plan_for_cpu_upload; +pub(crate) use self::grayscale::{ + prepare_direct_grayscale_plan, prepare_referenced_classic_grayscale_plan, + prepare_referenced_htj2k_grayscale_plan, +}; +use self::ht::prepare_referenced_ht_sub_band; +pub(in crate::compute) use self::ht::{prepare_ht_sub_band, prepare_ht_sub_band_groups}; +use self::referenced::{ + append_referenced_classic_component_steps, append_referenced_htj2k_component_steps, + finish_referenced_component_plan, validate_payload_record_span, + validate_referenced_component_metadata, +}; diff --git a/crates/j2k-metal/src/compute/direct_prepare/classic.rs b/crates/j2k-metal/src/compute/direct_prepare/classic.rs new file mode 100644 index 00000000..7c0fb76f --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_prepare/classic.rs @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Classic JPEG 2000 sub-band and grouped payload preparation. + +use super::{ + classic_style_flags, prepare_direct_tier1_input_buffer, with_runtime, BandRequiredRegion, + DirectTier1Mode, Error, J2kClassicCleanupBatchJob, J2kClassicCodeBlockPayload, + J2kClassicSegment, J2kCodestreamRange, J2kReferencedClassicPlan, PreparedClassicSubBand, + PreparedClassicSubBandGroup, PreparedClassicSubBandGroupMember, PreparedDirectGrayscaleStep, +}; + +#[cfg(target_os = "macos")] +struct ClassicGroupOwners { + members: Vec, + jobs: Vec, + segments: Vec, + coded_data: Vec, +} + +#[cfg(target_os = "macos")] +pub(super) struct ReferencedClassicPayloadCursor<'a> { + input: &'a [u8], + payloads: &'a [J2kClassicCodeBlockPayload], + ranges: &'a [J2kCodestreamRange], + pub(super) next_payload: usize, + next_range: usize, +} + +#[cfg(target_os = "macos")] +impl<'a> ReferencedClassicPayloadCursor<'a> { + pub(super) fn new(input: &'a [u8], plan: &'a J2kReferencedClassicPlan) -> Self { + Self { + input, + payloads: plan.payloads(), + ranges: plan.ranges(), + next_payload: 0, + next_range: 0, + } + } + + fn expected_payload_bytes(&self, count: usize) -> Result { + let end = self + .next_payload + .checked_add(count) + .ok_or(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "payload traversal count overflowed", + })?; + let payloads = + self.payloads + .get(self.next_payload..end) + .ok_or(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "geometry contains more jobs than retained payload descriptors", + })?; + Ok(crate::batch_allocation::checked_count_sum( + payloads.iter().map(|payload| payload.combined_length), + "classic J2K referenced Metal coded payload", + )?) + } + + fn append_next(&mut self, coded_data: &mut Vec) -> Result { + let payload = + self.payloads + .get(self.next_payload) + .copied() + .ok_or(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "geometry contains more jobs than retained payload descriptors", + })?; + if payload.first_range != self.next_range { + return Err(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "payload fragment ranges are not contiguous in traversal order", + }); + } + let end_range = payload.end_range().ok_or(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "payload fragment range overflowed", + })?; + let fragments = + self.ranges + .get(payload.first_range..end_range) + .ok_or(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "payload fragment range exceeds the retained range table", + })?; + let before = coded_data.len(); + for range in fragments { + let end = range.end().ok_or(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "encoded payload byte range overflowed", + })?; + let fragment = self + .input + .get(range.offset..end) + .ok_or(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "encoded payload byte range exceeds the retained input", + })?; + coded_data.extend_from_slice(fragment); + } + let appended = coded_data + .len() + .checked_sub(before) + .ok_or(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "coded payload length moved backwards", + })?; + if appended != payload.combined_length { + return Err(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "concatenated fragments do not match their retained payload length", + }); + } + self.next_payload = self + .next_payload + .checked_add(1) + .ok_or(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "payload cursor overflowed", + })?; + self.next_range = end_range; + Ok(appended) + } + + pub(super) fn ensure_exhausted(&self) -> Result<(), Error> { + if self.next_payload == self.payloads.len() && self.next_range == self.ranges.len() { + Ok(()) + } else { + Err(Error::MetalStateInvariant { + state: "classic J2K referenced payload cursor", + reason: "retained payload descriptors or ranges were left unused", + }) + } + } +} + +#[cfg(target_os = "macos")] +fn allocate_classic_group_owners( + sub_bands: &[&PreparedClassicSubBand], +) -> Result { + let job_count = crate::batch_allocation::checked_count_sum( + sub_bands.iter().map(|sub_band| sub_band.jobs.len()), + "classic J2K MetalDirect grouped jobs", + )?; + let segment_count = crate::batch_allocation::checked_count_sum( + sub_bands.iter().map(|sub_band| sub_band.segments.len()), + "classic J2K MetalDirect grouped segment table", + )?; + let coded_len = crate::batch_allocation::checked_count_sum( + sub_bands.iter().map(|sub_band| sub_band.coded_data.len()), + "classic J2K MetalDirect grouped coded payload", + )?; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "classic J2K MetalDirect prepared sub-band group", + ); + Ok(ClassicGroupOwners { + members: budget.try_vec(sub_bands.len(), "classic J2K MetalDirect grouped members")?, + jobs: budget.try_vec(job_count, "classic J2K MetalDirect grouped jobs")?, + segments: budget.try_vec(segment_count, "classic J2K MetalDirect grouped segments")?, + coded_data: budget.try_vec(coded_len, "classic J2K MetalDirect grouped coded payload")?, + }) +} + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn prepare_classic_sub_band( + job: &j2k_native::J2kOwnedSubBandPlan, + tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let coded_len = crate::batch_allocation::checked_count_sum( + job.jobs.iter().map(|block| block.data.len()), + "classic J2K MetalDirect coded payload", + )?; + prepare_classic_sub_band_with_payloads( + job, + tier1_prepare_mode, + coded_len, + |block_index, coded_data| { + let before = coded_data.len(); + coded_data.extend_from_slice(&job.jobs[block_index].data); + Ok(coded_data.len() - before) + }, + ) +} + +#[cfg(target_os = "macos")] +fn prepare_classic_sub_band_with_payloads( + job: &j2k_native::J2kOwnedSubBandPlan, + tier1_prepare_mode: DirectTier1Mode, + coded_len: usize, + mut append_payload: impl FnMut(usize, &mut Vec) -> Result, +) -> Result { + let segment_count = crate::batch_allocation::checked_count_sum( + job.jobs.iter().map(|block| block.segments.len()), + "classic J2K MetalDirect segment table", + )?; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "classic J2K MetalDirect prepared sub-band", + ); + let mut jobs = budget.try_vec(job.jobs.len(), "classic J2K MetalDirect jobs")?; + let mut coded_data = budget.try_vec(coded_len, "classic J2K MetalDirect coded payload")?; + let mut segments = budget.try_vec(segment_count, "classic J2K MetalDirect segment table")?; + + for (block_index, block) in job.jobs.iter().enumerate() { + let coded_offset = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect coded payload exceeds u32".to_string(), + })?; + let block_coded_len = append_payload(block_index, &mut coded_data)?; + let segment_offset = u32::try_from(segments.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect segment table exceeds u32".to_string(), + })?; + for segment in &block.segments { + let data_offset = coded_offset + .checked_add(segment.data_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect segment offset overflow".to_string(), + })?; + segments.push(J2kClassicSegment { + data_offset, + data_length: segment.data_length, + start_coding_pass: u32::from(segment.start_coding_pass), + end_coding_pass: u32::from(segment.end_coding_pass), + use_arithmetic: u32::from(segment.use_arithmetic), + }); + } + jobs.push(J2kClassicCleanupBatchJob { + coded_offset, + coded_len: u32::try_from(block_coded_len).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect coded payload exceeds u32".to_string(), + })?, + segment_offset, + segment_count: u32::try_from(block.segments.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect segment count exceeds u32".to_string(), + })?, + width: block.width, + height: block.height, + output_stride: job.width, + output_offset: block + .output_y + .checked_mul(job.width) + .and_then(|row| row.checked_add(block.output_x)) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect output offset overflow".to_string(), + })?, + missing_msbs: u32::from(block.missing_bit_planes), + total_bitplanes: u32::from(block.total_bitplanes), + roi_shift: u32::from(block.roi_shift), + number_of_coding_passes: u32::from(block.number_of_coding_passes), + sub_band_type: match block.sub_band_type { + j2k_native::J2kSubBandType::LowLow => 0, + j2k_native::J2kSubBandType::HighLow => 1, + j2k_native::J2kSubBandType::LowHigh => 2, + j2k_native::J2kSubBandType::HighHigh => 3, + }, + style_flags: classic_style_flags(block.style), + strict: u32::from(block.strict), + dequantization_step: block.dequantization_step, + }); + } + if coded_data.len() != coded_len { + return Err(Error::MetalStateInvariant { + state: "classic J2K MetalDirect prepared sub-band", + reason: "appended payload bytes do not match the preflight allocation", + }); + } + let zero_fill = jobs + .iter() + .any(|job| job.coded_len == 0 || job.number_of_coding_passes == 0); + + with_runtime(|runtime| { + let coded_buffer = + prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode)?; + let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode)?; + let segments_buffer = + prepare_direct_tier1_input_buffer(runtime, &segments, tier1_prepare_mode)?; + Ok(PreparedClassicSubBand { + band_id: job.band_id, + width: job.width, + height: job.height, + zero_fill, + coded_data, + coded_buffer, + jobs, + jobs_buffer, + segments, + segments_buffer, + }) + }) +} + +#[cfg(target_os = "macos")] +pub(super) fn prepare_referenced_classic_sub_band( + job: &j2k_native::J2kOwnedSubBandPlan, + payloads: &mut ReferencedClassicPayloadCursor<'_>, +) -> Result { + if job.jobs.iter().any(|block| !block.data.is_empty()) { + return Err(Error::MetalStateInvariant { + state: "classic J2K referenced Metal sub-band", + reason: "referenced geometry unexpectedly owns compressed payload bytes", + }); + } + let coded_len = payloads.expected_payload_bytes(job.jobs.len())?; + prepare_classic_sub_band_with_payloads( + job, + DirectTier1Mode::Metal, + coded_len, + |_, coded_data| payloads.append_next(coded_data), + ) +} + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn prepare_sub_band_groups<'a, SubBand: 'a, Group>( + steps: &'a [PreparedDirectGrayscaleStep], + tier1_prepare_mode: DirectTier1Mode, + mut sub_band_for_step: impl FnMut(&'a PreparedDirectGrayscaleStep) -> Option<&'a SubBand>, + mut prepare_group: impl FnMut(usize, usize, &[&'a SubBand], DirectTier1Mode) -> Result, +) -> Result, Error> { + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K MetalDirect prepared sub-band groups", + ); + let mut groups = budget.try_vec( + steps.len(), + "J2K MetalDirect prepared sub-band group results", + )?; + let mut sub_bands = + budget.try_vec(steps.len(), "J2K MetalDirect grouped sub-band references")?; + let mut step_idx = 0; + while step_idx < steps.len() { + let start_step = step_idx; + sub_bands.clear(); + while let Some(sub_band) = steps.get(step_idx).and_then(&mut sub_band_for_step) { + sub_bands.push(sub_band); + step_idx += 1; + } + if sub_bands.len() > 1 { + groups.push(prepare_group( + start_step, + step_idx, + &sub_bands, + tier1_prepare_mode, + )?); + } + if step_idx == start_step { + step_idx += 1; + } + } + Ok(groups) +} + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn prepare_classic_sub_band_groups( + steps: &[PreparedDirectGrayscaleStep], + tier1_prepare_mode: DirectTier1Mode, +) -> Result, Error> { + prepare_sub_band_groups( + steps, + tier1_prepare_mode, + |step| match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), + _ => None, + }, + prepare_classic_sub_band_group, + ) +} + +#[cfg(target_os = "macos")] +pub(super) fn prepare_classic_sub_band_group( + start_step: usize, + end_step: usize, + sub_bands: &[&PreparedClassicSubBand], + tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let ClassicGroupOwners { + mut members, + mut jobs, + mut segments, + mut coded_data, + } = allocate_classic_group_owners(sub_bands)?; + let mut output_base = 0usize; + + for sub_band in sub_bands { + members.push(PreparedClassicSubBandGroupMember { + band_id: sub_band.band_id, + offset_elements: output_base, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + + let coded_base = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect grouped coded payload exceeds u32".to_string(), + })?; + let segment_base = u32::try_from(segments.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect grouped segment table exceeds u32".to_string(), + })?; + let output_base_u32 = u32::try_from(output_base).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect grouped coefficient arena exceeds u32".to_string(), + })?; + + for segment in &sub_band.segments { + let mut grouped_segment = *segment; + grouped_segment.data_offset = + coded_base + .checked_add(segment.data_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped segment offset overflow" + .to_string(), + })?; + segments.push(grouped_segment); + } + + for job in &sub_band.jobs { + let mut grouped_job = *job; + grouped_job.coded_offset = + coded_base + .checked_add(job.coded_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped job coded offset overflow" + .to_string(), + })?; + grouped_job.segment_offset = + segment_base + .checked_add(job.segment_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped job segment offset overflow" + .to_string(), + })?; + grouped_job.output_offset = + output_base_u32 + .checked_add(job.output_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped output offset overflow" + .to_string(), + })?; + jobs.push(grouped_job); + } + + coded_data.extend_from_slice(&sub_band.coded_data); + let sub_band_len = + sub_band + .width + .checked_mul(sub_band.height) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped sub-band size overflow".to_string(), + })? as usize; + output_base = output_base + .checked_add(sub_band_len) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped coefficient arena overflow".to_string(), + })?; + } + + with_runtime(|runtime| { + let coded_buffer = + prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode)?; + let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode)?; + let segments_buffer = + prepare_direct_tier1_input_buffer(runtime, &segments, tier1_prepare_mode)?; + Ok(PreparedClassicSubBandGroup { + start_step, + end_step, + total_coefficients: output_base, + zero_fill: sub_bands.iter().any(|sub_band| sub_band.zero_fill), + coded_data, + coded_buffer, + jobs, + jobs_buffer, + segments, + segments_buffer, + members, + }) + }) +} diff --git a/crates/j2k-metal/src/compute/direct_prepare/color.rs b/crates/j2k-metal/src/compute/direct_prepare/color.rs new file mode 100644 index 00000000..05c0b868 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_prepare/color.rs @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Retained classic and HTJ2K RGB/RGBA plan preparation. + +use super::{ + append_referenced_classic_component_steps, append_referenced_htj2k_component_steps, + finish_referenced_component_plan, validate_payload_record_span, + validate_referenced_component_metadata, Error, J2kReferencedClassicPlan, + J2kReferencedHtj2kPlan, NativeGrayscalePlan, PreparedDirectColorPlan, + PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, ReferencedClassicPayloadCursor, +}; +use std::sync::Arc; + +#[cfg(target_os = "macos")] +pub(crate) fn prepare_referenced_classic_color_plan( + referenced: &J2kReferencedClassicPlan, + input: &[u8], + signed: bool, +) -> Result { + let prepared = prepare_referenced_classic_color_tiles( + referenced, + input, + 3, + "J2K MetalDirect referenced classic RGB plan", + )?; + Ok(PreparedDirectColorPlan { + dimensions: prepared.dimensions, + bit_depths: [ + prepared.bit_depths[0], + prepared.bit_depths[1], + prepared.bit_depths[2], + ], + alpha_bit_depth: None, + signed, + mct: prepared.mct, + transform: prepared.transform, + component_plans: prepared.component_plans, + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn prepare_referenced_classic_rgba_plan( + referenced: &J2kReferencedClassicPlan, + input: &[u8], + signed: bool, +) -> Result { + let prepared = prepare_referenced_classic_color_tiles( + referenced, + input, + 4, + "J2K MetalDirect referenced classic RGBA plan", + )?; + Ok(PreparedDirectColorPlan { + dimensions: prepared.dimensions, + bit_depths: [ + prepared.bit_depths[0], + prepared.bit_depths[1], + prepared.bit_depths[2], + ], + alpha_bit_depth: Some(prepared.bit_depths[3]), + signed, + mct: prepared.mct, + transform: prepared.transform, + component_plans: prepared.component_plans, + }) +} + +#[cfg(target_os = "macos")] +fn prepare_referenced_classic_color_tiles( + referenced: &J2kReferencedClassicPlan, + input: &[u8], + expected_component_count: usize, + context: &'static str, +) -> Result { + let first = referenced + .tiles() + .first() + .and_then(|tile| referenced_color_geometry(tile, expected_component_count)) + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal color prepared path received incompatible classic J2K tile geometry", + })?; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new(context); + let mut component_steps = budget.try_vec(expected_component_count, context)?; + for component_index in 0..expected_component_count { + let step_count = crate::batch_allocation::checked_count_sum( + referenced.tiles().iter().map(|tile| { + referenced_color_geometry(tile, expected_component_count) + .and_then(|geometry| geometry.component_plans.get(component_index)) + .map_or(0, |component| component.steps.len()) + }), + context, + )?; + component_steps.push(budget.try_vec(step_count, context)?); + } + let mut payloads = ReferencedClassicPayloadCursor::new(input, referenced); + for tile in referenced.tiles() { + let geometry = + referenced_color_geometry(tile, expected_component_count) + .ok_or(Error::UnsupportedMetalRequest { + reason: + "J2K Metal color prepared path received incompatible classic J2K tile geometry", + })?; + validate_referenced_color_metadata(first, geometry)?; + let expected_end = validate_payload_record_span( + tile.payload_records(), + payloads.next_payload, + referenced.payloads().len(), + "classic color tile", + )?; + for (component, steps) in geometry.component_plans.iter().zip(&mut component_steps) { + append_referenced_classic_component_steps(component, &mut payloads, steps, context)?; + } + if payloads.next_payload != expected_end { + return Err(Error::MetalStateInvariant { + state: "classic color tile payload traversal", + reason: "tile geometry job count does not match its payload-record span", + }); + } + } + payloads.ensure_exhausted()?; + finish_referenced_color_tiles(first, component_steps, context) +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +struct ReferencedColorGeometry<'a> { + dimensions: (u32, u32), + bit_depths: [u8; 4], + mct: bool, + transform: j2k_native::J2kWaveletTransform, + component_plans: &'a [NativeGrayscalePlan], +} + +#[cfg(target_os = "macos")] +struct PreparedReferencedColorTiles { + dimensions: (u32, u32), + bit_depths: [u8; 4], + mct: bool, + transform: j2k_native::J2kWaveletTransform, + component_plans: Vec, +} + +#[cfg(target_os = "macos")] +fn referenced_color_geometry( + tile: &j2k_native::J2kReferencedTilePlan, + expected_component_count: usize, +) -> Option> { + match expected_component_count { + 3 => tile + .color_geometry() + .map(|geometry| ReferencedColorGeometry { + dimensions: geometry.dimensions, + bit_depths: [ + geometry.bit_depths[0], + geometry.bit_depths[1], + geometry.bit_depths[2], + 0, + ], + mct: geometry.mct, + transform: geometry.transform, + component_plans: &geometry.component_plans, + }), + 4 => tile + .rgba_geometry() + .map(|geometry| ReferencedColorGeometry { + dimensions: geometry.dimensions, + bit_depths: geometry.bit_depths, + mct: geometry.mct, + transform: geometry.transform, + component_plans: &geometry.component_plans, + }), + _ => None, + } +} + +#[cfg(target_os = "macos")] +fn validate_referenced_color_metadata( + first: ReferencedColorGeometry<'_>, + tile: ReferencedColorGeometry<'_>, +) -> Result<(), Error> { + if tile.dimensions != first.dimensions + || tile.bit_depths != first.bit_depths + || tile.mct != first.mct + || tile.transform != first.transform + || tile.component_plans.len() != first.component_plans.len() + { + return Err(Error::MetalStateInvariant { + state: "referenced multi-tile color metadata", + reason: "tile color dimensions, precision, transform, or component count changed", + }); + } + for (first_component, tile_component) in first.component_plans.iter().zip(tile.component_plans) + { + validate_referenced_component_metadata(first_component, tile_component)?; + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn finish_referenced_color_tiles( + first: ReferencedColorGeometry<'_>, + component_steps: Vec>, + context: &'static str, +) -> Result { + if component_steps.len() != first.component_plans.len() { + return Err(Error::MetalStateInvariant { + state: "referenced multi-tile color preparation", + reason: "prepared component step count does not match color metadata", + }); + } + let mut budget = crate::batch_allocation::BatchMetadataBudget::new(context); + let mut component_plans = budget.try_vec(component_steps.len(), context)?; + for (component, steps) in first.component_plans.iter().zip(component_steps) { + component_plans.push(finish_referenced_component_plan( + component.dimensions, + component.bit_depth, + steps, + context, + )?); + } + Ok(PreparedReferencedColorTiles { + dimensions: first.dimensions, + bit_depths: first.bit_depths, + mct: first.mct, + transform: first.transform, + component_plans, + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn prepare_referenced_htj2k_color_plan( + referenced: &J2kReferencedHtj2kPlan, + input: &Arc<[u8]>, + signed: bool, +) -> Result { + let prepared = prepare_referenced_htj2k_color_tiles( + referenced, + input, + 3, + "J2K MetalDirect referenced HTJ2K RGB plan", + )?; + Ok(PreparedDirectColorPlan { + dimensions: prepared.dimensions, + bit_depths: [ + prepared.bit_depths[0], + prepared.bit_depths[1], + prepared.bit_depths[2], + ], + alpha_bit_depth: None, + signed, + mct: prepared.mct, + transform: prepared.transform, + component_plans: prepared.component_plans, + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn prepare_referenced_htj2k_rgba_plan( + referenced: &J2kReferencedHtj2kPlan, + input: &Arc<[u8]>, + signed: bool, +) -> Result { + let prepared = prepare_referenced_htj2k_color_tiles( + referenced, + input, + 4, + "J2K MetalDirect referenced HTJ2K RGBA plan", + )?; + Ok(PreparedDirectColorPlan { + dimensions: prepared.dimensions, + bit_depths: [ + prepared.bit_depths[0], + prepared.bit_depths[1], + prepared.bit_depths[2], + ], + alpha_bit_depth: Some(prepared.bit_depths[3]), + signed, + mct: prepared.mct, + transform: prepared.transform, + component_plans: prepared.component_plans, + }) +} + +#[cfg(target_os = "macos")] +fn prepare_referenced_htj2k_color_tiles( + referenced: &J2kReferencedHtj2kPlan, + input: &Arc<[u8]>, + expected_component_count: usize, + context: &'static str, +) -> Result { + let first = referenced + .tiles() + .first() + .and_then(|tile| referenced_color_geometry(tile, expected_component_count)) + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal color prepared path received incompatible HTJ2K tile geometry", + })?; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new(context); + let mut component_steps = budget.try_vec(expected_component_count, context)?; + for component_index in 0..expected_component_count { + let step_count = crate::batch_allocation::checked_count_sum( + referenced.tiles().iter().map(|tile| { + referenced_color_geometry(tile, expected_component_count) + .and_then(|geometry| geometry.component_plans.get(component_index)) + .map_or(0, |component| component.steps.len()) + }), + context, + )?; + component_steps.push(budget.try_vec(step_count, context)?); + } + let mut payload_cursor = 0usize; + for tile in referenced.tiles() { + let geometry = referenced_color_geometry(tile, expected_component_count).ok_or( + Error::UnsupportedMetalRequest { + reason: "J2K Metal color prepared path received incompatible HTJ2K tile geometry", + }, + )?; + validate_referenced_color_metadata(first, geometry)?; + let expected_end = validate_payload_record_span( + tile.payload_records(), + payload_cursor, + referenced.payloads().len(), + "HTJ2K color tile", + )?; + for (component, steps) in geometry.component_plans.iter().zip(&mut component_steps) { + append_referenced_htj2k_component_steps( + component, + input, + referenced.payloads(), + &mut payload_cursor, + steps, + )?; + } + if payload_cursor != expected_end { + return Err(Error::MetalStateInvariant { + state: "HTJ2K color tile payload traversal", + reason: "tile geometry job count does not match its payload-record span", + }); + } + } + if payload_cursor != referenced.payloads().len() { + return Err(Error::MetalKernel { + message: format!("{context} has unused payload ranges"), + }); + } + finish_referenced_color_tiles(first, component_steps, context) +} diff --git a/crates/j2k-metal/src/compute/direct_prepare/grayscale.rs b/crates/j2k-metal/src/compute/direct_prepare/grayscale.rs new file mode 100644 index 00000000..7dfd8606 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_prepare/grayscale.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Full and retained grayscale direct-plan preparation. + +use super::{ + append_referenced_classic_component_steps, append_referenced_htj2k_component_steps, + finish_referenced_component_plan, prepare_classic_sub_band, prepare_classic_sub_band_groups, + prepare_ht_sub_band, prepare_ht_sub_band_groups, validate_payload_record_span, + validate_referenced_component_metadata, Arc, BandRequiredRegion, CpuTier1CoefficientCache, + DirectTier1Mode, Error, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, + J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, PreparedDirectGrayscalePlan, + PreparedDirectGrayscaleStep, PreparedDirectIdwt, ReferencedClassicPayloadCursor, +}; + +#[cfg(target_os = "macos")] +pub(crate) fn prepare_direct_grayscale_plan( + plan: &J2kDirectGrayscalePlan, +) -> Result { + prepare_direct_grayscale_plan_with_tier1_mode(plan, DirectTier1Mode::Metal) +} + +#[cfg(target_os = "macos")] +pub(crate) fn prepare_referenced_htj2k_grayscale_plan( + referenced: &J2kReferencedHtj2kPlan, + input: &Arc<[u8]>, +) -> Result { + let first = referenced + .tiles() + .first() + .and_then(j2k_native::J2kReferencedTilePlan::grayscale_geometry) + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal grayscale prepared path received a color HTJ2K plan", + })?; + let step_count = crate::batch_allocation::checked_count_sum( + referenced.tiles().iter().map(|tile| { + tile.grayscale_geometry() + .map_or(0, |geometry| geometry.steps.len()) + }), + "J2K MetalDirect referenced HTJ2K grayscale tile steps", + )?; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K MetalDirect referenced HTJ2K grayscale plan", + ); + let mut steps = budget.try_vec( + step_count, + "J2K MetalDirect referenced HTJ2K grayscale tile steps", + )?; + let mut payload_cursor = 0usize; + for tile in referenced.tiles() { + let geometry = tile + .grayscale_geometry() + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal grayscale prepared path received a color HTJ2K tile", + })?; + validate_referenced_component_metadata(first, geometry)?; + let expected_end = validate_payload_record_span( + tile.payload_records(), + payload_cursor, + referenced.payloads().len(), + "HTJ2K grayscale tile", + )?; + append_referenced_htj2k_component_steps( + geometry, + input, + referenced.payloads(), + &mut payload_cursor, + &mut steps, + )?; + if payload_cursor != expected_end { + return Err(Error::MetalStateInvariant { + state: "HTJ2K grayscale tile payload traversal", + reason: "tile geometry job count does not match its payload-record span", + }); + } + } + if payload_cursor != referenced.payloads().len() { + return Err(Error::MetalKernel { + message: "HTJ2K referenced plan has unused payload ranges".to_string(), + }); + } + finish_referenced_component_plan( + first.dimensions, + first.bit_depth, + steps, + "J2K MetalDirect referenced HTJ2K grayscale plan", + ) +} + +#[cfg(target_os = "macos")] +pub(crate) fn prepare_referenced_classic_grayscale_plan( + referenced: &J2kReferencedClassicPlan, + input: &[u8], +) -> Result { + let first = referenced + .tiles() + .first() + .and_then(j2k_native::J2kReferencedTilePlan::grayscale_geometry) + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal grayscale prepared path received a color classic J2K plan", + })?; + let step_count = crate::batch_allocation::checked_count_sum( + referenced.tiles().iter().map(|tile| { + tile.grayscale_geometry() + .map_or(0, |geometry| geometry.steps.len()) + }), + "J2K MetalDirect referenced classic grayscale tile steps", + )?; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K MetalDirect referenced classic grayscale plan", + ); + let mut steps = budget.try_vec( + step_count, + "J2K MetalDirect referenced classic grayscale tile steps", + )?; + let mut payloads = ReferencedClassicPayloadCursor::new(input, referenced); + for tile in referenced.tiles() { + let geometry = tile + .grayscale_geometry() + .ok_or(Error::UnsupportedMetalRequest { + reason: "J2K Metal grayscale prepared path received a color classic J2K tile", + })?; + validate_referenced_component_metadata(first, geometry)?; + let expected_end = validate_payload_record_span( + tile.payload_records(), + payloads.next_payload, + referenced.payloads().len(), + "classic grayscale tile", + )?; + append_referenced_classic_component_steps( + geometry, + &mut payloads, + &mut steps, + "J2K MetalDirect referenced classic grayscale plan", + )?; + if payloads.next_payload != expected_end { + return Err(Error::MetalStateInvariant { + state: "classic grayscale tile payload traversal", + reason: "tile geometry job count does not match its payload-record span", + }); + } + } + payloads.ensure_exhausted()?; + finish_referenced_component_plan( + first.dimensions, + first.bit_depth, + steps, + "J2K MetalDirect referenced classic grayscale plan", + ) +} + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn prepare_direct_grayscale_plan_for_cpu_upload( + plan: &J2kDirectGrayscalePlan, +) -> Result { + prepare_direct_grayscale_plan_with_tier1_mode(plan, DirectTier1Mode::CpuUpload) +} + +#[cfg(target_os = "macos")] +pub(super) fn prepare_direct_grayscale_plan_with_tier1_mode( + plan: &J2kDirectGrayscalePlan, + tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "J2K MetalDirect prepared grayscale plan", + ); + let mut steps = budget.try_vec(plan.steps.len(), "J2K MetalDirect prepared grayscale steps")?; + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { + steps.push(PreparedDirectGrayscaleStep::ClassicSubBand( + prepare_classic_sub_band(sub_band, tier1_prepare_mode)?, + )); + } + J2kDirectGrayscaleStep::HtSubBand(sub_band) => { + steps.push(PreparedDirectGrayscaleStep::HtSubBand(prepare_ht_sub_band( + sub_band, + tier1_prepare_mode, + )?)); + } + J2kDirectGrayscaleStep::Idwt(idwt) => { + steps.push(PreparedDirectGrayscaleStep::Idwt(PreparedDirectIdwt { + step: *idwt, + output_window: BandRequiredRegion::full(idwt.rect.width(), idwt.rect.height()), + })); + } + J2kDirectGrayscaleStep::Store(store) => { + steps.push(PreparedDirectGrayscaleStep::Store(*store)); + } + } + } + let classic_groups = prepare_classic_sub_band_groups(&steps, tier1_prepare_mode)?; + let ht_groups = prepare_ht_sub_band_groups(&steps, tier1_prepare_mode)?; + Ok(PreparedDirectGrayscalePlan { + dimensions: plan.dimensions, + bit_depth: plan.bit_depth, + tier1_prepare_mode, + steps, + classic_groups, + ht_groups, + cpu_tier1_cache: Arc::new(CpuTier1CoefficientCache::default()), + }) +} diff --git a/crates/j2k-metal/src/compute/direct_prepare/ht.rs b/crates/j2k-metal/src/compute/direct_prepare/ht.rs new file mode 100644 index 00000000..9986b156 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_prepare/ht.rs @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! HTJ2K sub-band and grouped payload preparation. + +use super::{ + prepare_sub_band_groups, BandRequiredRegion, DirectTier1Mode, Error, HtCodeBlockPayloadRanges, + J2kCodestreamRange, J2kHtCleanupBatchJob, PreparedDirectGrayscaleStep, PreparedHtPayloadSource, + PreparedHtSubBand, PreparedHtSubBandGroup, PreparedHtSubBandGroupMember, +}; +use crate::compute::direct_plan_types::PreparedHtExecutionOwner; +use std::sync::Arc; + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn prepare_ht_sub_band( + job: &j2k_native::HtOwnedSubBandPlan, + _tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let coded_len = crate::batch_allocation::checked_count_sum( + job.jobs.iter().map(|block| block.data.len()), + "HTJ2K MetalDirect coded payload", + )?; + let mut budget = + crate::batch_allocation::BatchMetadataBudget::new("HTJ2K MetalDirect prepared sub-band"); + let mut jobs = budget.try_vec(job.jobs.len(), "HTJ2K MetalDirect jobs")?; + let mut coded_data = budget.try_vec(coded_len, "HTJ2K MetalDirect coded payload")?; + for block in &job.jobs { + let coded_offset = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect coded payload exceeds u32".to_string(), + })?; + coded_data.extend_from_slice(&block.data); + jobs.push(J2kHtCleanupBatchJob { + coded_offset, + width: block.width, + height: block.height, + coded_len: u32::try_from(block.data.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect coded payload exceeds u32".to_string(), + })?, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + missing_msbs: u32::from(block.missing_bit_planes), + num_bitplanes: u32::from(block.num_bitplanes), + roi_shift: u32::from(block.roi_shift), + number_of_coding_passes: u32::from(block.number_of_coding_passes), + output_stride: job.width, + output_offset: block + .output_y + .checked_mul(job.width) + .and_then(|row| row.checked_add(block.output_x)) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect output offset overflow".to_string(), + })?, + dequantization_step: block.dequantization_step, + stripe_causal: u32::from(block.stripe_causal), + }); + } + + Ok(PreparedHtSubBand { + band_id: job.band_id, + width: job.width, + height: job.height, + payload_source: PreparedHtPayloadSource::Contiguous(coded_data), + jobs, + execution_owner: Arc::new(PreparedHtExecutionOwner), + }) +} + +#[cfg(target_os = "macos")] +#[expect( + clippy::too_many_lines, + reason = "referenced preparation keeps payload validation and ABI job offsets synchronized" +)] +pub(super) fn prepare_referenced_ht_sub_band( + job: &j2k_native::HtOwnedSubBandPlan, + input: &Arc<[u8]>, + payloads: &[HtCodeBlockPayloadRanges], + payload_cursor: &mut usize, +) -> Result { + let payload_end = + payload_cursor + .checked_add(job.jobs.len()) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K referenced payload cursor overflow".to_string(), + })?; + let job_payloads = + payloads + .get(*payload_cursor..payload_end) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K referenced plan has fewer payload ranges than code-block jobs" + .to_string(), + })?; + let coded_len = crate::batch_allocation::checked_count_sum( + job_payloads.iter().flat_map(|payload| { + core::iter::once(payload.cleanup.length) + .chain(payload.refinement.map(|range| range.length)) + }), + "HTJ2K MetalDirect referenced coded payload", + )?; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "HTJ2K MetalDirect referenced prepared sub-band", + ); + let mut jobs = budget.try_vec(job.jobs.len(), "HTJ2K MetalDirect referenced jobs")?; + let mut ranges = budget.try_vec( + job.jobs.len(), + "HTJ2K MetalDirect referenced payload ranges", + )?; + let mut logical_coded_len = 0usize; + for (block, payload) in job.jobs.iter().zip(job_payloads) { + if !block.data.is_empty() { + return Err(Error::MetalStateInvariant { + state: "HTJ2K referenced direct plan", + reason: "referenced plan geometry unexpectedly owns code-block payload bytes", + }); + } + let refinement_len = payload.refinement.map_or(0, |range| range.length); + if usize::try_from(block.cleanup_length).ok() != Some(payload.cleanup.length) + || usize::try_from(block.refinement_length).ok() != Some(refinement_len) + { + return Err(Error::MetalKernel { + message: "HTJ2K referenced payload lengths do not match code-block geometry" + .to_string(), + }); + } + let coded_offset = u32::try_from(logical_coded_len).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect referenced coded payload exceeds u32".to_string(), + })?; + referenced_codestream_slice(input, payload.cleanup)?; + if let Some(refinement) = payload.refinement { + referenced_codestream_slice(input, refinement)?; + } + let block_coded_len = payload + .cleanup + .length + .checked_add(refinement_len) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K referenced code-block payload length overflow".to_string(), + })?; + logical_coded_len = logical_coded_len + .checked_add(block_coded_len) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K referenced prepared payload length overflow".to_string(), + })?; + ranges.push(*payload); + jobs.push(J2kHtCleanupBatchJob { + coded_offset, + width: block.width, + height: block.height, + coded_len: u32::try_from(block_coded_len).map_err(|_| Error::MetalKernel { + message: "HTJ2K referenced code-block payload exceeds u32".to_string(), + })?, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + missing_msbs: u32::from(block.missing_bit_planes), + num_bitplanes: u32::from(block.num_bitplanes), + roi_shift: u32::from(block.roi_shift), + number_of_coding_passes: u32::from(block.number_of_coding_passes), + output_stride: job.width, + output_offset: block + .output_y + .checked_mul(job.width) + .and_then(|row| row.checked_add(block.output_x)) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K referenced output offset overflow".to_string(), + })?, + dequantization_step: block.dequantization_step, + stripe_causal: u32::from(block.stripe_causal), + }); + } + if logical_coded_len != coded_len { + return Err(Error::MetalStateInvariant { + state: "HTJ2K referenced prepared sub-band", + reason: "validated payload lengths do not match the planned logical arena", + }); + } + *payload_cursor = payload_end; + + Ok(PreparedHtSubBand { + band_id: job.band_id, + width: job.width, + height: job.height, + payload_source: PreparedHtPayloadSource::Referenced { + input: input.clone(), + ranges, + }, + jobs, + execution_owner: Arc::new(PreparedHtExecutionOwner), + }) +} + +#[cfg(target_os = "macos")] +fn referenced_codestream_slice( + codestream: &[u8], + range: J2kCodestreamRange, +) -> Result<&[u8], Error> { + let end = range.end().ok_or_else(|| Error::MetalKernel { + message: "HTJ2K referenced payload range overflows usize".to_string(), + })?; + codestream + .get(range.offset..end) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K referenced payload range exceeds retained codestream".to_string(), + }) +} + +#[cfg(target_os = "macos")] +pub(in crate::compute) fn prepare_ht_sub_band_groups( + steps: &[PreparedDirectGrayscaleStep], + tier1_prepare_mode: DirectTier1Mode, +) -> Result, Error> { + prepare_sub_band_groups( + steps, + tier1_prepare_mode, + |step| match step { + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => Some(sub_band), + _ => None, + }, + prepare_ht_sub_band_group, + ) +} + +#[cfg(target_os = "macos")] +#[expect( + clippy::too_many_lines, + reason = "group assembly keeps payload ownership, rebased offsets, and arena lengths aligned" +)] +pub(super) fn prepare_ht_sub_band_group( + start_step: usize, + end_step: usize, + sub_bands: &[&PreparedHtSubBand], + _tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let job_count = crate::batch_allocation::checked_count_sum( + sub_bands.iter().map(|sub_band| sub_band.jobs.len()), + "HTJ2K MetalDirect grouped jobs", + )?; + let coded_len = crate::batch_allocation::checked_count_sum( + sub_bands + .iter() + .flat_map(|sub_band| sub_band.jobs.iter()) + .map(|job| job.coded_len as usize), + "HTJ2K MetalDirect grouped coded payload", + )?; + let mut budget = crate::batch_allocation::BatchMetadataBudget::new( + "HTJ2K MetalDirect prepared sub-band group", + ); + let mut members = budget.try_vec(sub_bands.len(), "HTJ2K MetalDirect grouped members")?; + let mut jobs = budget.try_vec(job_count, "HTJ2K MetalDirect grouped jobs")?; + let first = sub_bands.first().ok_or(Error::MetalStateInvariant { + state: "HTJ2K MetalDirect prepared sub-band group", + reason: "group preparation received no sub-bands", + })?; + let mut payload_source = match &first.payload_source { + PreparedHtPayloadSource::Contiguous(_) => PreparedHtPayloadSource::Contiguous( + budget.try_vec(coded_len, "HTJ2K MetalDirect grouped coded payload")?, + ), + PreparedHtPayloadSource::Referenced { input, .. } => PreparedHtPayloadSource::Referenced { + input: input.clone(), + ranges: budget.try_vec( + job_count, + "HTJ2K MetalDirect grouped referenced payload ranges", + )?, + }, + }; + let mut output_base = 0usize; + let mut logical_coded_base = 0usize; + + for sub_band in sub_bands { + members.push(PreparedHtSubBandGroupMember { + band_id: sub_band.band_id, + offset_elements: output_base, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + + let coded_base = u32::try_from(logical_coded_base).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped coded payload exceeds u32".to_string(), + })?; + let output_base_u32 = u32::try_from(output_base).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped coefficient arena exceeds u32".to_string(), + })?; + for job in &sub_band.jobs { + let mut grouped_job = *job; + grouped_job.coded_offset = + coded_base + .checked_add(job.coded_offset) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped coded offset overflow".to_string(), + })?; + grouped_job.output_offset = + output_base_u32 + .checked_add(job.output_offset) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped output offset overflow".to_string(), + })?; + jobs.push(grouped_job); + } + match (&mut payload_source, &sub_band.payload_source) { + ( + PreparedHtPayloadSource::Contiguous(grouped), + PreparedHtPayloadSource::Contiguous(source), + ) => grouped.extend_from_slice(source), + ( + PreparedHtPayloadSource::Referenced { + input: grouped_input, + ranges: grouped_ranges, + }, + PreparedHtPayloadSource::Referenced { + input: source_input, + ranges: source_ranges, + }, + ) => { + if !Arc::ptr_eq(grouped_input, source_input) { + return Err(Error::MetalStateInvariant { + state: "HTJ2K MetalDirect referenced sub-band group", + reason: "grouped sub-bands reference different encoded input owners", + }); + } + if source_ranges.len() != sub_band.jobs.len() { + return Err(Error::MetalStateInvariant { + state: "HTJ2K MetalDirect referenced sub-band group", + reason: "payload range count does not match grouped job count", + }); + } + grouped_ranges.extend_from_slice(source_ranges); + } + _ => { + return Err(Error::MetalStateInvariant { + state: "HTJ2K MetalDirect prepared sub-band group", + reason: "group mixes contiguous and referenced payload ownership", + }); + } + } + logical_coded_base = sub_band + .jobs + .iter() + .try_fold(logical_coded_base, |total, job| { + total + .checked_add(job.coded_len as usize) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped coded payload overflow".to_string(), + }) + })?; + let sub_band_len = + sub_band + .width + .checked_mul(sub_band.height) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped sub-band size overflow".to_string(), + })? as usize; + output_base = output_base + .checked_add(sub_band_len) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped coefficient arena overflow".to_string(), + })?; + } + if logical_coded_base != coded_len { + return Err(Error::MetalStateInvariant { + state: "HTJ2K MetalDirect prepared sub-band group", + reason: "grouped logical payload length does not match its planned arena", + }); + } + match &payload_source { + PreparedHtPayloadSource::Contiguous(data) if data.len() != coded_len => { + return Err(Error::MetalStateInvariant { + state: "HTJ2K MetalDirect prepared sub-band group", + reason: "grouped contiguous payload length does not match its jobs", + }); + } + PreparedHtPayloadSource::Referenced { ranges, .. } if ranges.len() != job_count => { + return Err(Error::MetalStateInvariant { + state: "HTJ2K MetalDirect referenced sub-band group", + reason: "grouped payload range count does not match its jobs", + }); + } + PreparedHtPayloadSource::Contiguous(_) | PreparedHtPayloadSource::Referenced { .. } => {} + } + + Ok(PreparedHtSubBandGroup { + start_step, + end_step, + total_coefficients: output_base, + payload_source, + jobs, + members, + execution_owner: Arc::new(PreparedHtExecutionOwner), + }) +} diff --git a/crates/j2k-metal/src/compute/direct_prepare/referenced.rs b/crates/j2k-metal/src/compute/direct_prepare/referenced.rs new file mode 100644 index 00000000..03682b93 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_prepare/referenced.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Shared retained-plan traversal and component assembly. + +use super::{ + prepare_classic_sub_band_groups, prepare_ht_sub_band_groups, + prepare_referenced_classic_sub_band, prepare_referenced_ht_sub_band, Arc, BandRequiredRegion, + CpuTier1CoefficientCache, DirectTier1Mode, Error, HtCodeBlockPayloadRanges, + J2kDirectGrayscaleStep, NativeGrayscalePlan, PreparedDirectGrayscalePlan, + PreparedDirectGrayscaleStep, PreparedDirectIdwt, ReferencedClassicPayloadCursor, +}; + +pub(super) fn validate_payload_record_span( + span: j2k_native::J2kReferencedPayloadRecordSpan, + cursor: usize, + payload_count: usize, + state: &'static str, +) -> Result { + if span.first_record != cursor { + return Err(Error::MetalStateInvariant { + state, + reason: "tile payload-record spans are not contiguous in geometry traversal order", + }); + } + let end = span.end_record().ok_or(Error::MetalStateInvariant { + state, + reason: "tile payload-record span overflowed", + })?; + if end > payload_count { + return Err(Error::MetalStateInvariant { + state, + reason: "tile payload-record span exceeds the retained payload table", + }); + } + Ok(end) +} + +#[cfg(target_os = "macos")] +pub(super) fn validate_referenced_component_metadata( + first: &NativeGrayscalePlan, + component: &NativeGrayscalePlan, +) -> Result<(), Error> { + if component.dimensions != first.dimensions || component.bit_depth != first.bit_depth { + return Err(Error::MetalStateInvariant { + state: "referenced multi-tile component metadata", + reason: "tile component dimensions or bit depth changed within one image", + }); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +#[cfg(target_os = "macos")] +pub(super) fn append_referenced_classic_component_steps( + geometry: &NativeGrayscalePlan, + payloads: &mut ReferencedClassicPayloadCursor<'_>, + steps: &mut Vec, + _allocation_context: &'static str, +) -> Result<(), Error> { + for step in &geometry.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { + steps.push(PreparedDirectGrayscaleStep::ClassicSubBand( + prepare_referenced_classic_sub_band(sub_band, payloads)?, + )); + } + J2kDirectGrayscaleStep::HtSubBand(_) => { + return Err(Error::MetalStateInvariant { + state: "classic J2K referenced direct plan", + reason: "classic referenced plan contains an HT code-block step", + }); + } + J2kDirectGrayscaleStep::Idwt(idwt) => { + steps.push(PreparedDirectGrayscaleStep::Idwt(PreparedDirectIdwt { + step: *idwt, + output_window: BandRequiredRegion::full(idwt.rect.width(), idwt.rect.height()), + })); + } + J2kDirectGrayscaleStep::Store(store) => { + steps.push(PreparedDirectGrayscaleStep::Store(*store)); + } + } + } + Ok(()) +} + +#[cfg(target_os = "macos")] +pub(super) fn finish_referenced_component_plan( + dimensions: (u32, u32), + bit_depth: u8, + steps: Vec, + _allocation_context: &'static str, +) -> Result { + let tier1_prepare_mode = DirectTier1Mode::Metal; + let classic_groups = prepare_classic_sub_band_groups(&steps, tier1_prepare_mode)?; + let ht_groups = prepare_ht_sub_band_groups(&steps, tier1_prepare_mode)?; + Ok(PreparedDirectGrayscalePlan { + dimensions, + bit_depth, + tier1_prepare_mode, + steps, + classic_groups, + ht_groups, + cpu_tier1_cache: Arc::new(CpuTier1CoefficientCache::default()), + }) +} + +#[cfg(target_os = "macos")] +pub(super) fn append_referenced_htj2k_component_steps( + geometry: &NativeGrayscalePlan, + input: &Arc<[u8]>, + payloads: &[HtCodeBlockPayloadRanges], + payload_cursor: &mut usize, + steps: &mut Vec, +) -> Result<(), Error> { + for step in &geometry.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(_) => { + return Err(Error::MetalStateInvariant { + state: "HTJ2K referenced direct plan", + reason: "HTJ2K referenced plan contains a classic code-block step", + }); + } + J2kDirectGrayscaleStep::HtSubBand(sub_band) => { + steps.push(PreparedDirectGrayscaleStep::HtSubBand( + prepare_referenced_ht_sub_band(sub_band, input, payloads, payload_cursor)?, + )); + } + J2kDirectGrayscaleStep::Idwt(idwt) => { + steps.push(PreparedDirectGrayscaleStep::Idwt(PreparedDirectIdwt { + step: *idwt, + output_window: BandRequiredRegion::full(idwt.rect.width(), idwt.rect.height()), + })); + } + J2kDirectGrayscaleStep::Store(store) => { + steps.push(PreparedDirectGrayscaleStep::Store(*store)); + } + } + } + Ok(()) +} diff --git a/crates/j2k-metal/src/compute/direct_roi.rs b/crates/j2k-metal/src/compute/direct_roi.rs index 7daebb00..b52aa615 100644 --- a/crates/j2k-metal/src/compute/direct_roi.rs +++ b/crates/j2k-metal/src/compute/direct_roi.rs @@ -1,15 +1,43 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use super::abi::{ + J2kClassicCleanupBatchJob, J2kHtCleanupBatchJob, J2kIdwtSingleDecompositionParams, + J2kRepeatedIdwtSingleDecompositionParams, +}; +use super::direct_prepare::{prepare_classic_sub_band_groups, prepare_ht_sub_band_groups}; use super::{ - copied_slice_buffer, idwt_required_input_windows, idwt_required_output_margin, - prepare_classic_sub_band_groups, prepare_ht_sub_band_groups, - prepare_ungrouped_ht_sub_band_buffers, with_runtime, DirectBandSlice, DirectTier1Mode, Error, - J2kClassicCleanupBatchJob, J2kDirectBandId, J2kDirectIdwtStep, J2kDirectStoreStep, - J2kHtCleanupBatchJob, J2kIdwtSingleDecompositionParams, - J2kRepeatedIdwtSingleDecompositionParams, J2kRequiredBandRegion, PreparedDirectGrayscalePlan, + copied_slice_buffer, idwt_required_input_windows, idwt_required_output_margin, with_runtime, + DirectBandSlice, DirectTier1Mode, Error, J2kDirectBandId, J2kDirectIdwtStep, + J2kDirectStoreStep, J2kRequiredBandRegion, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, PreparedDirectIdwt, PreparedHtSubBand, Rect, }; +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct CheckedF32Span { + pub(super) elements: usize, + pub(super) bytes: usize, +} + +#[cfg(target_os = "macos")] +pub(super) fn checked_f32_span( + width: usize, + height: usize, + context: &str, +) -> Result { + let elements = width + .checked_mul(height) + .ok_or_else(|| Error::MetalKernel { + message: format!("{context} element count overflow"), + })?; + let bytes = elements + .checked_mul(std::mem::size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: format!("{context} byte count overflow"), + })?; + Ok(CheckedF32Span { elements, bytes }) +} + #[cfg(target_os = "macos")] pub(crate) fn crop_prepared_direct_grayscale_plan_to_output_region( plan: &mut PreparedDirectGrayscalePlan, @@ -190,11 +218,6 @@ pub(super) fn prune_prepared_direct_grayscale_plan_to_store_windows( apply_prepared_direct_idwt_output_windows(plan, &idwt_outputs)?; plan.classic_groups = prepare_classic_sub_band_groups(&plan.steps, plan.tier1_prepare_mode)?; plan.ht_groups = prepare_ht_sub_band_groups(&plan.steps, plan.tier1_prepare_mode)?; - prepare_ungrouped_ht_sub_band_buffers( - &mut plan.steps, - &plan.ht_groups, - plan.tier1_prepare_mode, - )?; Ok(()) } @@ -350,8 +373,13 @@ pub(super) struct PreparedIdwtInputStrides { } #[cfg(target_os = "macos")] -pub(super) fn prepared_idwt_output_len(idwt: &PreparedDirectIdwt) -> usize { - idwt.output_window.width() as usize * idwt.output_window.height() as usize +pub(super) fn prepared_idwt_output_len(idwt: &PreparedDirectIdwt) -> Result { + checked_f32_span( + idwt.output_window.width() as usize, + idwt.output_window.height() as usize, + "J2K MetalDirect IDWT output", + ) + .map(|span| span.elements) } #[cfg(target_os = "macos")] @@ -489,6 +517,14 @@ pub(super) fn compact_ht_sub_band_coded_data( sub_band: &mut PreparedHtSubBand, _tier1_prepare_mode: DirectTier1Mode, ) -> Result<(), Error> { + let coded_data = + sub_band + .payload_source + .contiguous_mut() + .ok_or(Error::MetalStateInvariant { + state: "HTJ2K MetalDirect cropped plan", + reason: "post-prepare ROI compaction requires a legacy contiguous payload plan", + })?; let compacted_len = crate::batch_allocation::checked_count_sum( sub_band.jobs.iter().map(|job| job.coded_len as usize), "HTJ2K MetalDirect cropped coded payload", @@ -503,7 +539,7 @@ pub(super) fn compact_ht_sub_band_coded_data( .ok_or_else(|| Error::MetalKernel { message: "HTJ2K MetalDirect cropped coded payload range overflow".to_string(), })?; - if end > sub_band.coded_data.len() { + if end > coded_data.len() { return Err(Error::MetalKernel { message: "HTJ2K MetalDirect cropped coded payload range out of bounds".to_string(), }); @@ -513,7 +549,7 @@ pub(super) fn compact_ht_sub_band_coded_data( "HTJ2K MetalDirect cropped coded payload", ); let mut compacted = budget.try_vec(compacted_len, "HTJ2K MetalDirect cropped coded payload")?; - let previous = std::mem::take(&mut sub_band.coded_data); + let previous = std::mem::take(coded_data); for job in &mut sub_band.jobs { let start = job.coded_offset as usize; @@ -525,9 +561,7 @@ pub(super) fn compact_ht_sub_band_coded_data( compacted.extend_from_slice(&previous[start..end]); } - sub_band.coded_data = compacted; - sub_band.coded_buffer = None; - sub_band.jobs_buffer = None; + *coded_data = compacted; Ok(()) } @@ -597,3 +631,36 @@ pub(super) fn crop_direct_store_step_to_output_region( store.output_y = intersection.1 - region_bounds.1; Ok(()) } + +#[cfg(all(test, target_os = "macos"))] +mod span_tests { + use std::mem::size_of; + + use super::*; + + #[test] + fn non_stacked_f32_span_accepts_exact_byte_boundary_and_rejects_one_over() { + let exact_elements = usize::MAX / size_of::(); + let exact = checked_f32_span(exact_elements, 1, "J2K MetalDirect non-stacked test span") + .expect("largest exactly representable f32 span"); + assert_eq!(exact.elements, exact_elements); + assert_eq!(exact.bytes, exact_elements * size_of::()); + + assert!(matches!( + checked_f32_span( + exact_elements + 1, + 1, + "J2K MetalDirect non-stacked test span", + ), + Err(Error::MetalKernel { message }) if message.contains("byte count overflow") + )); + assert!(matches!( + checked_f32_span( + usize::MAX, + 2, + "J2K MetalDirect non-stacked test span", + ), + Err(Error::MetalKernel { message }) if message.contains("element count overflow") + )); + } +} diff --git a/crates/j2k-metal/src/compute/direct_scratch.rs b/crates/j2k-metal/src/compute/direct_scratch.rs index a675a475..a45edc00 100644 --- a/crates/j2k-metal/src/compute/direct_scratch.rs +++ b/crates/j2k-metal/src/compute/direct_scratch.rs @@ -25,10 +25,15 @@ pub(super) fn recycle_scratch_buffers( runtime: &MetalRuntime, scratch_buffers: Vec, ) -> Result<(), Error> { + let mut first_error = None; for scratch in scratch_buffers { - runtime.recycle_private_buffer(scratch.buffer)?; + if let Err(error) = runtime.recycle_private_buffer(scratch.buffer) { + if first_error.is_none() { + first_error = Some(error); + } + } } - Ok(()) + first_error.map_or(Ok(()), Err) } pub(super) fn take_recyclable_private_buffer( diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch.rs b/crates/j2k-metal/src/compute/direct_stacked_batch.rs index e581cd14..01f1cdf9 100644 --- a/crates/j2k-metal/src/compute/direct_stacked_batch.rs +++ b/crates/j2k-metal/src/compute/direct_stacked_batch.rs @@ -3,25 +3,29 @@ use std::sync::Arc; use std::time::Instant; -use metal::{Buffer, CommandBufferRef}; +use metal::{Buffer, CommandBufferRef, ComputeCommandEncoderRef}; +use super::decode_dispatch::mct::dispatch_inverse_mct_buffers_in_command_buffer; +use super::direct_grayscale_execute::{ + checked_coefficient_len, encode_prepared_direct_component_plane_in_command_buffer, + DirectComponentPlaneRequest, +}; use super::{ - build_flattened_cpu_tier1_cache, dispatch_inverse_mct_buffers_in_command_buffer, elapsed_us, + build_flattened_cpu_tier1_cache, elapsed_us, encode_batched_mct_rgb8_to_surfaces_in_command_buffer, encode_mct_rgb8_to_surface_in_command_buffer, encode_plane_stage_to_surface_in_command_buffer, - encode_prepared_direct_component_plane_in_command_buffer, encode_repeated_mct_rgb8_to_surfaces_in_command_buffer, flattened_hybrid_cpu_tier1_enabled, - metal_profile_stages_enabled, repeated_shared_direct_color_plan_count, + metal_profile_stages_enabled, record_hybrid_stacked_component_batch, + record_stacked_component_batch, repeated_shared_direct_color_plan_count, should_flatten_hybrid_cpu_tier1_color_batch, DirectColorBatchCommandBuffers, - DirectComponentPlaneRequest, DirectHybridStageTimings, DirectScratchBuffer, DirectStatusCheck, - DirectTier1Mode, Error, FlattenedCpuTier1Cache, MetalRuntime, NativeColorSpace, PixelFormat, - PlaneStage, PreparedDirectColorPlan, PreparedDirectGrayscalePlan, Surface, + DirectHybridStageTimings, DirectScratchBuffer, DirectStatusCheck, DirectTier1Mode, Error, + FlattenedCpuTier1Cache, MetalRuntime, NativeColorSpace, PixelFormat, PlaneStage, + PreparedDirectColorPlan, PreparedDirectGrayscalePlan, Surface, }; mod command_submission; mod repeated_grayscale; mod resources; -mod result; mod validation; use self::command_submission::submit_stacked_component_commands; @@ -33,7 +37,6 @@ pub(super) use self::resources::{ lookup_direct_band_slice, lookup_direct_band_slice_entry, lookup_repeated_direct_band_layout_entry, DirectBandSlice, }; -use self::result::assemble_stacked_component_result; use self::validation::plan_stacked_component_batch; pub(super) use self::validation::supports_stacked_direct_component_plane_batch; @@ -112,7 +115,11 @@ pub(super) fn encode_prepared_direct_color_plan_in_command_buffer( } if plan.mct { - let len = plan.dimensions.0 as usize * plan.dimensions.1 as usize; + let len = checked_coefficient_len( + plan.dimensions.0, + plan.dimensions.1, + "J2K MetalDirect color MCT plane span overflow", + )?; let encode_started = metal_profile_stages_enabled().then(Instant::now); status_checks.push(dispatch_inverse_mct_buffers_in_command_buffer( runtime, @@ -256,6 +263,7 @@ pub(super) fn try_encode_stacked_mct_rgb8_direct_color_batch( StackedDirectComponentPlaneBatchRequest { runtime, command_buffers, + compute_encoder: None, plans: &component_plan_refs, component_idx, flattened_cpu_tier1_cache: flattened_cpu_tier1_cache.as_ref(), @@ -312,6 +320,7 @@ pub(super) fn try_encode_stacked_mct_rgb8_direct_color_batch( pub(super) struct StackedDirectComponentPlaneBatchRequest<'a, 'p> { pub(super) runtime: &'a MetalRuntime, pub(super) command_buffers: DirectColorBatchCommandBuffers<'a>, + pub(super) compute_encoder: Option<&'a ComputeCommandEncoderRef>, pub(super) plans: &'a [&'p PreparedDirectGrayscalePlan], pub(super) component_idx: usize, pub(super) flattened_cpu_tier1_cache: Option<&'a FlattenedCpuTier1Cache>, @@ -330,5 +339,14 @@ pub(super) fn encode_stacked_direct_component_plane_batch( let plan = plan_stacked_component_batch(request.plans, tier1_mode)?; let mut resources = prepare_stacked_component_resources(plan.count, plan.first.steps.len())?; submit_stacked_component_commands(request, &plan, &mut resources)?; - assemble_stacked_component_result(resources, &plan, tier1_mode) + let final_plane = resources.final_plane.ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect color component batch did not produce a final plane".to_string(), + })?; + record_stacked_component_batch(); + record_hybrid_stacked_component_batch(tier1_mode); + Ok(StackedDirectComponentPlane { + buffer: final_plane.buffer, + dimensions: plan.first.dimensions, + count: plan.count, + }) } diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission.rs index 1e2bfa29..ba307469 100644 --- a/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission.rs +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission.rs @@ -1,41 +1,27 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::mem::size_of; +use metal::{Buffer, ComputeCommandEncoderRef}; -use metal::Buffer; - -use crate::batch_allocation::{BatchMetadataBudget, BatchMetadataRequest}; +use crate::batch_allocation::BatchMetadataBudget; use super::super::{ - decode_classic_inputs_on_cpu_with_plan_cache, decode_ht_inputs_on_cpu_with_plan_cache, - direct_preflight_invariant, - dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets, - dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets, - dispatch_store_component_repeated_in_command_buffer, elapsed_us, - encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer, - encode_distinct_classic_sub_bands_to_buffer_in_command_buffer, - encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer, - encode_distinct_ht_sub_bands_to_buffer_in_command_buffer, idwt_input_windows_from_slices, - metal_profile_stages_enabled, prepared_idwt_output_len, prepared_idwt_params, - repeated_idwt_params, take_f32_scratch_buffer, upload_cpu_decoded_coefficients, - BandRequiredRegion, ClassicCpuDecodeInput, CpuTier1DecodeSubstageCounters, - DirectColorBatchCommandBuffers, DirectHybridStageTimings, DirectScratchBuffer, - DirectStatusCheck, DirectTier1Mode, Error, FlattenedCpuTier1Cache, HtCpuDecodeInput, - IdwtSubBandBuffers, Instant, J2kRepeatedStoreParams, J2kWaveletTransform, MetalRuntime, - PreparedClassicSubBand, PreparedClassicSubBandGroup, PreparedDirectGrayscalePlan, - PreparedDirectGrayscaleStep, PreparedDirectIdwt, PreparedHtSubBand, PreparedHtSubBandGroup, - PreparedIdwtInputStrides, RepeatedIdwtDispatch, SingleIdwtDispatch, -}; -use super::resources::{ - lookup_direct_band_slice_entry, lookup_repeated_direct_band_layout_entry, - retain_metal_tier1_output, DirectBandSlice, StackedComponentResources, + metal_profile_stages_enabled, DirectColorBatchCommandBuffers, DirectHybridStageTimings, + DirectScratchBuffer, DirectStatusCheck, DirectTier1Mode, Error, FlattenedCpuTier1Cache, + MetalRuntime, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, }; +use super::resources::StackedComponentResources; use super::validation::StackedComponentBatchPlan; use super::StackedDirectComponentPlaneBatchRequest; +mod classic_tier1; +mod final_store; +mod ht_tier1; +mod reconstruction; + struct SubmissionContext<'a, 'p, 'r> { runtime: &'a MetalRuntime, command_buffers: DirectColorBatchCommandBuffers<'a>, + compute_encoder: Option<&'a ComputeCommandEncoderRef>, plans: &'a [&'p PreparedDirectGrayscalePlan], component_idx: usize, flattened_cpu_tier1_cache: Option<&'a FlattenedCpuTier1Cache>, @@ -85,6 +71,7 @@ pub(super) fn submit_stacked_component_commands<'p>( let StackedDirectComponentPlaneBatchRequest { runtime, command_buffers, + compute_encoder, plans, component_idx, flattened_cpu_tier1_cache, @@ -98,6 +85,7 @@ pub(super) fn submit_stacked_component_commands<'p>( let mut context = SubmissionContext { runtime, command_buffers, + compute_encoder, plans, component_idx, flattened_cpu_tier1_cache, @@ -145,737 +133,3 @@ pub(super) fn submit_stacked_component_commands<'p>( Ok(()) } - -impl SubmissionContext<'_, '_, '_> { - fn submit_classic_group( - &mut self, - first: &PreparedDirectGrayscalePlan, - step_idx: usize, - group: &PreparedClassicSubBandGroup, - ) -> Result<(), Error> { - let input_count = planned_cpu_input_count( - self.tier1_mode, - self.flattened_cpu_tier1_cache.is_some(), - self.broadcast_tier1_inputs, - self.plans.len(), - ); - let mut metadata_budget = - BatchMetadataBudget::new("J2K Metal stacked classic group submission metadata"); - metadata_budget.preflight(&[ - BatchMetadataRequest::of::<&PreparedClassicSubBandGroup>(self.plans.len()), - BatchMetadataRequest::of::>(input_count), - ])?; - let groups = try_collect_submission_items( - &mut metadata_budget, - self.plans.iter().map(|plan| { - plan.classic_group_starting_at(step_idx).ok_or_else(|| { - direct_preflight_invariant( - "classic group step mismatch in stacked component batch", - ) - }) - }), - "J2K Metal stacked classic group references", - )?; - let buffer = match self.tier1_mode { - DirectTier1Mode::Metal => { - let output = - take_f32_scratch_buffer(self.runtime, group.total_coefficients * self.count)?; - let (buffers, status_check) = - encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer( - self.runtime, - self.command_buffers.default, - &groups, - &output.buffer, - self.scratch_buffers, - )?; - retain_metal_tier1_output( - output, - buffers, - status_check, - self.retained_buffers, - self.status_checks, - self.scratch_buffers, - ) - } - DirectTier1Mode::CpuUpload => self.prepare_classic_group_cpu_buffer( - first, - step_idx, - group, - &groups, - &mut metadata_budget, - )?, - }; - - let stride_bytes = group.total_coefficients * size_of::(); - for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { - let source_group = if self.broadcast_tier1_inputs { - groups[0] - } else { - groups[instance_idx] - }; - let instance_offset = if self.broadcast_tier1_inputs { - 0 - } else { - instance_idx * stride_bytes - }; - for member in &source_group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: buffer.clone(), - offset_bytes: instance_offset + member.offset_elements * size_of::(), - window: member.window, - }); - } - } - Ok(()) - } - - fn prepare_classic_group_cpu_buffer( - &mut self, - first: &PreparedDirectGrayscalePlan, - step_idx: usize, - group: &PreparedClassicSubBandGroup, - groups: &[&PreparedClassicSubBandGroup], - metadata_budget: &mut BatchMetadataBudget, - ) -> Result { - let input_groups = if self.broadcast_tier1_inputs { - &groups[..1] - } else { - groups - }; - if let Some(cache) = self.flattened_cpu_tier1_cache { - return cache.buffer_for( - self.component_idx, - step_idx, - group.total_coefficients, - input_groups.len(), - ); - } - - let inputs = try_collect_submission_items( - metadata_budget, - input_groups.iter().map(|group| { - Ok(ClassicCpuDecodeInput { - coded_data: &group.coded_data, - segments: &group.segments, - jobs: &group.jobs, - output_len: group.total_coefficients, - }) - }), - "J2K Metal stacked classic group CPU inputs", - )?; - let decode_started = self.profile_stages.then(Instant::now); - let cpu_tier1_counters = self - .profile_stages - .then(CpuTier1DecodeSubstageCounters::default); - let coefficients = decode_classic_inputs_on_cpu_with_plan_cache( - first, - step_idx, - &inputs, - cpu_tier1_counters.as_ref(), - )?; - if let Some(started) = decode_started { - self.stage_timings.cpu_tier1 += elapsed_us(started); - } - if let Some(counters) = &cpu_tier1_counters { - counters.add_to_stage_timings(self.stage_timings); - } - let upload_started = self.profile_stages.then(Instant::now); - let buffer = - upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; - if let Some(started) = upload_started { - self.stage_timings.coefficient_upload += elapsed_us(started); - } - Ok(buffer) - } - - fn submit_ht_group( - &mut self, - first: &PreparedDirectGrayscalePlan, - step_idx: usize, - group: &PreparedHtSubBandGroup, - ) -> Result<(), Error> { - let input_count = planned_cpu_input_count( - self.tier1_mode, - self.flattened_cpu_tier1_cache.is_some(), - self.broadcast_tier1_inputs, - self.plans.len(), - ); - let mut metadata_budget = - BatchMetadataBudget::new("J2K Metal stacked HT group submission metadata"); - metadata_budget.preflight(&[ - BatchMetadataRequest::of::<&PreparedHtSubBandGroup>(self.plans.len()), - BatchMetadataRequest::of::>(input_count), - ])?; - let groups = try_collect_submission_items( - &mut metadata_budget, - self.plans.iter().map(|plan| { - plan.ht_group_starting_at(step_idx).ok_or_else(|| { - direct_preflight_invariant("HT group step mismatch in stacked component batch") - }) - }), - "J2K Metal stacked HT group references", - )?; - let buffer = match self.tier1_mode { - DirectTier1Mode::Metal => { - let output = - take_f32_scratch_buffer(self.runtime, group.total_coefficients * self.count)?; - let (buffers, status_check) = - encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer( - self.runtime, - self.command_buffers.default, - &groups, - &output.buffer, - )?; - retain_metal_tier1_output( - output, - buffers, - status_check, - self.retained_buffers, - self.status_checks, - self.scratch_buffers, - ) - } - DirectTier1Mode::CpuUpload => self.prepare_ht_group_cpu_buffer( - first, - step_idx, - group, - &groups, - &mut metadata_budget, - )?, - }; - - let stride_bytes = group.total_coefficients * size_of::(); - for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { - let source_group = if self.broadcast_tier1_inputs { - groups[0] - } else { - groups[instance_idx] - }; - let instance_offset = if self.broadcast_tier1_inputs { - 0 - } else { - instance_idx * stride_bytes - }; - for member in &source_group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: buffer.clone(), - offset_bytes: instance_offset + member.offset_elements * size_of::(), - window: member.window, - }); - } - } - Ok(()) - } - - fn prepare_ht_group_cpu_buffer( - &mut self, - first: &PreparedDirectGrayscalePlan, - step_idx: usize, - group: &PreparedHtSubBandGroup, - groups: &[&PreparedHtSubBandGroup], - metadata_budget: &mut BatchMetadataBudget, - ) -> Result { - let input_groups = if self.broadcast_tier1_inputs { - &groups[..1] - } else { - groups - }; - if let Some(cache) = self.flattened_cpu_tier1_cache { - return cache.buffer_for( - self.component_idx, - step_idx, - group.total_coefficients, - input_groups.len(), - ); - } - - let inputs = try_collect_submission_items( - metadata_budget, - input_groups.iter().map(|group| { - Ok(HtCpuDecodeInput { - coded_data: &group.coded_arena.data, - jobs: &group.jobs, - output_len: group.total_coefficients, - }) - }), - "J2K Metal stacked HT group CPU inputs", - )?; - let decode_started = self.profile_stages.then(Instant::now); - let cpu_tier1_counters = self - .profile_stages - .then(CpuTier1DecodeSubstageCounters::default); - let coefficients = decode_ht_inputs_on_cpu_with_plan_cache( - first, - step_idx, - &inputs, - cpu_tier1_counters.as_ref(), - )?; - if let Some(started) = decode_started { - self.stage_timings.cpu_tier1 += elapsed_us(started); - } - if let Some(counters) = &cpu_tier1_counters { - counters.add_to_stage_timings(self.stage_timings); - } - let upload_started = self.profile_stages.then(Instant::now); - let buffer = - upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; - if let Some(started) = upload_started { - self.stage_timings.coefficient_upload += elapsed_us(started); - } - Ok(buffer) - } - - fn submit_classic_sub_band( - &mut self, - first: &PreparedDirectGrayscalePlan, - step_idx: usize, - sub_band: &PreparedClassicSubBand, - ) -> Result<(), Error> { - let input_count = planned_cpu_input_count( - self.tier1_mode, - self.flattened_cpu_tier1_cache.is_some(), - self.broadcast_tier1_inputs, - self.plans.len(), - ); - let mut metadata_budget = - BatchMetadataBudget::new("J2K Metal stacked classic sub-band submission metadata"); - metadata_budget.preflight(&[ - BatchMetadataRequest::of::<&PreparedClassicSubBand>(self.plans.len()), - BatchMetadataRequest::of::>(input_count), - ])?; - let sub_bands = try_collect_submission_items( - &mut metadata_budget, - self.plans - .iter() - .map(|plan| match plan.steps.get(step_idx) { - Some(PreparedDirectGrayscaleStep::ClassicSubBand(other)) => Ok(other), - _ => Err(direct_preflight_invariant( - "classic sub-band step mismatch in stacked component batch", - )), - }), - "J2K Metal stacked classic sub-band references", - )?; - let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let buffer = match self.tier1_mode { - DirectTier1Mode::Metal => { - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - let (buffers, status_check) = - encode_distinct_classic_sub_bands_to_buffer_in_command_buffer( - self.runtime, - self.command_buffers.default, - &sub_bands, - &output.buffer, - self.scratch_buffers, - )?; - retain_metal_tier1_output( - output, - buffers, - status_check, - self.retained_buffers, - self.status_checks, - self.scratch_buffers, - ) - } - DirectTier1Mode::CpuUpload => self.prepare_classic_sub_band_cpu_buffer( - first, - step_idx, - per_instance_len, - &sub_bands, - &mut metadata_budget, - )?, - }; - - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { - let source_sub_band = if self.broadcast_tier1_inputs { - sub_bands[0] - } else { - sub_bands[instance_idx] - }; - let instance_offset = if self.broadcast_tier1_inputs { - 0 - } else { - instance_idx * stride_bytes - }; - bands.push(DirectBandSlice { - band_id: source_sub_band.band_id, - buffer: buffer.clone(), - offset_bytes: instance_offset, - window: BandRequiredRegion::full(source_sub_band.width, source_sub_band.height), - }); - } - Ok(()) - } - - fn prepare_classic_sub_band_cpu_buffer( - &mut self, - first: &PreparedDirectGrayscalePlan, - step_idx: usize, - per_instance_len: usize, - sub_bands: &[&PreparedClassicSubBand], - metadata_budget: &mut BatchMetadataBudget, - ) -> Result { - let input_sub_bands = if self.broadcast_tier1_inputs { - &sub_bands[..1] - } else { - sub_bands - }; - if let Some(cache) = self.flattened_cpu_tier1_cache { - return cache.buffer_for( - self.component_idx, - step_idx, - per_instance_len, - input_sub_bands.len(), - ); - } - - let inputs = try_collect_submission_items( - metadata_budget, - input_sub_bands.iter().map(|sub_band| { - Ok(ClassicCpuDecodeInput { - coded_data: &sub_band.coded_data, - segments: &sub_band.segments, - jobs: &sub_band.jobs, - output_len: per_instance_len, - }) - }), - "J2K Metal stacked classic sub-band CPU inputs", - )?; - let decode_started = self.profile_stages.then(Instant::now); - let cpu_tier1_counters = self - .profile_stages - .then(CpuTier1DecodeSubstageCounters::default); - let coefficients = decode_classic_inputs_on_cpu_with_plan_cache( - first, - step_idx, - &inputs, - cpu_tier1_counters.as_ref(), - )?; - if let Some(started) = decode_started { - self.stage_timings.cpu_tier1 += elapsed_us(started); - } - if let Some(counters) = &cpu_tier1_counters { - counters.add_to_stage_timings(self.stage_timings); - } - let upload_started = self.profile_stages.then(Instant::now); - let buffer = - upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; - if let Some(started) = upload_started { - self.stage_timings.coefficient_upload += elapsed_us(started); - } - Ok(buffer) - } - - fn submit_ht_sub_band( - &mut self, - first: &PreparedDirectGrayscalePlan, - step_idx: usize, - sub_band: &PreparedHtSubBand, - ) -> Result<(), Error> { - let input_count = planned_cpu_input_count( - self.tier1_mode, - self.flattened_cpu_tier1_cache.is_some(), - self.broadcast_tier1_inputs, - self.plans.len(), - ); - let mut metadata_budget = - BatchMetadataBudget::new("J2K Metal stacked HT sub-band submission metadata"); - metadata_budget.preflight(&[ - BatchMetadataRequest::of::<&PreparedHtSubBand>(self.plans.len()), - BatchMetadataRequest::of::>(input_count), - ])?; - let sub_bands = try_collect_submission_items( - &mut metadata_budget, - self.plans - .iter() - .map(|plan| match plan.steps.get(step_idx) { - Some(PreparedDirectGrayscaleStep::HtSubBand(other)) => Ok(other), - _ => Err(direct_preflight_invariant( - "HT sub-band step mismatch in stacked component batch", - )), - }), - "J2K Metal stacked HT sub-band references", - )?; - let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let buffer = match self.tier1_mode { - DirectTier1Mode::Metal => { - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - let (buffers, status_check) = - encode_distinct_ht_sub_bands_to_buffer_in_command_buffer( - self.runtime, - self.command_buffers.default, - &sub_bands, - &output.buffer, - )?; - retain_metal_tier1_output( - output, - buffers, - status_check, - self.retained_buffers, - self.status_checks, - self.scratch_buffers, - ) - } - DirectTier1Mode::CpuUpload => self.prepare_ht_sub_band_cpu_buffer( - first, - step_idx, - per_instance_len, - &sub_bands, - &mut metadata_budget, - )?, - }; - - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { - let source_sub_band = if self.broadcast_tier1_inputs { - sub_bands[0] - } else { - sub_bands[instance_idx] - }; - let instance_offset = if self.broadcast_tier1_inputs { - 0 - } else { - instance_idx * stride_bytes - }; - bands.push(DirectBandSlice { - band_id: source_sub_band.band_id, - buffer: buffer.clone(), - offset_bytes: instance_offset, - window: BandRequiredRegion::full(source_sub_band.width, source_sub_band.height), - }); - } - Ok(()) - } - - fn prepare_ht_sub_band_cpu_buffer( - &mut self, - first: &PreparedDirectGrayscalePlan, - step_idx: usize, - per_instance_len: usize, - sub_bands: &[&PreparedHtSubBand], - metadata_budget: &mut BatchMetadataBudget, - ) -> Result { - let input_sub_bands = if self.broadcast_tier1_inputs { - &sub_bands[..1] - } else { - sub_bands - }; - if let Some(cache) = self.flattened_cpu_tier1_cache { - return cache.buffer_for( - self.component_idx, - step_idx, - per_instance_len, - input_sub_bands.len(), - ); - } - - let inputs = try_collect_submission_items( - metadata_budget, - input_sub_bands.iter().map(|sub_band| { - Ok(HtCpuDecodeInput { - coded_data: &sub_band.coded_data, - jobs: &sub_band.jobs, - output_len: per_instance_len, - }) - }), - "J2K Metal stacked HT sub-band CPU inputs", - )?; - let decode_started = self.profile_stages.then(Instant::now); - let cpu_tier1_counters = self - .profile_stages - .then(CpuTier1DecodeSubstageCounters::default); - let coefficients = decode_ht_inputs_on_cpu_with_plan_cache( - first, - step_idx, - &inputs, - cpu_tier1_counters.as_ref(), - )?; - if let Some(started) = decode_started { - self.stage_timings.cpu_tier1 += elapsed_us(started); - } - if let Some(counters) = &cpu_tier1_counters { - counters.add_to_stage_timings(self.stage_timings); - } - let upload_started = self.profile_stages.then(Instant::now); - let buffer = - upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; - if let Some(started) = upload_started { - self.stage_timings.coefficient_upload += elapsed_us(started); - } - Ok(buffer) - } - - #[expect( - clippy::too_many_lines, - reason = "IDWT command submission keeps scratch lifetimes and stage order explicit" - )] - fn submit_idwt(&mut self, step_idx: usize, idwt: &PreparedDirectIdwt) -> Result<(), Error> { - let per_instance_len = prepared_idwt_output_len(idwt); - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - let encode_started = self.profile_stages.then(Instant::now); - match idwt.step.transform { - J2kWaveletTransform::Reversible53 => { - let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( - &self.resources.band_sets, - idwt.step.ll_band_id, - idwt.step.ll, - )?; - let (hl, high_low_stride) = lookup_repeated_direct_band_layout_entry( - &self.resources.band_sets, - idwt.step.hl_band_id, - idwt.step.hl, - )?; - let (lh, low_high_stride) = lookup_repeated_direct_band_layout_entry( - &self.resources.band_sets, - idwt.step.lh_band_id, - idwt.step.lh, - )?; - let (hh, high_high_stride) = lookup_repeated_direct_band_layout_entry( - &self.resources.band_sets, - idwt.step.hh_band_id, - idwt.step.hh, - )?; - let params = repeated_idwt_params( - idwt, - idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), - PreparedIdwtInputStrides { - ll: low_low_stride, - hl: high_low_stride, - lh: low_high_stride, - hh: high_high_stride, - }, - self.count, - "color", - )?; - dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( - self.command_buffers.idwt, - RepeatedIdwtDispatch { - runtime: self.runtime, - sub_bands: IdwtSubBandBuffers { - ll: &ll.buffer, - ll_offset: ll.offset_bytes, - hl: &hl.buffer, - hl_offset: hl.offset_bytes, - lh: &lh.buffer, - lh_offset: lh.offset_bytes, - hh: &hh.buffer, - hh_offset: hh.offset_bytes, - }, - params, - decoded: &output.buffer, - }, - )?; - } - J2kWaveletTransform::Irreversible97 => { - for (instance_idx, bands) in self.resources.band_sets.iter().enumerate() { - let PreparedDirectGrayscaleStep::Idwt(step) = - &self.plans[instance_idx].steps[step_idx] - else { - return Err(direct_preflight_invariant( - "IDWT step mismatch in stacked component batch", - )); - }; - let ll = - lookup_direct_band_slice_entry(bands, step.step.ll_band_id, step.step.ll)?; - let hl = - lookup_direct_band_slice_entry(bands, step.step.hl_band_id, step.step.hl)?; - let lh = - lookup_direct_band_slice_entry(bands, step.step.lh_band_id, step.step.lh)?; - let hh = - lookup_direct_band_slice_entry(bands, step.step.hh_band_id, step.step.hh)?; - let params = prepared_idwt_params( - step, - idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), - ); - self.status_checks.push( - dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets( - self.command_buffers.idwt.interleave, - SingleIdwtDispatch { - runtime: self.runtime, - sub_bands: IdwtSubBandBuffers { - ll: &ll.buffer, - ll_offset: ll.offset_bytes, - hl: &hl.buffer, - hl_offset: hl.offset_bytes, - lh: &lh.buffer, - lh_offset: lh.offset_bytes, - hh: &hh.buffer, - hh_offset: hh.offset_bytes, - }, - params, - decoded: &output.buffer, - decoded_offset: instance_idx - * per_instance_len - * size_of::(), - }, - )?, - ); - } - } - } - if let Some(started) = encode_started { - self.stage_timings.metal_idwt_encode += elapsed_us(started); - } - - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { - let PreparedDirectGrayscaleStep::Idwt(step) = &self.plans[instance_idx].steps[step_idx] - else { - return Err(direct_preflight_invariant( - "IDWT output step mismatch in stacked component batch", - )); - }; - bands.push(DirectBandSlice { - band_id: step.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, - window: step.output_window, - }); - } - self.scratch_buffers.push(output); - Ok(()) - } - - fn submit_store(&mut self, store: &j2k_native::J2kDirectStoreStep) -> Result<(), Error> { - let (input, input_instance_stride) = lookup_repeated_direct_band_layout_entry( - &self.resources.band_sets, - store.input_band_id, - store.input_rect, - )?; - let per_instance_len = store.output_width as usize * store.output_height as usize; - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - let encode_started = self.profile_stages.then(Instant::now); - dispatch_store_component_repeated_in_command_buffer( - self.runtime, - self.command_buffers.store, - &input.buffer, - input.offset_bytes, - &output.buffer, - J2kRepeatedStoreParams { - input_width: store.input_rect.width(), - input_height: store.input_rect.height(), - input_instance_stride, - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_height: store.output_height, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - batch_count: u32::try_from(self.count).map_err(|_| Error::MetalKernel { - message: "J2K MetalDirect color store batch count exceeds u32".to_string(), - })?, - }, - )?; - if let Some(started) = encode_started { - self.stage_timings.metal_store_encode += elapsed_us(started); - } - self.resources.final_plane = Some(output.buffer.clone()); - self.scratch_buffers.push(output); - Ok(()) - } -} diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/classic_tier1.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/classic_tier1.rs new file mode 100644 index 00000000..02783306 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/classic_tier1.rs @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::time::Instant; + +use metal::Buffer; + +use crate::batch_allocation::{BatchMetadataBudget, BatchMetadataRequest}; +use crate::compute::decode_dispatch::{ + encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer, + encode_distinct_classic_sub_band_groups_to_buffer_in_encoder, + encode_distinct_classic_sub_bands_to_buffer_in_command_buffer, + encode_distinct_classic_sub_bands_to_buffer_in_encoder, +}; +use crate::compute::direct_grayscale_execute::upload_cpu_decoded_coefficients; +use crate::compute::direct_roi::BandRequiredRegion; +use crate::compute::{ + decode_classic_inputs_on_cpu_with_plan_cache, direct_preflight_invariant, elapsed_us, + take_f32_scratch_buffer, ClassicCpuDecodeInput, CpuTier1DecodeSubstageCounters, + DirectTier1Mode, Error, PreparedClassicSubBand, PreparedClassicSubBandGroup, + PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, +}; + +use super::super::resources::{retain_metal_tier1_output, DirectBandSlice}; +use super::super::validation::{ + checked_f32_batch_span, checked_f32_dimension_span, checked_f32_element_offset, + checked_f32_instance_offset, +}; +use super::{planned_cpu_input_count, try_collect_submission_items, SubmissionContext}; + +impl SubmissionContext<'_, '_, '_> { + pub(super) fn submit_classic_group( + &mut self, + first: &PreparedDirectGrayscalePlan, + step_idx: usize, + group: &PreparedClassicSubBandGroup, + ) -> Result<(), Error> { + let input_count = planned_cpu_input_count( + self.tier1_mode, + self.flattened_cpu_tier1_cache.is_some(), + self.broadcast_tier1_inputs, + self.plans.len(), + ); + let mut metadata_budget = + BatchMetadataBudget::new("J2K Metal stacked classic group submission metadata"); + metadata_budget.preflight(&[ + BatchMetadataRequest::of::<&PreparedClassicSubBandGroup>(self.plans.len()), + BatchMetadataRequest::of::>(input_count), + ])?; + let groups = try_collect_submission_items( + &mut metadata_budget, + self.plans.iter().map(|plan| { + plan.classic_group_starting_at(step_idx).ok_or_else(|| { + direct_preflight_invariant( + "classic group step mismatch in stacked component batch", + ) + }) + }), + "J2K Metal stacked classic group references", + )?; + let span = checked_f32_batch_span( + group.total_coefficients, + self.count, + "J2K MetalDirect stacked classic group", + )?; + let buffer = match self.tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let (buffers, status_check) = if let Some(encoder) = self.compute_encoder { + encode_distinct_classic_sub_band_groups_to_buffer_in_encoder( + self.runtime, + encoder, + &groups, + &output.buffer, + self.scratch_buffers, + )? + } else { + encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer( + self.runtime, + self.command_buffers.default, + &groups, + &output.buffer, + self.scratch_buffers, + )? + }; + if let Some(encoder) = self.compute_encoder { + encoder.memory_barrier_with_resources(&[&output.buffer]); + } + retain_metal_tier1_output( + output, + buffers, + status_check, + self.retained_buffers, + self.status_checks, + self.scratch_buffers, + ) + } + DirectTier1Mode::CpuUpload => self.prepare_classic_group_cpu_buffer( + first, + step_idx, + group, + &groups, + &mut metadata_budget, + )?, + }; + + for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { + let source_idx = if self.broadcast_tier1_inputs { + 0 + } else { + instance_idx + }; + let source_group = groups[source_idx]; + for member in &source_group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes: checked_f32_element_offset( + &span, + source_idx, + member.offset_elements, + "J2K MetalDirect stacked classic group member", + )?, + window: member.window, + }); + } + } + Ok(()) + } + + fn prepare_classic_group_cpu_buffer( + &mut self, + first: &PreparedDirectGrayscalePlan, + step_idx: usize, + group: &PreparedClassicSubBandGroup, + groups: &[&PreparedClassicSubBandGroup], + metadata_budget: &mut BatchMetadataBudget, + ) -> Result { + let input_groups = if self.broadcast_tier1_inputs { + &groups[..1] + } else { + groups + }; + if let Some(cache) = self.flattened_cpu_tier1_cache { + return cache.buffer_for( + self.component_idx, + step_idx, + group.total_coefficients, + input_groups.len(), + ); + } + + let inputs = try_collect_submission_items( + metadata_budget, + input_groups.iter().map(|group| { + Ok(ClassicCpuDecodeInput { + coded_data: &group.coded_data, + segments: &group.segments, + jobs: &group.jobs, + output_len: group.total_coefficients, + }) + }), + "J2K Metal stacked classic group CPU inputs", + )?; + let decode_started = self.profile_stages.then(Instant::now); + let cpu_tier1_counters = self + .profile_stages + .then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_classic_inputs_on_cpu_with_plan_cache( + first, + step_idx, + &inputs, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + self.stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(self.stage_timings); + } + let upload_started = self.profile_stages.then(Instant::now); + let buffer = + upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; + if let Some(started) = upload_started { + self.stage_timings.coefficient_upload += elapsed_us(started); + } + Ok(buffer) + } + + pub(super) fn submit_classic_sub_band( + &mut self, + first: &PreparedDirectGrayscalePlan, + step_idx: usize, + sub_band: &PreparedClassicSubBand, + ) -> Result<(), Error> { + let input_count = planned_cpu_input_count( + self.tier1_mode, + self.flattened_cpu_tier1_cache.is_some(), + self.broadcast_tier1_inputs, + self.plans.len(), + ); + let mut metadata_budget = + BatchMetadataBudget::new("J2K Metal stacked classic sub-band submission metadata"); + metadata_budget.preflight(&[ + BatchMetadataRequest::of::<&PreparedClassicSubBand>(self.plans.len()), + BatchMetadataRequest::of::>(input_count), + ])?; + let sub_bands = try_collect_submission_items( + &mut metadata_budget, + self.plans + .iter() + .map(|plan| match plan.steps.get(step_idx) { + Some(PreparedDirectGrayscaleStep::ClassicSubBand(other)) => Ok(other), + _ => Err(direct_preflight_invariant( + "classic sub-band step mismatch in stacked component batch", + )), + }), + "J2K Metal stacked classic sub-band references", + )?; + let span = checked_f32_dimension_span( + sub_band.width, + sub_band.height, + self.count, + "J2K MetalDirect stacked classic sub-band", + )?; + let buffer = match self.tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let (buffers, status_check) = if let Some(encoder) = self.compute_encoder { + encode_distinct_classic_sub_bands_to_buffer_in_encoder( + self.runtime, + encoder, + &sub_bands, + &output.buffer, + self.scratch_buffers, + )? + } else { + encode_distinct_classic_sub_bands_to_buffer_in_command_buffer( + self.runtime, + self.command_buffers.default, + &sub_bands, + &output.buffer, + self.scratch_buffers, + )? + }; + if let Some(encoder) = self.compute_encoder { + encoder.memory_barrier_with_resources(&[&output.buffer]); + } + retain_metal_tier1_output( + output, + buffers, + status_check, + self.retained_buffers, + self.status_checks, + self.scratch_buffers, + ) + } + DirectTier1Mode::CpuUpload => self.prepare_classic_sub_band_cpu_buffer( + first, + step_idx, + span.per_instance_elements, + &sub_bands, + &mut metadata_budget, + )?, + }; + + for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { + let source_idx = if self.broadcast_tier1_inputs { + 0 + } else { + instance_idx + }; + let source_sub_band = sub_bands[source_idx]; + bands.push(DirectBandSlice { + band_id: source_sub_band.band_id, + buffer: buffer.clone(), + offset_bytes: checked_f32_instance_offset( + &span, + source_idx, + "J2K MetalDirect stacked classic sub-band", + )?, + window: BandRequiredRegion::full(source_sub_band.width, source_sub_band.height), + }); + } + Ok(()) + } + + fn prepare_classic_sub_band_cpu_buffer( + &mut self, + first: &PreparedDirectGrayscalePlan, + step_idx: usize, + per_instance_len: usize, + sub_bands: &[&PreparedClassicSubBand], + metadata_budget: &mut BatchMetadataBudget, + ) -> Result { + let input_sub_bands = if self.broadcast_tier1_inputs { + &sub_bands[..1] + } else { + sub_bands + }; + if let Some(cache) = self.flattened_cpu_tier1_cache { + return cache.buffer_for( + self.component_idx, + step_idx, + per_instance_len, + input_sub_bands.len(), + ); + } + + let inputs = try_collect_submission_items( + metadata_budget, + input_sub_bands.iter().map(|sub_band| { + Ok(ClassicCpuDecodeInput { + coded_data: &sub_band.coded_data, + segments: &sub_band.segments, + jobs: &sub_band.jobs, + output_len: per_instance_len, + }) + }), + "J2K Metal stacked classic sub-band CPU inputs", + )?; + let decode_started = self.profile_stages.then(Instant::now); + let cpu_tier1_counters = self + .profile_stages + .then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_classic_inputs_on_cpu_with_plan_cache( + first, + step_idx, + &inputs, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + self.stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(self.stage_timings); + } + let upload_started = self.profile_stages.then(Instant::now); + let buffer = + upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; + if let Some(started) = upload_started { + self.stage_timings.coefficient_upload += elapsed_us(started); + } + Ok(buffer) + } +} diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/final_store.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/final_store.rs new file mode 100644 index 00000000..96eef31c --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/final_store.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::time::Instant; + +use crate::compute::abi::J2kRepeatedStoreParams; +use crate::compute::decode_dispatch::{ + dispatch_store_component_repeated_in_command_buffer, + dispatch_store_component_repeated_in_encoder, +}; +use crate::compute::{elapsed_us, take_f32_scratch_buffer, Error}; + +use super::super::resources::{lookup_repeated_direct_band_layout_entry, StackedFinalPlane}; +use super::super::validation::checked_f32_dimension_span; +use super::SubmissionContext; + +impl SubmissionContext<'_, '_, '_> { + pub(super) fn submit_store( + &mut self, + store: &j2k_native::J2kDirectStoreStep, + ) -> Result<(), Error> { + let (input, input_instance_stride) = lookup_repeated_direct_band_layout_entry( + &self.resources.band_sets, + store.input_band_id, + store.input_rect, + )?; + let dimensions = (store.output_width, store.output_height); + let span = checked_f32_dimension_span( + store.output_width, + store.output_height, + self.count, + "J2K MetalDirect stacked store", + )?; + let required_bytes = u64::try_from(span.total_bytes).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect stacked store byte length exceeds u64".to_string(), + })?; + let output = if let Some(output) = self.resources.final_plane.as_ref() { + if output.dimensions != dimensions || output.len != span.total_elements { + return Err(Error::MetalStateInvariant { + state: "J2K MetalDirect stacked component tile store", + reason: "later tile store changed the final component plane shape", + }); + } + if output.buffer.length() < required_bytes { + return Err(Error::MetalStateInvariant { + state: "J2K MetalDirect stacked component tile store", + reason: "retained final component plane is smaller than the validated store", + }); + } + output.buffer.clone() + } else { + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let buffer = output.buffer.clone(); + self.resources.final_plane = Some(StackedFinalPlane { + buffer: buffer.clone(), + dimensions, + len: span.total_elements, + }); + self.scratch_buffers.push(output); + buffer + }; + let encode_started = self.profile_stages.then(Instant::now); + let params = J2kRepeatedStoreParams { + input_width: store.input_rect.width(), + input_height: store.input_rect.height(), + input_instance_stride, + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_height: store.output_height, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + batch_count: u32::try_from(self.count).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect color store batch count exceeds u32".to_string(), + })?, + }; + if let Some(encoder) = self.compute_encoder { + dispatch_store_component_repeated_in_encoder( + self.runtime, + encoder, + &input.buffer, + input.offset_bytes, + &output, + params, + ); + encoder.memory_barrier_with_resources(&[&output]); + } else { + dispatch_store_component_repeated_in_command_buffer( + self.runtime, + self.command_buffers.store, + &input.buffer, + input.offset_bytes, + &output, + params, + )?; + } + if let Some(started) = encode_started { + self.stage_timings.metal_store_encode += elapsed_us(started); + } + for bands in &mut self.resources.band_sets { + bands.clear(); + } + Ok(()) + } +} diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/ht_tier1.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/ht_tier1.rs new file mode 100644 index 00000000..b4c5fc96 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/ht_tier1.rs @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::time::Instant; + +use metal::Buffer; + +use crate::batch_allocation::{BatchMetadataBudget, BatchMetadataRequest}; +use crate::compute::decode_dispatch::{ + encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer, + encode_distinct_ht_sub_band_groups_to_buffer_in_encoder, + encode_distinct_ht_sub_bands_to_buffer_in_command_buffer, + encode_distinct_ht_sub_bands_to_buffer_in_encoder, +}; +use crate::compute::direct_grayscale_execute::upload_cpu_decoded_coefficients; +use crate::compute::direct_roi::BandRequiredRegion; +use crate::compute::{ + decode_ht_inputs_on_cpu_with_plan_cache, direct_preflight_invariant, elapsed_us, + take_f32_scratch_buffer, CpuTier1DecodeSubstageCounters, DirectTier1Mode, Error, + HtCpuDecodeInput, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, PreparedHtSubBand, + PreparedHtSubBandGroup, +}; + +use super::super::resources::{retain_metal_tier1_output, DirectBandSlice}; +use super::super::validation::{ + checked_f32_batch_span, checked_f32_dimension_span, checked_f32_element_offset, + checked_f32_instance_offset, +}; +use super::{planned_cpu_input_count, try_collect_submission_items, SubmissionContext}; + +impl SubmissionContext<'_, '_, '_> { + pub(super) fn submit_ht_group( + &mut self, + first: &PreparedDirectGrayscalePlan, + step_idx: usize, + group: &PreparedHtSubBandGroup, + ) -> Result<(), Error> { + let input_count = planned_cpu_input_count( + self.tier1_mode, + self.flattened_cpu_tier1_cache.is_some(), + self.broadcast_tier1_inputs, + self.plans.len(), + ); + let mut metadata_budget = + BatchMetadataBudget::new("J2K Metal stacked HT group submission metadata"); + metadata_budget.preflight(&[ + BatchMetadataRequest::of::<&PreparedHtSubBandGroup>(self.plans.len()), + BatchMetadataRequest::of::>(input_count), + ])?; + let groups = try_collect_submission_items( + &mut metadata_budget, + self.plans.iter().map(|plan| { + plan.ht_group_starting_at(step_idx).ok_or_else(|| { + direct_preflight_invariant("HT group step mismatch in stacked component batch") + }) + }), + "J2K Metal stacked HT group references", + )?; + let span = checked_f32_batch_span( + group.total_coefficients, + self.count, + "J2K MetalDirect stacked HT group", + )?; + let buffer = match self.tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let (buffers, status_check) = if let Some(encoder) = self.compute_encoder { + encode_distinct_ht_sub_band_groups_to_buffer_in_encoder( + self.runtime, + encoder, + &groups, + &output.buffer, + )? + } else { + encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer( + self.runtime, + self.command_buffers.default, + &groups, + &output.buffer, + )? + }; + if let Some(encoder) = self.compute_encoder { + encoder.memory_barrier_with_resources(&[&output.buffer]); + } + retain_metal_tier1_output( + output, + buffers, + status_check, + self.retained_buffers, + self.status_checks, + self.scratch_buffers, + ) + } + DirectTier1Mode::CpuUpload => self.prepare_ht_group_cpu_buffer( + first, + step_idx, + group, + &groups, + &mut metadata_budget, + )?, + }; + + for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { + let source_idx = if self.broadcast_tier1_inputs { + 0 + } else { + instance_idx + }; + let source_group = groups[source_idx]; + for member in &source_group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes: checked_f32_element_offset( + &span, + source_idx, + member.offset_elements, + "J2K MetalDirect stacked HT group member", + )?, + window: member.window, + }); + } + } + Ok(()) + } + + fn prepare_ht_group_cpu_buffer( + &mut self, + first: &PreparedDirectGrayscalePlan, + step_idx: usize, + group: &PreparedHtSubBandGroup, + groups: &[&PreparedHtSubBandGroup], + metadata_budget: &mut BatchMetadataBudget, + ) -> Result { + let input_groups = if self.broadcast_tier1_inputs { + &groups[..1] + } else { + groups + }; + if let Some(cache) = self.flattened_cpu_tier1_cache { + return cache.buffer_for( + self.component_idx, + step_idx, + group.total_coefficients, + input_groups.len(), + ); + } + + let inputs = try_collect_submission_items( + metadata_budget, + input_groups.iter().map(|group| { + Ok(HtCpuDecodeInput { + payload_source: &group.payload_source, + jobs: &group.jobs, + output_len: group.total_coefficients, + }) + }), + "J2K Metal stacked HT group CPU inputs", + )?; + let decode_started = self.profile_stages.then(Instant::now); + let cpu_tier1_counters = self + .profile_stages + .then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_ht_inputs_on_cpu_with_plan_cache( + first, + step_idx, + &inputs, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + self.stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(self.stage_timings); + } + let upload_started = self.profile_stages.then(Instant::now); + let buffer = + upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; + if let Some(started) = upload_started { + self.stage_timings.coefficient_upload += elapsed_us(started); + } + Ok(buffer) + } + + pub(super) fn submit_ht_sub_band( + &mut self, + first: &PreparedDirectGrayscalePlan, + step_idx: usize, + sub_band: &PreparedHtSubBand, + ) -> Result<(), Error> { + let input_count = planned_cpu_input_count( + self.tier1_mode, + self.flattened_cpu_tier1_cache.is_some(), + self.broadcast_tier1_inputs, + self.plans.len(), + ); + let mut metadata_budget = + BatchMetadataBudget::new("J2K Metal stacked HT sub-band submission metadata"); + metadata_budget.preflight(&[ + BatchMetadataRequest::of::<&PreparedHtSubBand>(self.plans.len()), + BatchMetadataRequest::of::>(input_count), + ])?; + let sub_bands = try_collect_submission_items( + &mut metadata_budget, + self.plans + .iter() + .map(|plan| match plan.steps.get(step_idx) { + Some(PreparedDirectGrayscaleStep::HtSubBand(other)) => Ok(other), + _ => Err(direct_preflight_invariant( + "HT sub-band step mismatch in stacked component batch", + )), + }), + "J2K Metal stacked HT sub-band references", + )?; + let span = checked_f32_dimension_span( + sub_band.width, + sub_band.height, + self.count, + "J2K MetalDirect stacked HT sub-band", + )?; + let buffer = match self.tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let (buffers, status_check) = if let Some(encoder) = self.compute_encoder { + encode_distinct_ht_sub_bands_to_buffer_in_encoder( + self.runtime, + encoder, + &sub_bands, + &output.buffer, + )? + } else { + encode_distinct_ht_sub_bands_to_buffer_in_command_buffer( + self.runtime, + self.command_buffers.default, + &sub_bands, + &output.buffer, + )? + }; + if let Some(encoder) = self.compute_encoder { + encoder.memory_barrier_with_resources(&[&output.buffer]); + } + retain_metal_tier1_output( + output, + buffers, + status_check, + self.retained_buffers, + self.status_checks, + self.scratch_buffers, + ) + } + DirectTier1Mode::CpuUpload => self.prepare_ht_sub_band_cpu_buffer( + first, + step_idx, + span.per_instance_elements, + &sub_bands, + &mut metadata_budget, + )?, + }; + + for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { + let source_idx = if self.broadcast_tier1_inputs { + 0 + } else { + instance_idx + }; + let source_sub_band = sub_bands[source_idx]; + bands.push(DirectBandSlice { + band_id: source_sub_band.band_id, + buffer: buffer.clone(), + offset_bytes: checked_f32_instance_offset( + &span, + source_idx, + "J2K MetalDirect stacked HT sub-band", + )?, + window: BandRequiredRegion::full(source_sub_band.width, source_sub_band.height), + }); + } + Ok(()) + } + + fn prepare_ht_sub_band_cpu_buffer( + &mut self, + first: &PreparedDirectGrayscalePlan, + step_idx: usize, + per_instance_len: usize, + sub_bands: &[&PreparedHtSubBand], + metadata_budget: &mut BatchMetadataBudget, + ) -> Result { + let input_sub_bands = if self.broadcast_tier1_inputs { + &sub_bands[..1] + } else { + sub_bands + }; + if let Some(cache) = self.flattened_cpu_tier1_cache { + return cache.buffer_for( + self.component_idx, + step_idx, + per_instance_len, + input_sub_bands.len(), + ); + } + + let inputs = try_collect_submission_items( + metadata_budget, + input_sub_bands.iter().map(|sub_band| { + Ok(HtCpuDecodeInput { + payload_source: &sub_band.payload_source, + jobs: &sub_band.jobs, + output_len: per_instance_len, + }) + }), + "J2K Metal stacked HT sub-band CPU inputs", + )?; + let decode_started = self.profile_stages.then(Instant::now); + let cpu_tier1_counters = self + .profile_stages + .then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_ht_inputs_on_cpu_with_plan_cache( + first, + step_idx, + &inputs, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + self.stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(self.stage_timings); + } + let upload_started = self.profile_stages.then(Instant::now); + let buffer = + upload_cpu_decoded_coefficients(self.runtime, &coefficients, self.retained_buffers)?; + if let Some(started) = upload_started { + self.stage_timings.coefficient_upload += elapsed_us(started); + } + Ok(buffer) + } +} diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/reconstruction.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/reconstruction.rs new file mode 100644 index 00000000..2ec5665a --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/command_submission/reconstruction.rs @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::time::Instant; + +use crate::compute::decode_dispatch::{ + dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets, + dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets, + dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets, + dispatch_reversible53_repeated_buffers_in_encoder_with_offsets, IdwtSubBandBuffers, + RepeatedIdwtDispatch, SingleIdwtDispatch, +}; +use crate::compute::direct_roi::{ + idwt_input_windows_from_slices, prepared_idwt_params, repeated_idwt_params, + PreparedIdwtInputStrides, +}; +use crate::compute::{ + direct_preflight_invariant, elapsed_us, take_f32_scratch_buffer, Error, J2kWaveletTransform, + PreparedDirectGrayscaleStep, PreparedDirectIdwt, +}; + +use super::super::resources::{ + lookup_direct_band_slice_entry, lookup_repeated_direct_band_layout_entry, DirectBandSlice, +}; +use super::super::validation::{ + checked_f32_dimension_span, checked_f32_instance_offset, CheckedF32BatchSpan, +}; +use super::SubmissionContext; + +impl SubmissionContext<'_, '_, '_> { + fn encode_repeated_reversible53_idwt( + &mut self, + idwt: &PreparedDirectIdwt, + output: &metal::Buffer, + ) -> Result<(), Error> { + let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( + &self.resources.band_sets, + idwt.step.ll_band_id, + idwt.step.ll, + )?; + let (hl, high_low_stride) = lookup_repeated_direct_band_layout_entry( + &self.resources.band_sets, + idwt.step.hl_band_id, + idwt.step.hl, + )?; + let (lh, low_high_stride) = lookup_repeated_direct_band_layout_entry( + &self.resources.band_sets, + idwt.step.lh_band_id, + idwt.step.lh, + )?; + let (hh, high_high_stride) = lookup_repeated_direct_band_layout_entry( + &self.resources.band_sets, + idwt.step.hh_band_id, + idwt.step.hh, + )?; + let params = repeated_idwt_params( + idwt, + idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), + PreparedIdwtInputStrides { + ll: low_low_stride, + hl: high_low_stride, + lh: low_high_stride, + hh: high_high_stride, + }, + self.count, + "color", + )?; + let dispatch = RepeatedIdwtDispatch { + runtime: self.runtime, + sub_bands: IdwtSubBandBuffers { + ll: &ll.buffer, + ll_offset: ll.offset_bytes, + hl: &hl.buffer, + hl_offset: hl.offset_bytes, + lh: &lh.buffer, + lh_offset: lh.offset_bytes, + hh: &hh.buffer, + hh_offset: hh.offset_bytes, + }, + params, + decoded: output, + }; + if let Some(encoder) = self.compute_encoder { + dispatch_reversible53_repeated_buffers_in_encoder_with_offsets(encoder, dispatch); + } else { + dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( + self.command_buffers.idwt, + dispatch, + )?; + } + Ok(()) + } + + fn encode_distinct_irreversible97_idwt( + &mut self, + step_idx: usize, + output: &metal::Buffer, + span: &CheckedF32BatchSpan, + ) -> Result<(), Error> { + for (instance_idx, bands) in self.resources.band_sets.iter().enumerate() { + let PreparedDirectGrayscaleStep::Idwt(step) = &self.plans[instance_idx].steps[step_idx] + else { + return Err(direct_preflight_invariant( + "IDWT step mismatch in stacked component batch", + )); + }; + let ll = lookup_direct_band_slice_entry(bands, step.step.ll_band_id, step.step.ll)?; + let hl = lookup_direct_band_slice_entry(bands, step.step.hl_band_id, step.step.hl)?; + let lh = lookup_direct_band_slice_entry(bands, step.step.lh_band_id, step.step.lh)?; + let hh = lookup_direct_band_slice_entry(bands, step.step.hh_band_id, step.step.hh)?; + let params = + prepared_idwt_params(step, idwt_input_windows_from_slices(&ll, &hl, &lh, &hh)); + let dispatch = SingleIdwtDispatch { + runtime: self.runtime, + sub_bands: IdwtSubBandBuffers { + ll: &ll.buffer, + ll_offset: ll.offset_bytes, + hl: &hl.buffer, + hl_offset: hl.offset_bytes, + lh: &lh.buffer, + lh_offset: lh.offset_bytes, + hh: &hh.buffer, + hh_offset: hh.offset_bytes, + }, + params, + decoded: output, + decoded_offset: checked_f32_instance_offset( + span, + instance_idx, + "J2K MetalDirect stacked irreversible IDWT", + )?, + }; + let status = if let Some(encoder) = self.compute_encoder { + dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets( + encoder, dispatch, + )? + } else { + dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets( + self.command_buffers.idwt.interleave, + dispatch, + )? + }; + self.status_checks.push(status); + } + Ok(()) + } + + pub(super) fn submit_idwt( + &mut self, + step_idx: usize, + idwt: &PreparedDirectIdwt, + ) -> Result<(), Error> { + let span = checked_f32_dimension_span( + idwt.output_window.width(), + idwt.output_window.height(), + self.count, + "J2K MetalDirect stacked IDWT", + )?; + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let encode_started = self.profile_stages.then(Instant::now); + match idwt.step.transform { + J2kWaveletTransform::Reversible53 => { + self.encode_repeated_reversible53_idwt(idwt, &output.buffer)?; + } + J2kWaveletTransform::Irreversible97 => { + self.encode_distinct_irreversible97_idwt(step_idx, &output.buffer, &span)?; + } + } + if let Some(encoder) = self.compute_encoder { + encoder.memory_barrier_with_resources(&[&output.buffer]); + } + if let Some(started) = encode_started { + self.stage_timings.metal_idwt_encode += elapsed_us(started); + } + + for (instance_idx, bands) in self.resources.band_sets.iter_mut().enumerate() { + let PreparedDirectGrayscaleStep::Idwt(step) = &self.plans[instance_idx].steps[step_idx] + else { + return Err(direct_preflight_invariant( + "IDWT output step mismatch in stacked component batch", + )); + }; + bands.push(DirectBandSlice { + band_id: step.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: checked_f32_instance_offset( + &span, + instance_idx, + "J2K MetalDirect stacked IDWT output", + )?, + window: step.output_window, + }); + } + self.scratch_buffers.push(output); + Ok(()) + } +} diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution.rs index 840fee2f..ee5e6acd 100644 --- a/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution.rs +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution.rs @@ -1,37 +1,15 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::mem::size_of; - -use j2k_native::J2kDirectStoreStep; - use super::{ Buffer, CommandBufferRef, DirectScratchBuffer, DirectStatusCheck, Error, MetalRuntime, PixelFormat, RepeatedDirectGrayscalePlanRequest, Surface, }; -use crate::compute::direct_stacked_batch::{ - lookup_direct_band_slice, lookup_direct_band_slice_entry, - lookup_repeated_direct_band_layout_entry, DirectBandSlice, -}; -use crate::compute::{ - dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets, - dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets, - dispatch_reversible53_single_decomposition_buffers_in_command_buffer_with_offsets, - dispatch_store_component_buffer_in_command_buffer_with_offsets, - dispatch_store_component_repeated_in_command_buffer, - encode_gray_plane_to_surface_in_command_buffer_with_offset, - encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer, - encode_repeated_classic_sub_band_to_buffer_in_command_buffer, - encode_repeated_gray_plane_to_surfaces_in_command_buffer, - encode_repeated_gray_store_to_surfaces_in_command_buffer, - encode_repeated_ht_sub_band_group_to_buffer_in_command_buffer, - encode_repeated_ht_sub_band_to_buffer_in_command_buffer, idwt_input_windows_from_slices, - j2k_scalar_pack_params, prepared_idwt_output_len, prepared_idwt_params, repeated_idwt_params, - take_f32_scratch_buffer, BandRequiredRegion, DirectIdwtCommandBuffers, IdwtSubBandBuffers, - J2kRepeatedGrayStoreParams, J2kRepeatedStoreParams, J2kStoreParams, J2kWaveletTransform, - PreparedClassicSubBand, PreparedClassicSubBandGroup, PreparedDirectGrayscaleStep, - PreparedDirectIdwt, PreparedHtSubBand, PreparedHtSubBandGroup, PreparedIdwtInputStrides, - RepeatedIdwtDispatch, SingleIdwtDispatch, -}; +use crate::compute::direct_stacked_batch::DirectBandSlice; +use crate::compute::PreparedDirectGrayscaleStep; + +mod final_store; +mod reconstruction; +mod tier1; struct RepeatedGrayscaleExecution<'a> { runtime: &'a MetalRuntime, @@ -49,379 +27,6 @@ struct RepeatedGrayscaleExecution<'a> { } impl RepeatedGrayscaleExecution<'_> { - fn encode_classic_group(&mut self, group: &PreparedClassicSubBandGroup) -> Result<(), Error> { - let per_instance_len = group.total_coefficients; - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - let (buffers, status_check) = - encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer( - self.runtime, - self.command_buffer, - group, - self.count, - &output.buffer, - self.scratch_buffers, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in self.band_sets.iter_mut().enumerate() { - for member in &group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes - + member.offset_elements * size_of::(), - window: member.window, - }); - } - } - self.scratch_buffers.push(output); - Ok(()) - } - - fn encode_ht_group(&mut self, group: &PreparedHtSubBandGroup) -> Result<(), Error> { - let per_instance_len = group.total_coefficients; - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - let (buffers, status_check) = - encode_repeated_ht_sub_band_group_to_buffer_in_command_buffer( - self.runtime, - self.command_buffer, - group, - self.count, - &output.buffer, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in self.band_sets.iter_mut().enumerate() { - for member in &group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes - + member.offset_elements * size_of::(), - window: member.window, - }); - } - } - self.scratch_buffers.push(output); - Ok(()) - } - - fn retain_sub_band( - &mut self, - output: DirectScratchBuffer, - band_id: j2k_native::J2kDirectBandId, - width: u32, - height: u32, - ) { - let per_instance_len = width as usize * height as usize; - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in self.band_sets.iter_mut().enumerate() { - bands.push(DirectBandSlice { - band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, - window: BandRequiredRegion::full(width, height), - }); - } - self.scratch_buffers.push(output); - } - - fn encode_classic_sub_band(&mut self, sub_band: &PreparedClassicSubBand) -> Result<(), Error> { - let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - let (buffers, status_check) = encode_repeated_classic_sub_band_to_buffer_in_command_buffer( - self.runtime, - self.command_buffer, - sub_band, - self.count, - &output.buffer, - self.scratch_buffers, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - self.retain_sub_band(output, sub_band.band_id, sub_band.width, sub_band.height); - Ok(()) - } - - fn encode_ht_sub_band(&mut self, sub_band: &PreparedHtSubBand) -> Result<(), Error> { - let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - let (buffers, status_check) = encode_repeated_ht_sub_band_to_buffer_in_command_buffer( - self.runtime, - self.command_buffer, - sub_band, - self.count, - &output.buffer, - )?; - self.retained_buffers.extend(buffers); - self.status_checks.push(status_check); - self.retain_sub_band(output, sub_band.band_id, sub_band.width, sub_band.height); - Ok(()) - } - - fn encode_stacked_idwt(&mut self, idwt: &PreparedDirectIdwt) -> Result<(), Error> { - let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( - &self.band_sets, - idwt.step.ll_band_id, - idwt.step.ll, - )?; - let (hl, high_low_stride) = lookup_repeated_direct_band_layout_entry( - &self.band_sets, - idwt.step.hl_band_id, - idwt.step.hl, - )?; - let (lh, low_high_stride) = lookup_repeated_direct_band_layout_entry( - &self.band_sets, - idwt.step.lh_band_id, - idwt.step.lh, - )?; - let (hh, high_high_stride) = lookup_repeated_direct_band_layout_entry( - &self.band_sets, - idwt.step.hh_band_id, - idwt.step.hh, - )?; - let params = repeated_idwt_params( - idwt, - idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), - PreparedIdwtInputStrides { - ll: low_low_stride, - hl: high_low_stride, - lh: low_high_stride, - hh: high_high_stride, - }, - self.count, - "repeated", - )?; - let per_instance_len = prepared_idwt_output_len(idwt); - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( - DirectIdwtCommandBuffers::single(self.command_buffer), - RepeatedIdwtDispatch { - runtime: self.runtime, - sub_bands: IdwtSubBandBuffers { - ll: &ll.buffer, - ll_offset: ll.offset_bytes, - hl: &hl.buffer, - hl_offset: hl.offset_bytes, - lh: &lh.buffer, - lh_offset: lh.offset_bytes, - hh: &hh.buffer, - hh_offset: hh.offset_bytes, - }, - params, - decoded: &output.buffer, - }, - )?; - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in self.band_sets.iter_mut().enumerate() { - bands.push(DirectBandSlice { - band_id: idwt.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, - window: idwt.output_window, - }); - } - self.scratch_buffers.push(output); - Ok(()) - } - - fn encode_per_instance_idwt(&mut self, idwt: &PreparedDirectIdwt) -> Result<(), Error> { - for bands in &mut self.band_sets { - let ll = lookup_direct_band_slice_entry(bands, idwt.step.ll_band_id, idwt.step.ll)?; - let hl = lookup_direct_band_slice_entry(bands, idwt.step.hl_band_id, idwt.step.hl)?; - let lh = lookup_direct_band_slice_entry(bands, idwt.step.lh_band_id, idwt.step.lh)?; - let hh = lookup_direct_band_slice_entry(bands, idwt.step.hh_band_id, idwt.step.hh)?; - let params = - prepared_idwt_params(idwt, idwt_input_windows_from_slices(&ll, &hl, &lh, &hh)); - let output = take_f32_scratch_buffer(self.runtime, prepared_idwt_output_len(idwt))?; - let dispatch = SingleIdwtDispatch { - runtime: self.runtime, - sub_bands: IdwtSubBandBuffers { - ll: &ll.buffer, - ll_offset: ll.offset_bytes, - hl: &hl.buffer, - hl_offset: hl.offset_bytes, - lh: &lh.buffer, - lh_offset: lh.offset_bytes, - hh: &hh.buffer, - hh_offset: hh.offset_bytes, - }, - params, - decoded: &output.buffer, - decoded_offset: 0, - }; - match idwt.step.transform { - J2kWaveletTransform::Reversible53 => { - dispatch_reversible53_single_decomposition_buffers_in_command_buffer_with_offsets( - self.command_buffer, - dispatch, - )?; - } - J2kWaveletTransform::Irreversible97 => { - self.status_checks.push( - dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets( - self.command_buffer, - dispatch, - )?, - ); - } - } - bands.push(DirectBandSlice { - band_id: idwt.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: idwt.output_window, - }); - self.scratch_buffers.push(output); - } - Ok(()) - } - - fn encode_idwt(&mut self, idwt: &PreparedDirectIdwt) -> Result<(), Error> { - if idwt.step.transform == J2kWaveletTransform::Reversible53 && self.stacked_outputs { - self.encode_stacked_idwt(idwt) - } else { - self.stacked_outputs = false; - self.encode_per_instance_idwt(idwt) - } - } - - fn encode_stacked_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { - let (input, _) = - lookup_direct_band_slice(&self.band_sets[0], store.input_band_id, store.input_rect)?; - let batch_count = u32::try_from(self.count).map_err(|_| Error::MetalKernel { - message: "J2K MetalDirect repeated store batch count exceeds u32".to_string(), - })?; - if matches!(self.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - let scale = j2k_scalar_pack_params(u32::from(self.bit_depth)); - self.surfaces - .extend(encode_repeated_gray_store_to_surfaces_in_command_buffer( - self.runtime, - self.command_buffer, - &input, - J2kRepeatedGrayStoreParams { - input_width: store.input_rect.width(), - input_height: store.input_rect.height(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_height: store.output_height, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - batch_count, - max_value: scale.max_value, - u8_scale: scale.u8_scale, - u16_scale: scale.u16_scale, - }, - self.dimensions, - self.fmt, - self.count, - )?); - } else { - let per_instance_len = store.output_width as usize * store.output_height as usize; - let output = take_f32_scratch_buffer(self.runtime, per_instance_len * self.count)?; - dispatch_store_component_repeated_in_command_buffer( - self.runtime, - self.command_buffer, - &input, - 0, - &output.buffer, - J2kRepeatedStoreParams { - input_width: store.input_rect.width(), - input_height: store.input_rect.height(), - input_instance_stride: store - .input_rect - .width() - .checked_mul(store.input_rect.height()) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect repeated store input stride overflows u32" - .to_string(), - })?, - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_height: store.output_height, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - batch_count, - }, - )?; - self.retained_buffers.push(output.buffer.clone()); - self.surfaces - .extend(encode_repeated_gray_plane_to_surfaces_in_command_buffer( - self.runtime, - self.command_buffer, - &output.buffer, - self.dimensions, - self.bit_depth, - self.fmt, - self.count, - )?); - self.scratch_buffers.push(output); - } - Ok(()) - } - - fn encode_per_instance_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { - for bands in &self.band_sets { - let (input, input_offset) = - lookup_direct_band_slice(bands, store.input_band_id, store.input_rect)?; - let output = take_f32_scratch_buffer( - self.runtime, - store.output_width as usize * store.output_height as usize, - )?; - let params = J2kStoreParams { - input_width: store.input_rect.width(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - }; - dispatch_store_component_buffer_in_command_buffer_with_offsets( - self.runtime, - self.command_buffer, - &input, - input_offset, - &output.buffer, - 0, - params, - )?; - self.retained_buffers.push(output.buffer.clone()); - self.surfaces - .push(encode_gray_plane_to_surface_in_command_buffer_with_offset( - self.runtime, - self.command_buffer, - &output.buffer, - 0, - self.dimensions, - self.bit_depth, - self.fmt, - )?); - self.scratch_buffers.push(output); - } - Ok(()) - } - - fn encode_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { - if self.stacked_outputs { - self.encode_stacked_store(store) - } else { - self.encode_per_instance_store(store) - } - } - fn encode_step(&mut self, step: &PreparedDirectGrayscaleStep) -> Result<(), Error> { match step { PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/final_store.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/final_store.rs new file mode 100644 index 00000000..4e3a5567 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/final_store.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_native::J2kDirectStoreStep; + +use crate::compute::abi::{J2kRepeatedGrayStoreParams, J2kRepeatedStoreParams, J2kStoreParams}; +use crate::compute::decode_dispatch::{ + dispatch_store_component_buffer_in_command_buffer_with_offsets, + dispatch_store_component_repeated_in_command_buffer, + encode_repeated_gray_store_to_surfaces_in_command_buffer, +}; +use crate::compute::direct_stacked_batch::{ + lookup_direct_band_slice, validation::checked_f32_dimension_span, +}; +use crate::compute::{ + encode_gray_plane_to_surface_in_command_buffer_with_offset, + encode_repeated_gray_plane_to_surfaces_in_command_buffer, j2k_scalar_pack_params, + take_f32_scratch_buffer, Error, PixelFormat, +}; + +use super::RepeatedGrayscaleExecution; + +impl RepeatedGrayscaleExecution<'_> { + fn encode_stacked_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { + let first_bands = self.band_sets.first().ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated store has no band sets".to_string(), + })?; + let (input, _) = + lookup_direct_band_slice(first_bands, store.input_band_id, store.input_rect)?; + let span = checked_f32_dimension_span( + store.output_width, + store.output_height, + self.count, + "J2K MetalDirect repeated stacked store", + )?; + let batch_count = u32::try_from(self.count).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect repeated store batch count exceeds u32".to_string(), + })?; + if matches!(self.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { + let scale = j2k_scalar_pack_params(u32::from(self.bit_depth)); + self.surfaces + .extend(encode_repeated_gray_store_to_surfaces_in_command_buffer( + self.runtime, + self.command_buffer, + &input, + J2kRepeatedGrayStoreParams { + input_width: store.input_rect.width(), + input_height: store.input_rect.height(), + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_height: store.output_height, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + batch_count, + max_value: scale.max_value, + u8_scale: scale.u8_scale, + u16_scale: scale.u16_scale, + }, + self.dimensions, + self.fmt, + self.count, + )?); + } else { + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + dispatch_store_component_repeated_in_command_buffer( + self.runtime, + self.command_buffer, + &input, + 0, + &output.buffer, + J2kRepeatedStoreParams { + input_width: store.input_rect.width(), + input_height: store.input_rect.height(), + input_instance_stride: store + .input_rect + .width() + .checked_mul(store.input_rect.height()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated store input stride overflows u32" + .to_string(), + })?, + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_height: store.output_height, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + batch_count, + }, + )?; + self.retained_buffers.push(output.buffer.clone()); + self.surfaces + .extend(encode_repeated_gray_plane_to_surfaces_in_command_buffer( + self.runtime, + self.command_buffer, + &output.buffer, + self.dimensions, + self.bit_depth, + self.fmt, + self.count, + )?); + self.scratch_buffers.push(output); + } + Ok(()) + } + + fn encode_per_instance_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { + let span = checked_f32_dimension_span( + store.output_width, + store.output_height, + 1, + "J2K MetalDirect repeated per-instance store", + )?; + for bands in &self.band_sets { + let (input, input_offset) = + lookup_direct_band_slice(bands, store.input_band_id, store.input_rect)?; + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let params = J2kStoreParams { + input_width: store.input_rect.width(), + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + }; + dispatch_store_component_buffer_in_command_buffer_with_offsets( + self.runtime, + self.command_buffer, + &input, + input_offset, + &output.buffer, + 0, + params, + )?; + self.retained_buffers.push(output.buffer.clone()); + self.surfaces + .push(encode_gray_plane_to_surface_in_command_buffer_with_offset( + self.runtime, + self.command_buffer, + &output.buffer, + 0, + self.dimensions, + self.bit_depth, + self.fmt, + )?); + self.scratch_buffers.push(output); + } + Ok(()) + } + + pub(super) fn encode_store(&mut self, store: &J2kDirectStoreStep) -> Result<(), Error> { + if self.stacked_outputs { + self.encode_stacked_store(store) + } else { + self.encode_per_instance_store(store) + } + } +} diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/reconstruction.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/reconstruction.rs new file mode 100644 index 00000000..c595e38f --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/reconstruction.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::compute::decode_dispatch::{ + dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets, + dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets, + dispatch_reversible53_single_decomposition_buffers_in_command_buffer_with_offsets, + IdwtSubBandBuffers, RepeatedIdwtDispatch, SingleIdwtDispatch, +}; +use crate::compute::direct_roi::{ + idwt_input_windows_from_slices, prepared_idwt_params, repeated_idwt_params, + PreparedIdwtInputStrides, +}; +use crate::compute::direct_stacked_batch::validation::{ + checked_f32_dimension_span, checked_f32_instance_offset, +}; +use crate::compute::direct_stacked_batch::{ + lookup_direct_band_slice_entry, lookup_repeated_direct_band_layout_entry, DirectBandSlice, +}; +use crate::compute::{ + take_f32_scratch_buffer, DirectIdwtCommandBuffers, Error, J2kWaveletTransform, + PreparedDirectIdwt, +}; + +use super::RepeatedGrayscaleExecution; + +impl RepeatedGrayscaleExecution<'_> { + pub(super) fn encode_stacked_idwt(&mut self, idwt: &PreparedDirectIdwt) -> Result<(), Error> { + let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( + &self.band_sets, + idwt.step.ll_band_id, + idwt.step.ll, + )?; + let (hl, high_low_stride) = lookup_repeated_direct_band_layout_entry( + &self.band_sets, + idwt.step.hl_band_id, + idwt.step.hl, + )?; + let (lh, low_high_stride) = lookup_repeated_direct_band_layout_entry( + &self.band_sets, + idwt.step.lh_band_id, + idwt.step.lh, + )?; + let (hh, high_high_stride) = lookup_repeated_direct_band_layout_entry( + &self.band_sets, + idwt.step.hh_band_id, + idwt.step.hh, + )?; + let params = repeated_idwt_params( + idwt, + idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), + PreparedIdwtInputStrides { + ll: low_low_stride, + hl: high_low_stride, + lh: low_high_stride, + hh: high_high_stride, + }, + self.count, + "repeated", + )?; + let span = checked_f32_dimension_span( + idwt.output_window.width(), + idwt.output_window.height(), + self.count, + "J2K MetalDirect repeated stacked IDWT", + )?; + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( + DirectIdwtCommandBuffers::single(self.command_buffer), + RepeatedIdwtDispatch { + runtime: self.runtime, + sub_bands: IdwtSubBandBuffers { + ll: &ll.buffer, + ll_offset: ll.offset_bytes, + hl: &hl.buffer, + hl_offset: hl.offset_bytes, + lh: &lh.buffer, + lh_offset: lh.offset_bytes, + hh: &hh.buffer, + hh_offset: hh.offset_bytes, + }, + params, + decoded: &output.buffer, + }, + )?; + for (instance_idx, bands) in self.band_sets.iter_mut().enumerate() { + bands.push(DirectBandSlice { + band_id: idwt.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: checked_f32_instance_offset( + &span, + instance_idx, + "J2K MetalDirect repeated stacked IDWT output", + )?, + window: idwt.output_window, + }); + } + self.scratch_buffers.push(output); + Ok(()) + } + + pub(super) fn encode_per_instance_idwt( + &mut self, + idwt: &PreparedDirectIdwt, + ) -> Result<(), Error> { + let span = checked_f32_dimension_span( + idwt.output_window.width(), + idwt.output_window.height(), + 1, + "J2K MetalDirect repeated per-instance IDWT", + )?; + for bands in &mut self.band_sets { + let ll = lookup_direct_band_slice_entry(bands, idwt.step.ll_band_id, idwt.step.ll)?; + let hl = lookup_direct_band_slice_entry(bands, idwt.step.hl_band_id, idwt.step.hl)?; + let lh = lookup_direct_band_slice_entry(bands, idwt.step.lh_band_id, idwt.step.lh)?; + let hh = lookup_direct_band_slice_entry(bands, idwt.step.hh_band_id, idwt.step.hh)?; + let params = + prepared_idwt_params(idwt, idwt_input_windows_from_slices(&ll, &hl, &lh, &hh)); + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let dispatch = SingleIdwtDispatch { + runtime: self.runtime, + sub_bands: IdwtSubBandBuffers { + ll: &ll.buffer, + ll_offset: ll.offset_bytes, + hl: &hl.buffer, + hl_offset: hl.offset_bytes, + lh: &lh.buffer, + lh_offset: lh.offset_bytes, + hh: &hh.buffer, + hh_offset: hh.offset_bytes, + }, + params, + decoded: &output.buffer, + decoded_offset: 0, + }; + match idwt.step.transform { + J2kWaveletTransform::Reversible53 => { + dispatch_reversible53_single_decomposition_buffers_in_command_buffer_with_offsets( + self.command_buffer, + dispatch, + )?; + } + J2kWaveletTransform::Irreversible97 => { + self.status_checks.push( + dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets( + self.command_buffer, + dispatch, + )?, + ); + } + } + bands.push(DirectBandSlice { + band_id: idwt.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: idwt.output_window, + }); + self.scratch_buffers.push(output); + } + Ok(()) + } + + pub(super) fn encode_idwt(&mut self, idwt: &PreparedDirectIdwt) -> Result<(), Error> { + if idwt.step.transform == J2kWaveletTransform::Reversible53 && self.stacked_outputs { + self.encode_stacked_idwt(idwt) + } else { + self.stacked_outputs = false; + self.encode_per_instance_idwt(idwt) + } + } +} diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/tier1.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/tier1.rs new file mode 100644 index 00000000..6c13d087 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/tier1.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::compute::decode_dispatch::{ + encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer, + encode_repeated_classic_sub_band_to_buffer_in_command_buffer, + encode_repeated_ht_sub_band_group_to_buffer_in_command_buffer, + encode_repeated_ht_sub_band_to_buffer_in_command_buffer, +}; +use crate::compute::direct_roi::BandRequiredRegion; +use crate::compute::direct_stacked_batch::validation::{ + checked_f32_batch_span, checked_f32_dimension_span, checked_f32_element_offset, + checked_f32_instance_offset, CheckedF32BatchSpan, +}; +use crate::compute::{ + take_f32_scratch_buffer, Error, PreparedClassicSubBand, PreparedClassicSubBandGroup, + PreparedHtSubBand, PreparedHtSubBandGroup, +}; + +use super::{DirectBandSlice, DirectScratchBuffer, RepeatedGrayscaleExecution}; + +impl RepeatedGrayscaleExecution<'_> { + pub(super) fn encode_classic_group( + &mut self, + group: &PreparedClassicSubBandGroup, + ) -> Result<(), Error> { + let span = checked_f32_batch_span( + group.total_coefficients, + self.count, + "J2K MetalDirect repeated classic group", + )?; + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let (buffers, status_check) = + encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer( + self.runtime, + self.command_buffer, + group, + self.count, + &output.buffer, + self.scratch_buffers, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + for (instance_idx, bands) in self.band_sets.iter_mut().enumerate() { + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: output.buffer.clone(), + offset_bytes: checked_f32_element_offset( + &span, + instance_idx, + member.offset_elements, + "J2K MetalDirect repeated classic group member", + )?, + window: member.window, + }); + } + } + self.scratch_buffers.push(output); + Ok(()) + } + + pub(super) fn encode_ht_group(&mut self, group: &PreparedHtSubBandGroup) -> Result<(), Error> { + let span = checked_f32_batch_span( + group.total_coefficients, + self.count, + "J2K MetalDirect repeated HT group", + )?; + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let (buffers, status_check) = + encode_repeated_ht_sub_band_group_to_buffer_in_command_buffer( + self.runtime, + self.command_buffer, + group, + self.count, + &output.buffer, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + for (instance_idx, bands) in self.band_sets.iter_mut().enumerate() { + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: output.buffer.clone(), + offset_bytes: checked_f32_element_offset( + &span, + instance_idx, + member.offset_elements, + "J2K MetalDirect repeated HT group member", + )?, + window: member.window, + }); + } + } + self.scratch_buffers.push(output); + Ok(()) + } + + fn retain_sub_band( + &mut self, + output: DirectScratchBuffer, + band_id: j2k_native::J2kDirectBandId, + width: u32, + height: u32, + span: &CheckedF32BatchSpan, + ) -> Result<(), Error> { + for (instance_idx, bands) in self.band_sets.iter_mut().enumerate() { + bands.push(DirectBandSlice { + band_id, + buffer: output.buffer.clone(), + offset_bytes: checked_f32_instance_offset( + span, + instance_idx, + "J2K MetalDirect repeated sub-band", + )?, + window: BandRequiredRegion::full(width, height), + }); + } + self.scratch_buffers.push(output); + Ok(()) + } + + pub(super) fn encode_classic_sub_band( + &mut self, + sub_band: &PreparedClassicSubBand, + ) -> Result<(), Error> { + let span = checked_f32_dimension_span( + sub_band.width, + sub_band.height, + self.count, + "J2K MetalDirect repeated classic sub-band", + )?; + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let (buffers, status_check) = encode_repeated_classic_sub_band_to_buffer_in_command_buffer( + self.runtime, + self.command_buffer, + sub_band, + self.count, + &output.buffer, + self.scratch_buffers, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.retain_sub_band( + output, + sub_band.band_id, + sub_band.width, + sub_band.height, + &span, + ) + } + + pub(super) fn encode_ht_sub_band(&mut self, sub_band: &PreparedHtSubBand) -> Result<(), Error> { + let span = checked_f32_dimension_span( + sub_band.width, + sub_band.height, + self.count, + "J2K MetalDirect repeated HT sub-band", + )?; + let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; + let (buffers, status_check) = encode_repeated_ht_sub_band_to_buffer_in_command_buffer( + self.runtime, + self.command_buffer, + sub_band, + self.count, + &output.buffer, + )?; + self.retained_buffers.extend(buffers); + self.status_checks.push(status_check); + self.retain_sub_band( + output, + sub_band.band_id, + sub_band.width, + sub_band.height, + &span, + ) + } +} diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/resources.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/resources.rs index d34cbd7a..0b056a0b 100644 --- a/crates/j2k-metal/src/compute/direct_stacked_batch/resources.rs +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/resources.rs @@ -4,9 +4,8 @@ use std::mem::size_of; use metal::Buffer; -use super::super::{ - BandRequiredRegion, DirectScratchBuffer, DirectStatusCheck, Error, J2kDirectBandId, -}; +use super::super::direct_roi::BandRequiredRegion; +use super::super::{DirectScratchBuffer, DirectStatusCheck, Error, J2kDirectBandId}; #[derive(Clone)] pub(in super::super) struct DirectBandSlice { @@ -16,9 +15,15 @@ pub(in super::super) struct DirectBandSlice { pub(in super::super) window: BandRequiredRegion, } +pub(super) struct StackedFinalPlane { + pub(super) buffer: Buffer, + pub(super) dimensions: (u32, u32), + pub(super) len: usize, +} + pub(super) struct StackedComponentResources { pub(super) band_sets: Vec>, - pub(super) final_plane: Option, + pub(super) final_plane: Option, } pub(super) fn prepare_stacked_component_resources( @@ -127,7 +132,7 @@ pub(in super::super) fn lookup_repeated_direct_band_layout_entry( message: "J2K MetalDirect repeated band offsets are not monotonic".to_string(), })? } else { - entry.window.width() as usize * entry.window.height() as usize * size_of::() + checked_repeated_band_fallback_stride(entry.window.width(), entry.window.height())? }; if stride_bytes % size_of::() != 0 { return Err(Error::MetalKernel { @@ -141,6 +146,20 @@ pub(in super::super) fn lookup_repeated_direct_band_layout_entry( Ok((entry, stride_elements)) } +fn checked_repeated_band_fallback_stride(width: u32, height: u32) -> Result { + usize::try_from(width) + .ok() + .and_then(|width| { + usize::try_from(height) + .ok() + .and_then(|height| width.checked_mul(height)) + }) + .and_then(|elements| elements.checked_mul(size_of::())) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated band fallback stride overflow".to_string(), + }) +} + #[cfg(test)] mod tests { use j2k_core::BatchInfrastructureError; @@ -206,4 +225,17 @@ mod tests { if message == "missing J2K MetalDirect repeated band set" )); } + + #[test] + fn repeated_band_fallback_stride_checks_f32_byte_span() { + assert_eq!( + checked_repeated_band_fallback_stride(u32::MAX, 1) + .expect("u32::MAX f32 elements fit in usize on supported Metal hosts"), + u32::MAX as usize * size_of::() + ); + assert!(matches!( + checked_repeated_band_fallback_stride(u32::MAX, u32::MAX), + Err(Error::MetalKernel { message }) if message.contains("overflow") + )); + } } diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/result.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/result.rs deleted file mode 100644 index 5faab2fe..00000000 --- a/crates/j2k-metal/src/compute/direct_stacked_batch/result.rs +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::super::{record_hybrid_stacked_component_batch, DirectTier1Mode, Error}; -use super::resources::StackedComponentResources; -use super::validation::StackedComponentBatchPlan; -use super::StackedDirectComponentPlane; - -pub(super) fn assemble_stacked_component_result( - resources: StackedComponentResources, - plan: &StackedComponentBatchPlan<'_>, - tier1_mode: DirectTier1Mode, -) -> Result { - let buffer = resources.final_plane.ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect color component batch did not produce a final plane".to_string(), - })?; - record_hybrid_stacked_component_batch(tier1_mode); - Ok(StackedDirectComponentPlane { - buffer, - dimensions: plan.first.dimensions, - count: plan.count, - }) -} diff --git a/crates/j2k-metal/src/compute/direct_stacked_batch/validation.rs b/crates/j2k-metal/src/compute/direct_stacked_batch/validation.rs index bed840cf..2bbcc0f1 100644 --- a/crates/j2k-metal/src/compute/direct_stacked_batch/validation.rs +++ b/crates/j2k-metal/src/compute/direct_stacked_batch/validation.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use std::mem::size_of; + use super::super::{ classic_group_shapes_match, classic_sub_band_shapes_match, ht_group_shapes_match, ht_sub_band_shapes_match, idwt_shapes_match, store_shapes_match, DirectTier1Mode, Error, @@ -12,6 +14,96 @@ pub(super) struct StackedComponentBatchPlan<'p> { pub(super) broadcast_tier1_inputs: bool, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct CheckedF32BatchSpan { + pub(super) per_instance_elements: usize, + pub(super) instance_count: usize, + pub(super) total_elements: usize, + pub(super) stride_bytes: usize, + pub(super) total_bytes: usize, +} + +fn span_overflow(context: &'static str, operation: &'static str) -> Error { + Error::MetalKernel { + message: format!("{context} {operation} overflow"), + } +} + +pub(super) fn checked_f32_batch_span( + per_instance_elements: usize, + instance_count: usize, + context: &'static str, +) -> Result { + let total_elements = per_instance_elements + .checked_mul(instance_count) + .ok_or_else(|| span_overflow(context, "element count"))?; + let stride_bytes = per_instance_elements + .checked_mul(size_of::()) + .ok_or_else(|| span_overflow(context, "instance byte stride"))?; + let total_bytes = stride_bytes + .checked_mul(instance_count) + .ok_or_else(|| span_overflow(context, "batch byte count"))?; + Ok(CheckedF32BatchSpan { + per_instance_elements, + instance_count, + total_elements, + stride_bytes, + total_bytes, + }) +} + +pub(super) fn checked_f32_dimension_span( + width: u32, + height: u32, + instance_count: usize, + context: &'static str, +) -> Result { + let per_instance_elements = usize::try_from(width) + .ok() + .and_then(|width| { + usize::try_from(height) + .ok() + .and_then(|height| width.checked_mul(height)) + }) + .ok_or_else(|| span_overflow(context, "plane element count"))?; + checked_f32_batch_span(per_instance_elements, instance_count, context) +} + +pub(super) fn checked_f32_instance_offset( + span: &CheckedF32BatchSpan, + instance_index: usize, + context: &'static str, +) -> Result { + if instance_index >= span.instance_count { + return Err(Error::MetalKernel { + message: format!("{context} instance index is outside its batch"), + }); + } + instance_index + .checked_mul(span.stride_bytes) + .ok_or_else(|| span_overflow(context, "instance byte offset")) +} + +pub(super) fn checked_f32_element_offset( + span: &CheckedF32BatchSpan, + instance_index: usize, + element_offset: usize, + context: &'static str, +) -> Result { + if element_offset >= span.per_instance_elements { + return Err(Error::MetalKernel { + message: format!("{context} element offset is outside its instance"), + }); + } + let instance_offset = checked_f32_instance_offset(span, instance_index, context)?; + let member_offset = element_offset + .checked_mul(size_of::()) + .ok_or_else(|| span_overflow(context, "member byte offset"))?; + instance_offset + .checked_add(member_offset) + .ok_or_else(|| span_overflow(context, "aggregate byte offset")) +} + pub(super) fn plan_stacked_component_batch<'p>( plans: &[&'p PreparedDirectGrayscalePlan], tier1_mode: DirectTier1Mode, @@ -139,4 +231,100 @@ mod tests { if message == "J2K MetalDirect color batch has no component plans" )); } + + #[test] + fn distinct_stacked_f32_span_accepts_exact_boundary_and_rejects_one_over() { + let exact_elements = usize::MAX / size_of::(); + let span = checked_f32_batch_span( + exact_elements, + 1, + "J2K MetalDirect stacked distinct test span", + ) + .expect("largest exactly representable f32 byte span"); + assert_eq!(span.per_instance_elements, exact_elements); + assert_eq!(span.total_elements, exact_elements); + assert_eq!(span.stride_bytes, exact_elements * size_of::()); + assert_eq!(span.total_bytes, exact_elements * size_of::()); + + assert!(matches!( + checked_f32_batch_span( + exact_elements + 1, + 1, + "J2K MetalDirect stacked distinct test span", + ), + Err(Error::MetalKernel { message }) if message.contains("overflow") + )); + assert!(matches!( + checked_f32_batch_span( + 2, + usize::MAX, + "J2K MetalDirect stacked distinct test span", + ), + Err(Error::MetalKernel { message }) if message.contains("overflow") + )); + } + + #[test] + fn repeated_f32_offsets_are_checked_against_the_validated_instance_span() { + let exact_per_instance = usize::MAX / 8; + let exact = checked_f32_batch_span( + exact_per_instance, + 2, + "J2K MetalDirect repeated grayscale test span", + ) + .expect("largest two-instance f32 span below usize::MAX"); + assert_eq!(exact.total_bytes, exact_per_instance * 8); + assert!(matches!( + checked_f32_batch_span( + exact_per_instance + 1, + 2, + "J2K MetalDirect repeated grayscale test span", + ), + Err(Error::MetalKernel { message }) if message.contains("overflow") + )); + + let span = checked_f32_batch_span(4, 2, "J2K MetalDirect repeated grayscale test span") + .expect("small repeated span"); + assert_eq!( + checked_f32_element_offset( + &span, + 1, + 3, + "J2K MetalDirect repeated grayscale test offset", + ) + .expect("last element of last instance"), + 7 * size_of::() + ); + assert!(matches!( + checked_f32_element_offset( + &span, + 1, + 4, + "J2K MetalDirect repeated grayscale test offset", + ), + Err(Error::MetalKernel { message }) if message.contains("outside its instance") + )); + assert!(matches!( + checked_f32_element_offset( + &span, + usize::MAX, + 0, + "J2K MetalDirect repeated grayscale test offset", + ), + Err(Error::MetalKernel { message }) if message.contains("outside its batch") + )); + } + + #[test] + fn f32_dimension_span_rejects_width_height_byte_overflow() { + assert!(matches!( + checked_f32_dimension_span( + u32::MAX, + u32::MAX, + 1, + "J2K MetalDirect repeated grayscale dimension span", + ), + Err(Error::MetalKernel { message }) if message.contains("overflow") + )); + } } diff --git a/crates/j2k-metal/src/compute/direct_status.rs b/crates/j2k-metal/src/compute/direct_status.rs index 0227762d..b9176810 100644 --- a/crates/j2k-metal/src/compute/direct_status.rs +++ b/crates/j2k-metal/src/compute/direct_status.rs @@ -2,28 +2,140 @@ use metal::Buffer; -use crate::{Error, MetalDirectFallbackReason}; +use crate::{buffer_pool::PooledBuffer, Error, MetalDirectFallbackReason}; -use super::{ - direct_buffers::{checked_buffer_read, checked_buffer_slice}, +use super::abi::{ J2kClassicStatus, J2kHtStatus, J2kIdwtStatus, J2kMctStatus, J2K_CLASSIC_STATUS_FAIL, J2K_CLASSIC_STATUS_OK, J2K_CLASSIC_STATUS_UNSUPPORTED, J2K_HT_STATUS_FAIL, J2K_HT_STATUS_OK, J2K_HT_STATUS_UNSUPPORTED, J2K_IDWT_STATUS_FAIL, J2K_IDWT_STATUS_OK, J2K_MCT_STATUS_FAIL, J2K_MCT_STATUS_OK, }; +use super::direct_buffers::{checked_buffer_read, checked_buffer_slice}; pub(super) enum DirectStatusCheck { - Classic { buffer: Buffer, len: usize }, - Ht { buffer: Buffer, len: usize }, + Classic { + buffer: Buffer, + len: usize, + }, + Ht { + buffer: Buffer, + len: usize, + source_indices: Option>, + recyclable_status: Option, + }, Idwt(Buffer), Mct(Buffer), } -pub(super) fn validate_direct_status(status_check: DirectStatusCheck) -> Result<(), Error> { +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum DirectStatusRetirementMode { + Validate, + RecycleWithoutRead, +} + +impl DirectStatusCheck { + pub(super) fn remap_ht_source(&mut self, source_index: usize) -> Result<(), Error> { + let Self::Ht { + len, + source_indices, + .. + } = self + else { + return Ok(()); + }; + let indices = source_indices.as_mut().ok_or(Error::MetalStateInvariant { + state: "HTJ2K Metal batch status attribution", + reason: "chunked HT status check has no source identity table", + })?; + if indices.len() != *len { + return Err(Error::MetalStateInvariant { + state: "HTJ2K Metal batch status attribution", + reason: "HT source identity count does not match status count", + }); + } + indices.fill(source_index); + Ok(()) + } + + pub(super) fn remap_ht_sources(&mut self, sources: &[usize]) -> Result<(), Error> { + let Self::Ht { + len, + source_indices, + .. + } = self + else { + return Ok(()); + }; + let indices = source_indices.as_mut().ok_or(Error::MetalStateInvariant { + state: "HTJ2K Metal stacked batch status attribution", + reason: "chunked HT status check has no source identity table", + })?; + if indices.len() != *len { + return Err(Error::MetalStateInvariant { + state: "HTJ2K Metal stacked batch status attribution", + reason: "HT source identity count does not match status count", + }); + } + for source in indices { + *source = *sources.get(*source).ok_or(Error::MetalStateInvariant { + state: "HTJ2K Metal stacked batch status attribution", + reason: "local HT source identity exceeds prepared group", + })?; + } + Ok(()) + } +} + +#[cfg(test)] +pub(super) fn validate_direct_status( + runtime: &super::MetalRuntime, + status_check: DirectStatusCheck, +) -> Result<(), Error> { + retire_direct_status_checks( + runtime, + core::iter::once(status_check), + DirectStatusRetirementMode::Validate, + ) +} + +pub(super) fn retire_direct_status_checks( + runtime: &super::MetalRuntime, + status_checks: impl IntoIterator, + mode: DirectStatusRetirementMode, +) -> Result<(), Error> { + let mut first_validation_error = None; + let mut first_recycle_error = None; + for mut status_check in status_checks { + if mode == DirectStatusRetirementMode::Validate { + if let Err(error) = validate_direct_status_contents(&status_check) { + if first_validation_error.is_none() { + first_validation_error = Some(error); + } + } + } + if let DirectStatusCheck::Ht { + recyclable_status, .. + } = &mut status_check + { + if let Some(owner) = recyclable_status.take() { + if let Err(error) = runtime.recycle_shared_buffer(owner) { + if first_recycle_error.is_none() { + first_recycle_error = Some(error); + } + } + } + } + } + first_validation_error + .or(first_recycle_error) + .map_or(Ok(()), Err) +} + +fn validate_direct_status_contents(status_check: &DirectStatusCheck) -> Result<(), Error> { match status_check { DirectStatusCheck::Classic { buffer, len } => { let statuses = - checked_buffer_slice::(&buffer, len, "classic direct status")?; + checked_buffer_slice::(buffer, *len, "classic direct status")?; if let Some(status) = statuses .iter() .copied() @@ -32,24 +144,34 @@ pub(super) fn validate_direct_status(status_check: DirectStatusCheck) -> Result< return Err(decode_classic_status_error(status)); } } - DirectStatusCheck::Ht { buffer, len } => { - let statuses = checked_buffer_slice::(&buffer, len, "HT direct status")?; - if let Some(status) = statuses + DirectStatusCheck::Ht { + buffer, + len, + source_indices, + .. + } => { + let statuses = checked_buffer_slice::(buffer, *len, "HT direct status")?; + if let Some((status_index, status)) = statuses .iter() .copied() - .find(|status| status.code != J2K_HT_STATUS_OK) + .enumerate() + .find(|(_, status)| status.code != J2K_HT_STATUS_OK) { - return Err(decode_ht_status_error(status)); + let source_index = source_indices + .as_ref() + .and_then(|indices| indices.get(status_index)) + .copied(); + return Err(decode_ht_status_error_for_source(status, source_index)); } } DirectStatusCheck::Idwt(buffer) => { - let status = checked_buffer_read::(&buffer, "IDWT direct status")?; + let status = checked_buffer_read::(buffer, "IDWT direct status")?; if status.code != J2K_IDWT_STATUS_OK { return Err(decode_idwt_status_error(status)); } } DirectStatusCheck::Mct(buffer) => { - let status = checked_buffer_read::(&buffer, "MCT direct status")?; + let status = checked_buffer_read::(buffer, "MCT direct status")?; if status.code != J2K_MCT_STATUS_OK { return Err(decode_mct_status_error(status)); } @@ -102,11 +224,16 @@ pub(super) fn decode_mct_status_error(status: J2kMctStatus) -> Error { } pub(super) fn decode_ht_status_error(status: J2kHtStatus) -> Error { + decode_ht_status_error_for_source(status, None) +} + +fn decode_ht_status_error_for_source(status: J2kHtStatus, source_index: Option) -> Error { + let source = source_index.map_or_else(String::new, |index| format!(" for source {index}")); if status.code == J2K_HT_STATUS_UNSUPPORTED { return Error::MetalDirectFallback { message: format!( - "HTJ2K Metal kernel unsupported HT kernel input (detail={})", - status.detail + "HTJ2K Metal kernel unsupported HT kernel input{source} (detail={})", + status.detail, ), reason: MetalDirectFallbackReason::UnsupportedRuntimeInput, }; @@ -116,6 +243,138 @@ pub(super) fn decode_ht_status_error(status: J2kHtStatus) -> Error { _ => "unexpected HT kernel status", }; Error::MetalKernel { - message: format!("HTJ2K Metal kernel {kind} (detail={})", status.detail), + message: format!( + "HTJ2K Metal kernel {kind}{source} (detail={})", + status.detail + ), + } +} + +#[cfg(test)] +mod ht_source_tests { + use super::{decode_ht_status_error_for_source, Error, J2kHtStatus, J2K_HT_STATUS_FAIL}; + + #[test] + fn ht_status_failure_names_the_responsible_batch_source() { + let error = decode_ht_status_error_for_source( + J2kHtStatus { + code: J2K_HT_STATUS_FAIL, + detail: 17, + ..J2kHtStatus::default() + }, + Some(9), + ); + + assert!(matches!( + error, + Error::MetalKernel { message } + if message.contains("source 9") && message.contains("detail=17") + )); + } +} + +#[cfg(test)] +mod retirement_tests { + use core::mem::size_of; + + use super::{ + retire_direct_status_checks, DirectStatusCheck, DirectStatusRetirementMode, Error, + J2kHtStatus, J2K_HT_STATUS_FAIL, J2K_HT_STATUS_OK, + }; + use crate::compute::MetalRuntime; + + fn pooled_ht_status(runtime: &MetalRuntime, status: J2kHtStatus) -> DirectStatusCheck { + let owner = runtime + .take_shared_buffer(size_of::()) + .expect("pooled HT status allocation"); + // SAFETY: This fresh shared buffer is exclusively owned by `owner`, + // and no GPU command can access it during this host-only regression. + unsafe { j2k_metal_support::checked_buffer_write(owner.buffer(), 0, &[status]) } + .expect("write HT status fixture"); + DirectStatusCheck::Ht { + buffer: owner.buffer().clone(), + len: 1, + source_indices: Some(vec![0]), + recyclable_status: Some(owner), + } + } + + #[test] + fn failed_first_status_still_retires_later_pooled_status() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let runtime = MetalRuntime::new().expect("Metal runtime"); + let first = pooled_ht_status( + &runtime, + J2kHtStatus { + code: J2K_HT_STATUS_FAIL, + detail: 41, + ..J2kHtStatus::default() + }, + ); + let second = pooled_ht_status( + &runtime, + J2kHtStatus { + code: J2K_HT_STATUS_OK, + ..J2kHtStatus::default() + }, + ); + + let error = retire_direct_status_checks( + &runtime, + vec![first, second], + DirectStatusRetirementMode::Validate, + ) + .expect_err("first status must fail validation"); + assert!(matches!( + error, + Error::MetalKernel { message } if message.contains("detail=41") + )); + assert_eq!( + runtime + .buffer_pool_diagnostics() + .expect("pool diagnostics") + .shared + .cached_buffers, + 2, + "both pooled statuses must be retired despite the first validation failure" + ); + } + + #[test] + fn recycle_without_read_retires_status_with_unreadable_metadata() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let runtime = MetalRuntime::new().expect("Metal runtime"); + let mut status = pooled_ht_status( + &runtime, + J2kHtStatus { + code: J2K_HT_STATUS_OK, + ..J2kHtStatus::default() + }, + ); + let DirectStatusCheck::Ht { len, .. } = &mut status else { + unreachable!("fixture is an HT status") + }; + *len = usize::MAX; + + retire_direct_status_checks( + &runtime, + core::iter::once(status), + DirectStatusRetirementMode::RecycleWithoutRead, + ) + .expect("command-failure retirement must not read status contents"); + assert_eq!( + runtime + .buffer_pool_diagnostics() + .expect("pool diagnostics") + .shared + .cached_buffers, + 1 + ); } } diff --git a/crates/j2k-metal/src/compute/direct_surface_pack.rs b/crates/j2k-metal/src/compute/direct_surface_pack.rs index 64919d4e..528e3694 100644 --- a/crates/j2k-metal/src/compute/direct_surface_pack.rs +++ b/crates/j2k-metal/src/compute/direct_surface_pack.rs @@ -1,13 +1,30 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use std::mem::size_of; + +use j2k_core::checked_surface_len; +use j2k_metal_support::{dispatch_2d_pipeline, dispatch_3d_pipeline}; + +use super::abi::{J2kPackParams, J2kRepeatedGrayPackParams}; use super::{ - checked_metal_surface_len, dispatch_2d_pipeline, dispatch_3d_pipeline, j2k_pack_scale_arrays, - j2k_scalar_pack_params, j2k_u32_param, new_compute_command_encoder, new_shared_buffer, size_of, - Buffer, CommandBufferRef, ComputeCommandEncoderRef, ComputePipelineState, Error, J2kPackParams, - J2kRepeatedGrayPackParams, MetalRuntime, NativeColorSpace, PixelFormat, Rect, Surface, + j2k_pack_scale_arrays, j2k_scalar_pack_params, j2k_u32_param, new_compute_command_encoder, + new_shared_buffer, Buffer, CommandBufferRef, ComputeCommandEncoderRef, ComputePipelineState, + Error, MetalRuntime, NativeColorSpace, PixelFormat, Rect, Surface, }; use crate::error::metal_kernel_support_error; +pub(in crate::compute) fn checked_metal_surface_len( + dims: (u32, u32), + bytes_per_pixel: usize, + context: &'static str, +) -> Result<(usize, usize), Error> { + checked_surface_len(dims, bytes_per_pixel, usize::MAX, context).map_err(|error| { + Error::MetalKernel { + message: format!("{context}: {error}"), + } + }) +} + #[cfg(target_os = "macos")] pub(super) fn copy_plane_samples( buffer: &mut Buffer, diff --git a/crates/j2k-metal/src/compute/direct_tier1.rs b/crates/j2k-metal/src/compute/direct_tier1.rs index ef8cff2e..a286b60c 100644 --- a/crates/j2k-metal/src/compute/direct_tier1.rs +++ b/crates/j2k-metal/src/compute/direct_tier1.rs @@ -20,12 +20,12 @@ pub(super) enum DirectTier1Mode { } #[cfg(test)] -fn record_direct_tier1_input_buffer_prepare() { - super::test_counters::record_direct_tier1_input_buffer_prepare(); +fn record_direct_tier1_input_buffer_prepare(runtime: &MetalRuntime) { + super::test_counters::record_direct_tier1_input_buffer_prepare(runtime); } #[cfg(not(test))] -fn record_direct_tier1_input_buffer_prepare() {} +fn record_direct_tier1_input_buffer_prepare(_runtime: &MetalRuntime) {} pub(super) fn prepare_direct_tier1_input_buffer( runtime: &MetalRuntime, @@ -34,7 +34,7 @@ pub(super) fn prepare_direct_tier1_input_buffer( ) -> Result { match mode { DirectTier1Mode::Metal => { - record_direct_tier1_input_buffer_prepare(); + record_direct_tier1_input_buffer_prepare(runtime); copied_slice_buffer(&runtime.device, data) } DirectTier1Mode::CpuUpload => Ok(runtime.tier1_dummy_buffer.clone()), @@ -66,6 +66,14 @@ pub(super) fn record_hybrid_stacked_component_batch(tier1_mode: DirectTier1Mode) } } +#[cfg(test)] +pub(super) fn record_stacked_component_batch() { + super::test_counters::record_stacked_component_batch(); +} + +#[cfg(not(test))] +pub(super) fn record_stacked_component_batch() {} + #[cfg(not(test))] pub(super) fn record_hybrid_stacked_component_batch(_tier1_mode: DirectTier1Mode) {} diff --git a/crates/j2k-metal/src/compute/encode_capacity.rs b/crates/j2k-metal/src/compute/encode_capacity.rs index bd293735..49ad5db2 100644 --- a/crates/j2k-metal/src/compute/encode_capacity.rs +++ b/crates/j2k-metal/src/compute/encode_capacity.rs @@ -2,13 +2,13 @@ use j2k_native::{EncodeProgressionOrder, J2kPacketizationProgressionOrder}; -use super::{ - J2kLosslessCodestreamAssemblyJob, J2kLosslessCodestreamBlockCodingMode, HT_PACKET_CAPACITY_ENV, - J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, +use super::abi::{ + HT_PACKET_CAPACITY_ENV, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, J2K_HT_ENCODE_BASE_OUTPUT_SIZE, J2K_HT_ENCODE_MAX_SAMPLES, J2K_HT_ENCODE_MEL_SIZE, J2K_HT_ENCODE_MS_BYTES_PER_SAMPLE_FLOOR, J2K_HT_ENCODE_MS_SIZE, J2K_HT_ENCODE_VLC_SIZE, }; +use super::{J2kLosslessCodestreamAssemblyJob, J2kLosslessCodestreamBlockCodingMode}; use crate::Error; const JP2K_SOC_MARKER_BYTES: usize = 2; diff --git a/crates/j2k-metal/src/compute/forward_transform.rs b/crates/j2k-metal/src/compute/forward_transform.rs index 8fb543c9..c2fb51bb 100644 --- a/crates/j2k-metal/src/compute/forward_transform.rs +++ b/crates/j2k-metal/src/compute/forward_transform.rs @@ -1,13 +1,19 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use std::mem::{size_of, size_of_val}; + +use j2k_metal_support::{dispatch_2d_pipeline, dispatch_3d_pipeline}; + +use crate::profile_env::{label_command_buffer, label_compute_encoder}; + +use super::abi::{ + J2kForwardDwt53BatchedParams, J2kForwardDwt53Params, J2kLosslessDeinterleaveParams, +}; use super::{ - checked_buffer_slice, commit_and_wait_metal, copied_slice_buffer, dispatch_2d_pipeline, - dispatch_3d_pipeline, label_command_buffer, label_compute_encoder, new_command_buffer, - new_compute_command_encoder, new_shared_buffer, size_of, size_of_val, with_runtime, Buffer, - CommandBufferRef, ComputePipelineState, Error, J2kDeinterleaveToF32Job, - J2kForwardDwt53BatchedParams, J2kForwardDwt53Level, J2kForwardDwt53Output, - J2kForwardDwt53Params, J2kForwardDwt97Level, J2kForwardDwt97Output, - J2kLosslessDeinterleaveParams, + checked_buffer_slice, commit_and_wait_metal, copied_slice_buffer, new_command_buffer, + new_compute_command_encoder, new_shared_buffer, with_runtime, Buffer, CommandBufferRef, + ComputePipelineState, Error, J2kDeinterleaveToF32Job, J2kForwardDwt53Level, + J2kForwardDwt53Output, J2kForwardDwt97Level, J2kForwardDwt97Output, }; #[cfg(target_os = "macos")] diff --git a/crates/j2k-metal/src/compute/ht_forward_reader_tests.rs b/crates/j2k-metal/src/compute/ht_forward_reader_tests.rs new file mode 100644 index 00000000..3fda597a --- /dev/null +++ b/crates/j2k-metal/src/compute/ht_forward_reader_tests.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Golden model for the HT `SigProp` forward reader's byte-stuffing rule. + +const HT_CLEANUP_SOURCE: &str = include_str!("../ht_cleanup.metal"); + +#[derive(Default)] +struct ForwardReaderModel { + reservoir: u64, + bits: u32, + unstuff: bool, +} + +#[derive(Default)] +struct ReverseReaderModel { + reservoir: u64, + bits: u32, + unstuff: bool, +} + +impl ReverseReaderModel { + fn push(&mut self, raw_byte: u8) { + let stuffed = self.unstuff && (raw_byte & 0x7f) == 0x7f; + let value = if stuffed { raw_byte & 0x7f } else { raw_byte }; + self.reservoir |= u64::from(value) << self.bits; + self.bits += 8 - u32::from(stuffed); + self.unstuff = raw_byte > 0x8f; + } +} + +impl ForwardReaderModel { + fn push(&mut self, raw_byte: u8) { + let valid_bits = 8 - u32::from(self.unstuff); + let value = if self.unstuff { + raw_byte & 0x7f + } else { + raw_byte + }; + self.reservoir |= u64::from(value) << self.bits; + self.bits += valid_bits; + self.unstuff = raw_byte == 0xff; + } +} + +#[test] +fn sigprop_forward_reader_discards_stuffed_msb_from_overlap_byte() { + let mut reader = ForwardReaderModel::default(); + for byte in [0xff, 0x80, 0x00, 0x00, 0x00] { + reader.push(byte); + } + + assert_eq!(reader.reservoir, 0x0000_00ff_u64); + assert_eq!(reader.bits, 39); + assert!(!reader.unstuff); +} + +#[test] +fn metal_sigprop_reader_masks_value_but_uses_raw_byte_for_next_state() { + assert!( + HT_CLEANUP_SOURCE + .contains("const uchar value = reader.unstuff ? raw_byte & uchar(0x7F) : raw_byte;"), + "Metal forward reader must discard the stuffed MSB after 0xFF" + ); + assert!( + HT_CLEANUP_SOURCE.contains("reader.unstuff = raw_byte == uchar(0xFF);"), + "next-byte unstuff state must be derived from the unmasked byte" + ); +} + +#[test] +fn magref_reverse_reader_discards_stuffed_msb_from_shared_byte() { + let mut reader = ReverseReaderModel { + unstuff: true, + ..ReverseReaderModel::default() + }; + for byte in [0xff, 0x00, 0x00, 0x00, 0x00] { + reader.push(byte); + } + + assert_eq!(reader.reservoir, 0x0000_007f_u64); + assert_eq!(reader.bits, 39); + assert!(!reader.unstuff); +} + +#[test] +fn metal_magref_reader_masks_stuffed_value_but_keeps_raw_next_state() { + assert!( + HT_CLEANUP_SOURCE + .contains("const uchar value = stuffed ? raw_byte & uchar(0x7F) : raw_byte;"), + "Metal reverse reader must discard its stuffed MSB" + ); + assert!( + HT_CLEANUP_SOURCE.contains("reader.unstuff = raw_byte > uchar(0x8F);"), + "reverse next-byte unstuff state must be derived from the raw byte" + ); +} diff --git a/crates/j2k-metal/src/compute/ht_sigprop_context_tests.rs b/crates/j2k-metal/src/compute/ht_sigprop_context_tests.rs new file mode 100644 index 00000000..185239b0 --- /dev/null +++ b/crates/j2k-metal/src/compute/ht_sigprop_context_tests.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Golden model for `SigProp` saved-row context at adjacent x-quads. + +const HT_CLEANUP_SOURCE: &str = include_str!("../ht_cleanup.metal"); + +fn save_sigprop_context(prev_row: &mut [u16], index: usize, new_sig: u32, current_sig: u32) { + let combined = new_sig | (current_sig & 0xffff); + prev_row[index] = u16::try_from(combined).expect("saved-row sigma fits u16"); +} + +#[test] +fn sigprop_context_update_does_not_overwrite_next_x_quad() { + let mut prev_row = [0x1111, 0x2222, 0x3333]; + save_sigprop_context(&mut prev_row, 0, 0x0000_0040, 0xaaaa_0001); + + assert_eq!(prev_row, [0x0041, 0x2222, 0x3333]); +} + +#[test] +fn metal_sigprop_context_saves_only_current_quad() { + assert!( + HT_CLEANUP_SOURCE.contains("const uint combined_sig = new_sig | (cs & 0xFFFFu);"), + "SigProp must exclude the next x-quad's lookahead sigma" + ); + assert!( + !HT_CLEANUP_SOURCE.contains("prev_row_sig[idx + 1u] = ushort(combined_sig >> 16u);"), + "SigProp must preserve the next x-quad's saved above-row context" + ); +} diff --git a/crates/j2k-metal/src/compute/lossless_prepare.rs b/crates/j2k-metal/src/compute/lossless_prepare.rs index a702afde..56ff1dcd 100644 --- a/crates/j2k-metal/src/compute/lossless_prepare.rs +++ b/crates/j2k-metal/src/compute/lossless_prepare.rs @@ -1,23 +1,34 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use std::mem::size_of; + +use j2k_metal_support::{dispatch_1d_pipeline, dispatch_2d_pipeline, dispatch_3d_pipeline}; + +use crate::profile_env::{ + hybrid_stage_signpost, label_command_buffer, label_compute_encoder, + metal_profile_coefficient_prep_split_commands_enabled, +}; + +use super::abi::{ + J2kForwardDwt53BatchedParams, J2kForwardDwt53Params, J2kForwardIctParams, J2kForwardRctParams, + J2kLosslessCoefficientJob, J2kLosslessDeinterleaveParams, J2kMctStatus, + J2kPacketPayloadCopyParams, J2kQuantizeSubbandParams, J2K_MCT_STATUS_OK, + PACKET_PAYLOAD_COPY_BYTES_PER_STRIPE, PACKET_PAYLOAD_COPY_STRIPES_PER_JOB, +}; +use super::forward_transform::{ + active_forward_dwt53_buffers, dispatch_forward_dwt53_batched_pass, dispatch_forward_dwt53_pass, +}; +use super::resident_tier1::J2kBatchedPacketPayloadCopyDispatch; #[cfg(test)] use super::test_counters; use super::{ - active_forward_dwt53_buffers, checked_buffer_read, checked_buffer_slice, commit_and_wait_metal, - copied_slice_buffer, decode_mct_status_error, dispatch_1d_pipeline, dispatch_2d_pipeline, - dispatch_3d_pipeline, dispatch_forward_dwt53_batched_pass, dispatch_forward_dwt53_pass, - hybrid_stage_signpost, label_command_buffer, label_compute_encoder, - metal_profile_coefficient_prep_split_commands_enabled, new_command_buffer, - new_compute_command_encoder, new_private_buffer, new_resident_encode_command_buffer, - new_shared_buffer, size_of, take_recyclable_private_buffer, with_runtime, - with_runtime_for_session, zeroed_shared_buffer, Buffer, CommandBuffer, CommandBufferRef, Error, - J2kBatchedPacketPayloadCopyDispatch, J2kForwardDwt53BatchedParams, J2kForwardDwt53Params, - J2kForwardIctParams, J2kForwardRctParams, J2kLosslessCoefficientJob, - J2kLosslessDeinterleaveParams, J2kLosslessDeviceBatchPrepareItem, J2kLosslessDeviceCodeBlock, - J2kLosslessDevicePrepareJob, J2kMctStatus, J2kPacketPayloadCopyParams, - J2kPreparedLosslessDeviceCodeBlocks, J2kQuantizeSubbandJob, J2kQuantizeSubbandParams, MTLSize, - MetalRuntime, J2K_MCT_STATUS_OK, PACKET_PAYLOAD_COPY_BYTES_PER_STRIPE, - PACKET_PAYLOAD_COPY_STRIPES_PER_JOB, + checked_buffer_read, checked_buffer_slice, commit_and_wait_metal, copied_slice_buffer, + decode_mct_status_error, new_command_buffer, new_compute_command_encoder, new_private_buffer, + new_resident_encode_command_buffer, new_shared_buffer, take_recyclable_private_buffer, + with_runtime, with_runtime_for_session, zeroed_shared_buffer, Buffer, CommandBuffer, + CommandBufferRef, Error, J2kLosslessDeviceBatchPrepareItem, J2kLosslessDeviceCodeBlock, + J2kLosslessDevicePrepareJob, J2kPreparedLosslessDeviceCodeBlocks, J2kQuantizeSubbandJob, + MTLSize, MetalRuntime, }; mod batch; diff --git a/crates/j2k-metal/src/compute/resident_codestream.rs b/crates/j2k-metal/src/compute/resident_codestream.rs index b3ecb091..60e775b3 100644 --- a/crates/j2k-metal/src/compute/resident_codestream.rs +++ b/crates/j2k-metal/src/compute/resident_codestream.rs @@ -1,56 +1,23 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::{ - build_resident_batch_packet_plan, checked_buffer_read, checked_buffer_slice, - classic_cod_block_style_from_flags, classic_encode_code_blocks_pipeline, - classic_encode_output_capacity_for_mode, classic_encode_segment_capacity, - classic_encode_sub_band_code, classic_packet_output_capacity, - classic_resident_style_flags_from_env, classic_tier1_gpu_token_pack_requested, - classic_tier1_gpu_token_pack_supported, classic_tier1_split_gpu_token_pack_requested, +use std::{ + mem::size_of, + time::{Duration, Instant}, +}; + +use j2k_metal_support::{dispatch_1d_pipeline, dispatch_single_thread}; + +use crate::profile_env::{ + classic_tier1_gpu_token_pack_requested, classic_tier1_split_gpu_token_pack_requested, classic_tier1_split_mq_byte_gpu_token_pack_disabled, - classic_tier1_split_mq_byte_gpu_token_pack_requested, codestream_progression_order_code, - commit_and_wait_metal, copied_recyclable_shared_slice_buffer, copied_slice_buffer, - decode_ht_status_error, dispatch_1d_pipeline, dispatch_batched_packet_payload_copy, - dispatch_classic_tier1_arithmetic_pack_profile, dispatch_classic_tier1_density_profile, - dispatch_classic_tier1_pass_plan_profile, dispatch_classic_tier1_raw_pack_profile, - dispatch_classic_tier1_split_token_emit_for_gpu_pack, - dispatch_classic_tier1_split_token_emit_profile, - dispatch_classic_tier1_split_token_pack_from_gpu_tokens, - dispatch_classic_tier1_symbol_plan_profile, dispatch_classic_tier1_token_emit_for_gpu_pack, - dispatch_classic_tier1_token_emit_profile, dispatch_classic_tier1_token_pack_from_gpu_tokens, - dispatch_single_thread, dispatch_zero_u32_buffer_in_encoder, encode_status_error, - finish_resident_encode_split_command_buffer, finish_resident_encode_split_command_buffer_timed, - ht_batch_output_word_count, ht_encode_output_capacity, ht_output_word_count, - ht_packet_output_capacity_for_mode, hybrid_stage_signpost, label_command_buffer, - label_compute_encoder, lossless_codestream_assembly_capacity, + classic_tier1_split_mq_byte_gpu_token_pack_requested, hybrid_stage_signpost, + label_command_buffer, label_compute_encoder, metal_profile_classic_tier1_arithmetic_pack_enabled, metal_profile_classic_tier1_density_enabled, metal_profile_classic_tier1_pass_plan_enabled, metal_profile_classic_tier1_raw_pack_enabled, metal_profile_classic_tier1_split_token_emit_enabled, metal_profile_classic_tier1_symbol_plan_enabled, metal_profile_classic_tier1_token_emit_enabled, metal_profile_stages_enabled, - new_blit_command_encoder, new_command_buffer, new_compute_command_encoder, new_private_buffer, - new_resident_encode_command_buffer, new_shared_buffer, packet_tree_node_count, - prepared_lossless_batch_tiles, schedule_classic_tier1_gpu_token_pack_readback, - schedule_resident_tier1_status_readback, size_of, take_recyclable_private_buffer, - wait_resident_lossless_codestream, with_runtime, with_runtime_for_session, - zeroed_recyclable_shared_buffer, zeroed_shared_buffer, Buffer, CommandBufferRef, - ComputeCommandEncoderRef, DirectStatusCheck, Duration, Error, ForeignType, Instant, - J2kBatchedPacketPayloadCopyDispatch, J2kClassicEncodeBatchJob, - J2kClassicEncodeOutputCapacityMode, J2kClassicEncodeStatus, J2kClassicSegment, - J2kCodestreamAssemblyStatus, J2kHtCleanupBatchJob, J2kHtCleanupParams, J2kHtEncodeBatchJob, - J2kHtEncodeStatus, J2kHtPacketOutputCapacityMode, J2kHtRepeatedBatchParams, J2kHtStatus, - J2kLosslessCodestreamAssemblyJob, J2kLosslessCodestreamAssemblyParams, - J2kLosslessCodestreamBlockCodingMode, J2kPacketBlock, J2kPacketDescriptor, - J2kPacketEncodeParams, J2kPacketEncodeStatus, J2kPacketPayloadCopyJob, J2kPacketResolution, - J2kPacketStateBlock, J2kPacketSubband, J2kPacketizationBlockCodingMode, - J2kPacketizationEncodeJob, J2kPendingResidentLosslessCodestream, - J2kPendingResidentLosslessCodestreamBatch, J2kResidentBatchEncodeItem, - J2kResidentEncodeGpuStage, J2kResidentEncodeGpuStageCommandBuffer, J2kResidentEncodeStageStats, - J2kResidentLosslessCodestream, J2kResidentPacketBlock, J2kResidentPacketBlockParams, - J2kResidentPacketizationEncodeJob, MTLSize, MetalRuntime, ResidentBatchPacketPlan, - ResidentBatchPacketPlanParams, ResidentLosslessTier1Metal, ResidentTier1StatusReadbackRequest, - J2K_ENCODE_STATUS_OK, J2K_HT_STATUS_OK, PACKET_PAYLOAD_COPY_STRIPES_PER_JOB, SIGNPOST_ENCODE_HYBRID_CLASSIC_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKETIZATION_COMMAND_ENCODE, SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_BUFFER_SETUP, SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_PLAN, @@ -65,6 +32,56 @@ use super::{ SIGNPOST_ENCODE_HYBRID_HT_TIER1_COMMAND_ENCODE, SIGNPOST_ENCODE_HYBRID_HT_TIER1_SETUP, }; +use super::abi::{ + J2kClassicEncodeBatchJob, J2kClassicEncodeStatus, J2kClassicSegment, + J2kCodestreamAssemblyStatus, J2kHtCleanupBatchJob, J2kHtCleanupParams, J2kHtEncodeBatchJob, + J2kHtEncodeStatus, J2kHtRepeatedBatchParams, J2kHtStatus, J2kLosslessCodestreamAssemblyParams, + J2kPacketBlock, J2kPacketDescriptor, J2kPacketEncodeParams, J2kPacketEncodeStatus, + J2kPacketPayloadCopyJob, J2kPacketResolution, J2kPacketStateBlock, J2kPacketSubband, + J2kResidentPacketBlock, J2kResidentPacketBlockParams, J2K_ENCODE_STATUS_OK, J2K_HT_STATUS_OK, + PACKET_PAYLOAD_COPY_STRIPES_PER_JOB, +}; +use super::decode_dispatch::{ + dispatch_zero_u32_buffer_in_encoder, ht_batch_output_word_count, ht_output_word_count, +}; +use super::lossless_prepare::dispatch_batched_packet_payload_copy; +use super::resident_tier1::{ + dispatch_classic_tier1_arithmetic_pack_profile, dispatch_classic_tier1_density_profile, + dispatch_classic_tier1_pass_plan_profile, dispatch_classic_tier1_raw_pack_profile, + dispatch_classic_tier1_split_token_emit_for_gpu_pack, + dispatch_classic_tier1_split_token_emit_profile, + dispatch_classic_tier1_split_token_pack_from_gpu_tokens, + dispatch_classic_tier1_symbol_plan_profile, dispatch_classic_tier1_token_emit_for_gpu_pack, + dispatch_classic_tier1_token_emit_profile, dispatch_classic_tier1_token_pack_from_gpu_tokens, + schedule_classic_tier1_gpu_token_pack_readback, schedule_resident_tier1_status_readback, + wait_resident_lossless_codestream, J2kBatchedPacketPayloadCopyDispatch, + ResidentLosslessTier1Metal, ResidentTier1StatusReadbackRequest, +}; +use super::tier1_encode::{classic_encode_sub_band_code, encode_status_error}; +use super::{ + build_resident_batch_packet_plan, checked_buffer_read, checked_buffer_slice, + classic_cod_block_style_from_flags, classic_encode_code_blocks_pipeline, + classic_encode_output_capacity_for_mode, classic_encode_segment_capacity, + classic_packet_output_capacity, classic_resident_style_flags_from_env, + classic_tier1_gpu_token_pack_supported, codestream_progression_order_code, + commit_and_wait_metal, copied_recyclable_shared_slice_buffer, copied_slice_buffer, + decode_ht_status_error, finish_resident_encode_split_command_buffer, + finish_resident_encode_split_command_buffer_timed, ht_encode_output_capacity, + ht_packet_output_capacity_for_mode, lossless_codestream_assembly_capacity, + new_blit_command_encoder, new_command_buffer, new_compute_command_encoder, new_private_buffer, + new_resident_encode_command_buffer, new_shared_buffer, packet_tree_node_count, + prepared_lossless_batch_tiles, take_recyclable_private_buffer, with_runtime, + with_runtime_for_session, zeroed_recyclable_shared_buffer, zeroed_shared_buffer, Buffer, + ComputeCommandEncoderRef, Error, ForeignType, J2kClassicEncodeOutputCapacityMode, + J2kHtPacketOutputCapacityMode, J2kLosslessCodestreamAssemblyJob, + J2kLosslessCodestreamBlockCodingMode, J2kPacketizationBlockCodingMode, + J2kPacketizationEncodeJob, J2kPendingResidentLosslessCodestream, + J2kPendingResidentLosslessCodestreamBatch, J2kResidentBatchEncodeItem, + J2kResidentEncodeGpuStage, J2kResidentEncodeGpuStageCommandBuffer, J2kResidentEncodeStageStats, + J2kResidentLosslessCodestream, J2kResidentPacketizationEncodeJob, MTLSize, MetalRuntime, + ResidentBatchPacketPlan, ResidentBatchPacketPlanParams, +}; + mod batch_reporting; mod classic_labels; mod ht_cleanup; @@ -77,8 +94,9 @@ use self::classic_labels::{ }; pub(super) use self::ht_cleanup::{ dispatch_ht_cleanup, dispatch_ht_cleanup_batched, - dispatch_ht_cleanup_batched_in_command_buffer, dispatch_ht_cleanup_batched_in_encoder, - dispatch_ht_cleanup_repeated_batched_in_command_buffer, HtRepeatedCleanupDispatch, + dispatch_ht_cleanup_batched_in_encoder_with_status_offset, + dispatch_ht_cleanup_repeated_batched_in_encoder_with_status_offset, HtCleanupBatchDispatch, + HtCleanupRepeatedBatchDispatch, }; mod classic_packet; diff --git a/crates/j2k-metal/src/compute/resident_codestream/ht_cleanup.rs b/crates/j2k-metal/src/compute/resident_codestream/ht_cleanup.rs index d8ec2035..d5fc6576 100644 --- a/crates/j2k-metal/src/compute/resident_codestream/ht_cleanup.rs +++ b/crates/j2k-metal/src/compute/resident_codestream/ht_cleanup.rs @@ -5,10 +5,35 @@ use super::{ checked_buffer_read, checked_buffer_slice, commit_and_wait_metal, copied_slice_buffer, decode_ht_status_error, dispatch_single_thread, dispatch_zero_u32_buffer_in_encoder, ht_batch_output_word_count, ht_output_word_count, new_command_buffer, - new_compute_command_encoder, size_of, zeroed_shared_buffer, Buffer, CommandBufferRef, - ComputeCommandEncoderRef, DirectStatusCheck, Error, J2kHtCleanupBatchJob, J2kHtCleanupParams, - J2kHtRepeatedBatchParams, J2kHtStatus, MTLSize, MetalRuntime, J2K_HT_STATUS_OK, + new_compute_command_encoder, size_of, zeroed_shared_buffer, Buffer, ComputeCommandEncoderRef, + Error, J2kHtCleanupBatchJob, J2kHtCleanupParams, J2kHtRepeatedBatchParams, J2kHtStatus, + MTLSize, MetalRuntime, J2K_HT_STATUS_OK, }; +#[cfg(target_os = "macos")] +use crate::compute::decode_dispatch::MetalHtPipelineKind; + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +pub(in crate::compute) struct HtCleanupBatchDispatch<'a> { + pub(in crate::compute) coded_data: &'a Buffer, + pub(in crate::compute) jobs: &'a Buffer, + pub(in crate::compute) job_count: usize, + pub(in crate::compute) decoded: &'a Buffer, + pub(in crate::compute) status_buffer: &'a Buffer, + pub(in crate::compute) status_offset_bytes: u64, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +pub(in crate::compute) struct HtCleanupRepeatedBatchDispatch<'a> { + pub(in crate::compute) coded_data: &'a Buffer, + pub(in crate::compute) jobs: &'a Buffer, + pub(in crate::compute) base_job_count: usize, + pub(in crate::compute) repeated: J2kHtRepeatedBatchParams, + pub(in crate::compute) decoded: &'a Buffer, + pub(in crate::compute) status_buffer: &'a Buffer, + pub(in crate::compute) status_offset_bytes: u64, +} #[cfg(target_os = "macos")] pub(in crate::compute) fn dispatch_ht_cleanup( @@ -33,6 +58,7 @@ pub(in crate::compute) fn dispatch_ht_cleanup( params.height, )?, )?; + encoder.memory_barrier_with_resources(&[decoded]); encoder.set_compute_pipeline_state(&runtime.ht_cleanup); encoder.set_buffer(0, Some(&input), 0); encoder.set_buffer(1, Some(decoded), 0); @@ -80,6 +106,7 @@ pub(in crate::compute) fn dispatch_ht_cleanup_batched( decoded, ht_batch_output_word_count(jobs)?, )?; + encoder.memory_barrier_with_resources(&[decoded]); encoder.set_compute_pipeline_state(&runtime.ht_cleanup_batched); encoder.set_buffer(0, Some(&input), 0); encoder.set_buffer(1, Some(decoded), 0); @@ -122,94 +149,41 @@ pub(in crate::compute) fn dispatch_ht_cleanup_batched( Ok(()) } -#[cfg(target_os = "macos")] -pub(in crate::compute) fn dispatch_ht_cleanup_batched_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - coded_data: &Buffer, - jobs: &Buffer, - job_count: usize, - decoded: &Buffer, - decoded_word_count: usize, -) -> Result { - let status_buffer = - zeroed_shared_buffer(&runtime.device, job_count.max(1) * size_of::())?; - - let encoder = new_compute_command_encoder(command_buffer)?; - dispatch_zero_u32_buffer_in_encoder(runtime, &encoder, decoded, decoded_word_count)?; - dispatch_ht_cleanup_batched_in_encoder_with_status( - runtime, - &encoder, - coded_data, - jobs, - job_count, - decoded, - &status_buffer, - ); - encoder.end_encoding(); - - Ok(DirectStatusCheck::Ht { - buffer: status_buffer, - len: job_count, - }) -} - -#[cfg(target_os = "macos")] -pub(in crate::compute) fn dispatch_ht_cleanup_batched_in_encoder( - runtime: &MetalRuntime, - encoder: &ComputeCommandEncoderRef, - coded_data: &Buffer, - jobs: &Buffer, - job_count: usize, - decoded: &Buffer, - decoded_word_count: usize, -) -> Result { - let status_buffer = - zeroed_shared_buffer(&runtime.device, job_count.max(1) * size_of::())?; - dispatch_zero_u32_buffer_in_encoder(runtime, encoder, decoded, decoded_word_count)?; - dispatch_ht_cleanup_batched_in_encoder_with_status( - runtime, - encoder, - coded_data, - jobs, - job_count, - decoded, - &status_buffer, - ); - - Ok(DirectStatusCheck::Ht { - buffer: status_buffer, - len: job_count, - }) -} +#[cfg(all(test, target_os = "macos"))] +mod tests; #[cfg(target_os = "macos")] -pub(in crate::compute) fn dispatch_ht_cleanup_batched_in_encoder_with_status( +pub(in crate::compute) fn dispatch_ht_cleanup_batched_in_encoder_with_status_offset( runtime: &MetalRuntime, encoder: &ComputeCommandEncoderRef, - coded_data: &Buffer, - jobs: &Buffer, - job_count: usize, - decoded: &Buffer, - status_buffer: &Buffer, + pipeline_kind: MetalHtPipelineKind, + dispatch: HtCleanupBatchDispatch<'_>, ) { - encoder.set_compute_pipeline_state(&runtime.ht_cleanup_batched); - encoder.set_buffer(0, Some(coded_data), 0); - encoder.set_buffer(1, Some(decoded), 0); - encoder.set_buffer(2, Some(jobs), 0); + let pipeline = match pipeline_kind { + MetalHtPipelineKind::CleanupOnly => &runtime.ht_cleanup_batched_cleanup_only, + MetalHtPipelineKind::SigProp => &runtime.ht_cleanup_batched_sigprop, + MetalHtPipelineKind::MagRef => &runtime.ht_cleanup_batched_magref, + }; + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(dispatch.coded_data), 0); + encoder.set_buffer(1, Some(dispatch.decoded), 0); + encoder.set_buffer(2, Some(dispatch.jobs), 0); encoder.set_buffer(3, Some(&runtime.ht_vlc_table0), 0); encoder.set_buffer(4, Some(&runtime.ht_vlc_table1), 0); encoder.set_buffer(5, Some(&runtime.ht_uvlc_table0), 0); encoder.set_buffer(6, Some(&runtime.ht_uvlc_table1), 0); - encoder.set_buffer(7, Some(status_buffer), 0); - let width = runtime - .ht_cleanup_batched + encoder.set_buffer( + 7, + Some(dispatch.status_buffer), + dispatch.status_offset_bytes, + ); + let width = pipeline .thread_execution_width() .max(1) - .min(job_count as u64); + .min(dispatch.job_count as u64); encoder.dispatch_threads( MTLSize { - width: job_count as u64, + width: dispatch.job_count as u64, height: 1, depth: 1, }, @@ -222,85 +196,43 @@ pub(in crate::compute) fn dispatch_ht_cleanup_batched_in_encoder_with_status( } #[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -pub(in crate::compute) struct HtRepeatedCleanupDispatch<'a> { - pub(in crate::compute) runtime: &'a MetalRuntime, - pub(in crate::compute) command_buffer: &'a CommandBufferRef, - pub(in crate::compute) coded_data: &'a Buffer, - pub(in crate::compute) jobs: &'a Buffer, - pub(in crate::compute) base_job_count: usize, - pub(in crate::compute) total_job_count: usize, - pub(in crate::compute) output_plane_len: usize, - pub(in crate::compute) decoded: &'a Buffer, -} - -#[cfg(target_os = "macos")] -pub(in crate::compute) fn dispatch_ht_cleanup_repeated_batched_in_command_buffer( - dispatch: HtRepeatedCleanupDispatch<'_>, -) -> Result { - let HtRepeatedCleanupDispatch { - runtime, - command_buffer, - coded_data, - jobs, - base_job_count, - total_job_count, - output_plane_len, - decoded, - } = dispatch; - let status_buffer = zeroed_shared_buffer( - &runtime.device, - total_job_count.max(1) * size_of::(), - )?; - let batch_count = - total_job_count - .checked_div(base_job_count) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect repeated base job count is zero".to_string(), - })?; - let decoded_word_count = - output_plane_len - .checked_mul(batch_count) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect repeated output span overflow".to_string(), - })?; - let repeated = J2kHtRepeatedBatchParams { - job_count: u32::try_from(base_job_count).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect repeated base job count exceeds u32".to_string(), - })?, - output_plane_len: u32::try_from(output_plane_len).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect repeated output plane length exceeds u32".to_string(), - })?, - batch_count: u32::try_from(batch_count).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect repeated batch count exceeds u32".to_string(), - })?, +pub(in crate::compute) fn dispatch_ht_cleanup_repeated_batched_in_encoder_with_status_offset( + runtime: &MetalRuntime, + encoder: &ComputeCommandEncoderRef, + pipeline_kind: MetalHtPipelineKind, + dispatch: HtCleanupRepeatedBatchDispatch<'_>, +) { + let pipeline = match pipeline_kind { + MetalHtPipelineKind::CleanupOnly => &runtime.ht_cleanup_repeated_batched_cleanup_only, + MetalHtPipelineKind::SigProp => &runtime.ht_cleanup_repeated_batched_sigprop, + MetalHtPipelineKind::MagRef => &runtime.ht_cleanup_repeated_batched_magref, }; - - let encoder = new_compute_command_encoder(command_buffer)?; - dispatch_zero_u32_buffer_in_encoder(runtime, &encoder, decoded, decoded_word_count)?; - encoder.set_compute_pipeline_state(&runtime.ht_cleanup_repeated_batched); - encoder.set_buffer(0, Some(coded_data), 0); - encoder.set_buffer(1, Some(decoded), 0); - encoder.set_buffer(2, Some(jobs), 0); + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(dispatch.coded_data), 0); + encoder.set_buffer(1, Some(dispatch.decoded), 0); + encoder.set_buffer(2, Some(dispatch.jobs), 0); encoder.set_bytes( 3, size_of::() as u64, - (&raw const repeated).cast(), + (&raw const dispatch.repeated).cast(), ); encoder.set_buffer(4, Some(&runtime.ht_vlc_table0), 0); encoder.set_buffer(5, Some(&runtime.ht_vlc_table1), 0); encoder.set_buffer(6, Some(&runtime.ht_uvlc_table0), 0); encoder.set_buffer(7, Some(&runtime.ht_uvlc_table1), 0); - encoder.set_buffer(8, Some(&status_buffer), 0); - let width = runtime - .ht_cleanup_repeated_batched + encoder.set_buffer( + 8, + Some(dispatch.status_buffer), + dispatch.status_offset_bytes, + ); + let width = pipeline .thread_execution_width() .max(1) - .min(base_job_count as u64); + .min(dispatch.base_job_count as u64); encoder.dispatch_threads( MTLSize { - width: base_job_count as u64, - height: u64::from(repeated.batch_count), + width: dispatch.base_job_count as u64, + height: u64::from(dispatch.repeated.batch_count), depth: 1, }, MTLSize { @@ -309,10 +241,4 @@ pub(in crate::compute) fn dispatch_ht_cleanup_repeated_batched_in_command_buffer depth: 1, }, ); - encoder.end_encoding(); - - Ok(DirectStatusCheck::Ht { - buffer: status_buffer, - len: total_job_count, - }) } diff --git a/crates/j2k-metal/src/compute/resident_codestream/ht_cleanup/tests.rs b/crates/j2k-metal/src/compute/resident_codestream/ht_cleanup/tests.rs new file mode 100644 index 00000000..5ba1513d --- /dev/null +++ b/crates/j2k-metal/src/compute/resident_codestream/ht_cleanup/tests.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +fn function_body<'a>(source: &'a str, name: &str) -> &'a str { + source + .split_once(name) + .unwrap_or_else(|| panic!("missing {name}")) + .1 + .split("\n#[cfg(") + .next() + .expect("HT cleanup function body") +} + +#[test] +fn zero_fill_is_barriered_before_single_and_batched_ht_cleanup() { + let source = include_str!("../ht_cleanup.rs"); + for name in ["fn dispatch_ht_cleanup(", "fn dispatch_ht_cleanup_batched("] { + let body = function_body(source, name); + let zero_fill = body + .find("dispatch_zero_u32_buffer_in_encoder") + .unwrap_or_else(|| panic!("missing zero-fill dispatch in {name}")); + let barrier = body + .find("encoder.memory_barrier_with_resources(&[decoded]);") + .unwrap_or_else(|| panic!("missing decoded-buffer barrier in {name}")); + let cleanup = body + .find("encoder.set_compute_pipeline_state") + .unwrap_or_else(|| panic!("missing cleanup pipeline in {name}")); + assert!(zero_fill < barrier && barrier < cleanup, "{name}"); + } +} diff --git a/crates/j2k-metal/src/compute/resident_packet_plan.rs b/crates/j2k-metal/src/compute/resident_packet_plan.rs index 18563b74..307d69a8 100644 --- a/crates/j2k-metal/src/compute/resident_packet_plan.rs +++ b/crates/j2k-metal/src/compute/resident_packet_plan.rs @@ -3,15 +3,18 @@ use j2k_native::J2kPacketizationPacketDescriptor; use metal::{Buffer, CommandBuffer}; +use super::abi::{ + J2kBatchedCodestreamAssemblyJob, J2kBatchedPacketEncodeJob, J2kPacketDescriptor, + J2kPacketResolution, J2kPacketStateBlock, J2kPacketSubband, J2kResidentPacketBlock, +}; use super::{ encode_capacity::{ codestream_progression_order_code, lossless_codestream_assembly_capacity, lossless_codestream_payload_offset, packet_tree_node_count, }, - J2kBatchedCodestreamAssemblyJob, J2kBatchedPacketEncodeJob, J2kLosslessCodestreamAssemblyJob, - J2kLosslessDeviceCodeBlock, J2kPacketDescriptor, J2kPacketResolution, J2kPacketStateBlock, - J2kPacketSubband, J2kPreparedLosslessDeviceCodeBlocks, J2kResidentBatchEncodeItem, - J2kResidentPacketBlock, J2kResidentPacketizationResolution, + J2kLosslessCodestreamAssemblyJob, J2kLosslessDeviceCodeBlock, + J2kPreparedLosslessDeviceCodeBlocks, J2kResidentBatchEncodeItem, + J2kResidentPacketizationResolution, }; use crate::Error; diff --git a/crates/j2k-metal/src/compute/resident_tier1.rs b/crates/j2k-metal/src/compute/resident_tier1.rs index 46dcfad5..0a1cba12 100644 --- a/crates/j2k-metal/src/compute/resident_tier1.rs +++ b/crates/j2k-metal/src/compute/resident_tier1.rs @@ -1,36 +1,46 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -#[cfg(test)] -use super::test_counters; -use super::{ - accumulate_classic_tier1_scan_estimates, checked_buffer_read, checked_buffer_slice, - classic_encode_code_blocks_pipeline_kind, classic_tier1_gpu_token_pack_supported, - classic_tier1_pass_class_counts, completed_command_buffers_gpu_duration, - completed_command_buffers_gpu_duration_and_elapsed_window, dispatch_1d_pipeline, - duration_share, encode_status_error, hybrid_stage_signpost, label_compute_encoder, +use std::{mem::size_of, sync::Arc, time::Instant}; + +use j2k_metal_support::dispatch_1d_pipeline; + +use crate::profile_env::{ + hybrid_stage_signpost, label_compute_encoder, metal_profile_classic_tier1_arithmetic_pack_enabled, metal_profile_classic_tier1_density_enabled, metal_profile_classic_tier1_pass_plan_enabled, metal_profile_classic_tier1_raw_pack_enabled, metal_profile_classic_tier1_split_token_emit_enabled, metal_profile_classic_tier1_symbol_plan_enabled, metal_profile_classic_tier1_token_emit_enabled, metal_profile_classic_tier1_token_pack_enabled, - metal_profile_stages_enabled, new_blit_command_encoder, new_compute_command_encoder, - new_private_buffer, new_shared_buffer, pack_j2k_code_block_scalar_from_tier1_tokens, - packet_encode_status_error, record_completed_resident_encode_gpu_stages, - recycle_private_buffers, recycle_shared_buffers, size_of, take_recyclable_private_buffer, - wait_for_completion_metal, Arc, Buffer, CommandBuffer, CommandBufferRef, ComputePipelineState, - EncodeProgressionOrder, Error, HybridSignpostName, Instant, IntoParallelIterator, - J2kClassicEncodeBatchJob, J2kClassicEncodePipelineKind, J2kClassicEncodeStatus, - J2kClassicTier1DensityCounters, J2kClassicTier1PassPlanCounters, - J2kClassicTier1SymbolPlanCounters, J2kClassicTier1TokenSegment, J2kCodestreamAssemblyStatus, - J2kHtEncodeBatchJob, J2kHtEncodeStatus, J2kPacketEncodeStatus, + metal_profile_stages_enabled, HybridSignpostName, SIGNPOST_ENCODE_HYBRID_COMMAND_WAIT, + SIGNPOST_ENCODE_HYBRID_RESULT_HARVEST, +}; + +use super::abi::{ + J2kClassicEncodeBatchJob, J2kClassicEncodeStatus, J2kClassicTier1DensityCounters, + J2kClassicTier1PassPlanCounters, J2kClassicTier1SymbolPlanCounters, + J2kClassicTier1TokenSegment, J2kCodestreamAssemblyStatus, J2kHtEncodeBatchJob, + J2kHtEncodeStatus, J2kPacketEncodeStatus, CLASSIC_TIER1_MQ_BYTE_TOKEN_ARENA_BYTES, + CLASSIC_TIER1_TOKEN_ARENA_BYTES, CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY, J2K_ENCODE_STATUS_OK, + PACKET_PAYLOAD_COPY_STRIPES_PER_JOB, +}; +#[cfg(test)] +use super::test_counters; +use super::tier1_encode::{encode_status_error, packet_encode_status_error}; +use super::{ + accumulate_classic_tier1_scan_estimates, checked_buffer_read, checked_buffer_slice, + classic_encode_code_blocks_pipeline_kind, classic_tier1_gpu_token_pack_supported, + classic_tier1_pass_class_counts, completed_command_buffers_gpu_duration, + completed_command_buffers_gpu_duration_and_elapsed_window, duration_share, + new_blit_command_encoder, new_compute_command_encoder, new_private_buffer, new_shared_buffer, + pack_j2k_code_block_scalar_from_tier1_tokens, record_completed_resident_encode_gpu_stages, + recycle_private_buffers, recycle_shared_buffers, take_recyclable_private_buffer, + wait_for_completion_metal, Buffer, CommandBuffer, CommandBufferRef, ComputePipelineState, + EncodeProgressionOrder, Error, IntoParallelIterator, J2kClassicEncodePipelineKind, J2kPacketizationPacketDescriptor, J2kPendingResidentLosslessCodestream, J2kResidentEncodeGpuStageCommandBuffer, J2kResidentEncodeStageStats, J2kResidentLosslessCodestream, J2kResidentLosslessCodestreamBatchResult, J2kTier1TokenSegment, - MTLSize, MetalRuntime, ParallelIterator, CLASSIC_TIER1_MQ_BYTE_TOKEN_ARENA_BYTES, - CLASSIC_TIER1_TOKEN_ARENA_BYTES, CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY, J2K_ENCODE_STATUS_OK, - PACKET_PAYLOAD_COPY_STRIPES_PER_JOB, SIGNPOST_ENCODE_HYBRID_COMMAND_WAIT, - SIGNPOST_ENCODE_HYBRID_RESULT_HARVEST, + MTLSize, MetalRuntime, ParallelIterator, }; mod counter_validation; diff --git a/crates/j2k-metal/src/compute/runtime.rs b/crates/j2k-metal/src/compute/runtime.rs index de0289d5..fd71a254 100644 --- a/crates/j2k-metal/src/compute/runtime.rs +++ b/crates/j2k-metal/src/compute/runtime.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::{cell::RefCell, sync::Arc}; +use std::{ + cell::RefCell, + sync::{Arc, Mutex}, +}; use j2k_metal_support::{ checked_command_queue, checked_shared_buffer, checked_shared_buffer_with_slice, @@ -33,7 +36,7 @@ thread_local! { pub(crate) struct MetalRuntime { pub(super) device: Device, - pub(super) queue: CommandQueue, + pub(crate) queue: CommandQueue, pub(super) zero_u32_buffer: ComputePipelineState, pub(super) validate_bytes_equal: ComputePipelineState, pub(super) copy_interleaved_padded: ComputePipelineState, @@ -82,13 +85,26 @@ pub(crate) struct MetalRuntime { pub(super) store_component_repeated: ComputePipelineState, pub(super) store_component_repeated_gray_u8: ComputePipelineState, pub(super) store_component_repeated_gray_u16: ComputePipelineState, + pub(super) store_component_repeated_gray_i16: ComputePipelineState, pub(super) store_component_repeated_gray_u8_contiguous: ComputePipelineState, pub(super) store_component_repeated_gray_u16_contiguous: ComputePipelineState, pub(super) store_component_gray_u8: ComputePipelineState, pub(super) store_component_gray_u16: ComputePipelineState, + pub(super) store_component_gray_i16: ComputePipelineState, + pub(super) store_native_rgb_batch_u8: ComputePipelineState, + pub(super) store_native_rgb_batch_u16: ComputePipelineState, + pub(super) store_native_rgb_batch_i16: ComputePipelineState, + pub(super) store_native_rgba_batch_u8: ComputePipelineState, + pub(super) store_native_rgba_batch_u16: ComputePipelineState, + pub(super) store_native_rgba_batch_i16: ComputePipelineState, pub(super) ht_cleanup: ComputePipelineState, pub(super) ht_cleanup_batched: ComputePipelineState, - pub(super) ht_cleanup_repeated_batched: ComputePipelineState, + pub(super) ht_cleanup_batched_cleanup_only: ComputePipelineState, + pub(super) ht_cleanup_batched_sigprop: ComputePipelineState, + pub(super) ht_cleanup_batched_magref: ComputePipelineState, + pub(super) ht_cleanup_repeated_batched_cleanup_only: ComputePipelineState, + pub(super) ht_cleanup_repeated_batched_sigprop: ComputePipelineState, + pub(super) ht_cleanup_repeated_batched_magref: ComputePipelineState, pub(super) classic_encode_code_block: ComputePipelineState, pub(super) classic_encode_code_blocks: ComputePipelineState, pub(super) classic_encode_code_blocks_32: ComputePipelineState, @@ -125,6 +141,8 @@ pub(crate) struct MetalRuntime { pub(super) ht_uvlc_encode_table: Buffer, pub(super) tier1_dummy_buffer: Buffer, pub(super) buffer_pools: MetalBufferPools, + pub(in crate::compute) prepared_ht_execution_cache: + Mutex, } impl MetalRuntime { @@ -134,15 +152,22 @@ impl MetalRuntime { Self::new_with_device(&device) } + pub(crate) fn new_with_device(device: &Device) -> Result { + let queue = checked_command_queue(device)?; + Self::new_with_device_and_queue(device, queue) + } + #[expect( clippy::too_many_lines, reason = "pipeline inventory construction mirrors the fixed Metal runtime ABI" )] - pub(crate) fn new_with_device(device: &Device) -> Result { + pub(crate) fn new_with_device_and_queue( + device: &Device, + queue: CommandQueue, + ) -> Result { let shader_source = shader_source(); let loader = MetalPipelineLoader::new(device, &shader_source)?; let pipeline = |name: &str| loader.pipeline(name); - let queue = checked_command_queue(device)?; let ht_uvlc_encode_rows = (*ht_uvlc_encode_table()).map(J2kHtUvlcEncodeTableEntry::from); Ok(Self { device: device.clone(), @@ -211,6 +236,7 @@ impl MetalRuntime { store_component_repeated: pipeline("j2k_store_component_repeated")?, store_component_repeated_gray_u8: pipeline("j2k_store_component_repeated_gray_u8")?, store_component_repeated_gray_u16: pipeline("j2k_store_component_repeated_gray_u16")?, + store_component_repeated_gray_i16: pipeline("j2k_store_component_repeated_gray_i16")?, store_component_repeated_gray_u8_contiguous: pipeline( "j2k_store_component_repeated_gray_u8_contiguous", )?, @@ -219,9 +245,29 @@ impl MetalRuntime { )?, store_component_gray_u8: pipeline("j2k_store_component_gray_u8")?, store_component_gray_u16: pipeline("j2k_store_component_gray_u16")?, + store_component_gray_i16: pipeline("j2k_store_component_gray_i16")?, + store_native_rgb_batch_u8: pipeline("j2k_store_native_rgb_batch_u8")?, + store_native_rgb_batch_u16: pipeline("j2k_store_native_rgb_batch_u16")?, + store_native_rgb_batch_i16: pipeline("j2k_store_native_rgb_batch_i16")?, + store_native_rgba_batch_u8: pipeline("j2k_store_native_rgba_batch_u8")?, + store_native_rgba_batch_u16: pipeline("j2k_store_native_rgba_batch_u16")?, + store_native_rgba_batch_i16: pipeline("j2k_store_native_rgba_batch_i16")?, ht_cleanup: pipeline("j2k_decode_ht_cleanup")?, ht_cleanup_batched: pipeline("j2k_decode_ht_cleanup_batched")?, - ht_cleanup_repeated_batched: pipeline("j2k_decode_ht_cleanup_repeated_batched")?, + ht_cleanup_batched_cleanup_only: pipeline( + "j2k_decode_ht_cleanup_batched_cleanup_only", + )?, + ht_cleanup_batched_sigprop: pipeline("j2k_decode_ht_cleanup_batched_sigprop")?, + ht_cleanup_batched_magref: pipeline("j2k_decode_ht_cleanup_batched_magref")?, + ht_cleanup_repeated_batched_cleanup_only: pipeline( + "j2k_decode_ht_cleanup_repeated_batched_cleanup_only", + )?, + ht_cleanup_repeated_batched_sigprop: pipeline( + "j2k_decode_ht_cleanup_repeated_batched_sigprop", + )?, + ht_cleanup_repeated_batched_magref: pipeline( + "j2k_decode_ht_cleanup_repeated_batched_magref", + )?, classic_encode_code_block: pipeline("j2k_encode_classic_code_block")?, classic_encode_code_blocks: pipeline("j2k_encode_classic_code_blocks")?, classic_encode_code_blocks_32: pipeline("j2k_encode_classic_code_blocks_32")?, @@ -290,6 +336,9 @@ impl MetalRuntime { ht_uvlc_encode_table: checked_shared_buffer_with_slice(device, &ht_uvlc_encode_rows)?, tier1_dummy_buffer: checked_shared_buffer(device, 1)?, buffer_pools: MetalBufferPools::new(device), + prepared_ht_execution_cache: Mutex::new( + super::decode_dispatch::PreparedMetalHtExecutionCache::new(), + ), }) } diff --git a/crates/j2k-metal/src/compute/shader_source.rs b/crates/j2k-metal/src/compute/shader_source.rs index 57697f7a..d03d0481 100644 --- a/crates/j2k-metal/src/compute/shader_source.rs +++ b/crates/j2k-metal/src/compute/shader_source.rs @@ -562,6 +562,8 @@ kernel void j2k_pack_u16_repeated_gray( "\n", include_str!("../store.metal"), "\n", + include_str!("../store_native_color_batch.metal"), + "\n", include_str!("../ht_cleanup.metal"), ] .concat() diff --git a/crates/j2k-metal/src/compute/symbol_inventory.rs b/crates/j2k-metal/src/compute/symbol_inventory.rs deleted file mode 100644 index e862f37e..00000000 --- a/crates/j2k-metal/src/compute/symbol_inventory.rs +++ /dev/null @@ -1,219 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -//! Explicit symbol wiring for the `compute` facade. -//! -//! Keeping this inventory separate prevents import bookkeeping from obscuring -//! the facade's runtime helpers while preserving its established root paths. - -macro_rules! wire_compute_symbols { - () => { - pub(crate) use crate::compute::abi::J2K_HT_STATUS_OK; - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::abi::{ - J2kBatchedCodestreamAssemblyJob, J2kBatchedMctRgb8PackParams, - J2kBatchedPacketEncodeJob, J2kClassicCleanupBatchJob, J2kClassicEncodeBatchJob, - J2kClassicEncodeParams, J2kClassicEncodeStatus, J2kClassicRepeatedBatchParams, - J2kClassicSegment, J2kClassicStatus, J2kClassicTier1DensityCounters, - J2kClassicTier1PassPlanCounters, J2kClassicTier1SymbolPlanCounters, - J2kClassicTier1TokenSegment, J2kCodestreamAssemblyStatus, J2kCopyInterleavedParams, - J2kForwardDwt53BatchedParams, J2kForwardDwt53Params, J2kForwardIctParams, - J2kForwardRctParams, J2kGrayStoreParams, J2kHtCleanupBatchJob, J2kHtCleanupParams, - J2kHtEncodeBatchJob, J2kHtEncodeParams, J2kHtEncodeStatus, J2kHtRepeatedBatchParams, - J2kHtStatus, J2kIdwt97StepParams, J2kIdwtSingleDecompositionParams, J2kIdwtStatus, - J2kInverseMctParams, - J2kLosslessCodestreamAssemblyParams, J2kLosslessCoefficientJob, - J2kLosslessDeinterleaveParams, J2kMctRgb8PackParams, J2kMctStatus, J2kPackParams, - J2kPacketBlock, J2kPacketDescriptor, J2kPacketEncodeParams, J2kPacketEncodeStatus, - J2kPacketPayloadCopyJob, J2kPacketPayloadCopyParams, J2kPacketResolution, - J2kPacketStateBlock, J2kPacketSubband, J2kQuantizeSubbandParams, - J2kRepeatedGrayPackParams, J2kRepeatedGrayStoreParams, - J2kRepeatedIdwtSingleDecompositionParams, J2kRepeatedStoreParams, - J2kResidentPacketBlock, J2kResidentPacketBlockParams, J2kStoreParams, - J2kValidateBytesParams, J2kValidateBytesStatus, - CLASSIC_TIER1_MQ_BYTE_TOKEN_ARENA_BYTES, CLASSIC_TIER1_TOKEN_ARENA_BYTES, - CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY, HT_PACKET_CAPACITY_ENV, - J2K_CLASSIC_ENCODE_32_MAX_HEIGHT, J2K_CLASSIC_ENCODE_32_MAX_WIDTH, - J2K_CLASSIC_MAX_COEFF_COUNT, J2K_CLASSIC_MAX_HEIGHT, J2K_CLASSIC_MAX_WIDTH, - J2K_CLASSIC_STATUS_FAIL, J2K_CLASSIC_STATUS_OK, J2K_CLASSIC_STATUS_UNSUPPORTED, - J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES, - J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS, - J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, - J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, - J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT, J2K_ENCODE_STATUS_FAIL, - J2K_ENCODE_STATUS_OK, J2K_ENCODE_STATUS_UNSUPPORTED, - J2K_HT_ENCODE_BASE_OUTPUT_SIZE, J2K_HT_ENCODE_MAX_SAMPLES, J2K_HT_ENCODE_MEL_SIZE, - J2K_HT_ENCODE_MS_BYTES_PER_SAMPLE_FLOOR, J2K_HT_ENCODE_MS_SIZE, - J2K_HT_ENCODE_VLC_SIZE, J2K_HT_STATUS_FAIL, J2K_HT_STATUS_UNSUPPORTED, - J2K_IDWT_STATUS_FAIL, J2K_IDWT_STATUS_OK, J2K_MCT_STATUS_FAIL, J2K_MCT_STATUS_OK, - PACKET_PAYLOAD_COPY_BYTES_PER_STRIPE, PACKET_PAYLOAD_COPY_STRIPES_PER_JOB, - }; - - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::direct_prepare::prepare_direct_grayscale_plan; - #[cfg(all(target_os = "macos", test))] - use crate::compute::direct_prepare::prepare_sub_band_groups; - #[cfg(target_os = "macos")] - use crate::compute::direct_prepare::{ - prepare_classic_sub_band_groups, prepare_ht_sub_band_groups, - prepare_ungrouped_ht_sub_band_buffers, prepared_ht_buffer, - }; - - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::direct_roi::crop_prepared_direct_grayscale_plan_to_output_region; - #[cfg(all(target_os = "macos", test))] - use crate::compute::direct_roi::retain_ht_jobs_for_required_region; - #[cfg(target_os = "macos")] - use crate::compute::direct_roi::{ - idwt_input_windows_from_slices, prepared_idwt_output_len, prepared_idwt_params, - repeated_idwt_params, BandRequiredRegion, PreparedIdwtInputStrides, - }; - - #[cfg(all(target_os = "macos", test))] - use crate::compute::direct_grayscale_execute::execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test; - #[cfg(target_os = "macos")] - use crate::compute::direct_grayscale_execute::{ - checked_coefficient_len, encode_prepared_direct_component_plane_in_command_buffer, - upload_cpu_decoded_coefficients, DirectComponentPlaneRequest, - }; - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::direct_grayscale_execute::{ - execute_hybrid_cpu_tier1_direct_color_plan, - execute_hybrid_cpu_tier1_direct_color_plan_batch, - execute_hybrid_cpu_tier1_direct_color_plan_with_device, - execute_prepared_direct_color_plan, execute_prepared_direct_color_plan_batch, - execute_prepared_direct_color_plan_with_device, execute_prepared_direct_grayscale_plan, - execute_prepared_direct_grayscale_plan_batch, - execute_prepared_direct_grayscale_plan_with_device, - execute_repeated_prepared_direct_grayscale_plan, - }; - - #[cfg(target_os = "macos")] - use crate::compute::forward_transform::{ - active_forward_dwt53_buffers, dispatch_forward_dwt53_batched_pass, - dispatch_forward_dwt53_pass, - }; - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::forward_transform::{ - encode_deinterleave_to_f32, encode_forward_dwt53, encode_forward_dwt97, - }; - - #[cfg(all(target_os = "macos", test))] - use crate::compute::resident_tier1::dispatch_classic_tier1_split_token_emit_for_cpu_pack; - #[cfg(target_os = "macos")] - use crate::compute::resident_tier1::{ - dispatch_classic_tier1_arithmetic_pack_profile, - dispatch_classic_tier1_density_profile, dispatch_classic_tier1_pass_plan_profile, - dispatch_classic_tier1_raw_pack_profile, - dispatch_classic_tier1_split_token_emit_for_gpu_pack, - dispatch_classic_tier1_split_token_emit_profile, - dispatch_classic_tier1_split_token_pack_from_gpu_tokens, - dispatch_classic_tier1_symbol_plan_profile, - dispatch_classic_tier1_token_emit_for_gpu_pack, - dispatch_classic_tier1_token_emit_profile, - dispatch_classic_tier1_token_pack_from_gpu_tokens, - schedule_classic_tier1_gpu_token_pack_readback, schedule_resident_tier1_status_readback, - J2kBatchedPacketPayloadCopyDispatch, ResidentTier1StatusReadbackRequest, - }; - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::resident_tier1::{ - wait_resident_lossless_codestream, wait_resident_lossless_codestream_batch, - J2kLosslessCodestreamAssemblyJob, J2kLosslessCodestreamBlockCodingMode, - J2kLosslessDeviceBatchPrepareItem, - J2kLosslessDeviceCodeBlock, J2kLosslessDevicePrepareJob, - J2kPendingResidentLosslessCodestreamBatch, J2kPreparedLosslessDeviceCodeBlocks, - J2kResidentLosslessHtCodeBlocks, J2kResidentLosslessTier1CodeBlocks, - J2kResidentPacketizationEncodeJob, J2kResidentPacketizationResolution, - J2kResidentPacketizationSubband, ResidentLosslessTier1Metal, - }; - - #[cfg(target_os = "macos")] - use crate::compute::lossless_prepare::dispatch_batched_packet_payload_copy; - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::lossless_prepare::{ - encode_forward_ict, encode_forward_rct, encode_quantize_subband, - prepare_lossless_device_code_blocks, prepare_lossless_device_code_blocks_batch, - }; - - #[cfg(all(target_os = "macos", test))] - use crate::compute::decode_dispatch::{ - classic_batch_uses_plain_fast_path, classic_repeated_uses_plain_fast_path, - repeated_gray_store_is_contiguous_full_surface, - }; - #[cfg(all(target_os = "macos", test))] - pub(crate) use crate::compute::decode_dispatch::decode_irreversible97_staged_single_decomposition_idwt; - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::decode_dispatch::{ - decode_inverse_mct, decode_irreversible97_single_decomposition_idwt, - decode_reversible53_single_decomposition_idwt, decode_store_component_and_capture, - }; - #[cfg(target_os = "macos")] - use crate::compute::decode_dispatch::{ - dispatch_classic_cleanup_batched, dispatch_inverse_mct_buffers_in_command_buffer, - dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets, - dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets, - dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets, - dispatch_reversible53_single_decomposition_buffers_in_command_buffer_with_offsets, - dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets, - dispatch_store_component_buffer_in_command_buffer_with_offsets, - dispatch_store_component_buffer_in_encoder_with_offsets, - dispatch_store_component_repeated_in_command_buffer, dispatch_zero_u32_buffer_in_encoder, - encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer, - encode_distinct_classic_sub_bands_to_buffer_in_command_buffer, - encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer, - encode_distinct_ht_sub_bands_to_buffer_in_command_buffer, - encode_gray_store_to_surface_in_encoder, - encode_prepared_classic_sub_band_group_to_buffer_in_encoder, - encode_prepared_classic_sub_band_to_buffer_in_encoder, - encode_prepared_ht_sub_band_group_to_buffer_in_encoder, - encode_prepared_ht_sub_band_to_buffer_in_encoder, - encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer, - encode_repeated_classic_sub_band_to_buffer_in_command_buffer, - encode_repeated_gray_store_to_surfaces_in_command_buffer, - encode_repeated_ht_sub_band_group_to_buffer_in_command_buffer, - encode_repeated_ht_sub_band_to_buffer_in_command_buffer, ht_batch_output_word_count, - ht_output_word_count, required_ht_output_len, IdwtSubBandBuffers, RepeatedIdwtDispatch, - SingleIdwtDispatch, - }; - - #[cfg(target_os = "macos")] - use crate::compute::tier1_encode::{ - classic_encode_sub_band_code, encode_status_error, packet_encode_status_error, - }; - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::tier1_encode::{ - encode_classic_tier1_code_block, encode_classic_tier1_code_blocks, - encode_classic_tier1_prepared_device_code_blocks_resident, encode_ht_cleanup_code_block, - encode_ht_cleanup_code_blocks, encode_ht_prepared_device_code_blocks_resident, - read_resident_ht_tier1_code_blocks_for_cpu_packetization, - }; - #[cfg(all(target_os = "macos", test))] - pub(crate) use crate::compute::tier1_encode::{ - encode_classic_tier1_code_blocks_via_gpu_token_pack_for_test, - encode_classic_tier1_code_blocks_via_ordered_tokens_cpu_pack_for_test, - encode_classic_tier1_code_blocks_via_split_mq_byte_raw_tokens_gpu_pack_for_test, - encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_cpu_pack_for_test, - encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_gpu_pack_for_test, - }; - - #[cfg(target_os = "macos")] - use crate::compute::resident_codestream::{ - dispatch_ht_cleanup, dispatch_ht_cleanup_batched, - dispatch_ht_cleanup_batched_in_command_buffer, dispatch_ht_cleanup_batched_in_encoder, - dispatch_ht_cleanup_repeated_batched_in_command_buffer, HtRepeatedCleanupDispatch, - }; - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::resident_codestream::{ - encode_lossless_codestream_buffer_from_resident_tier1, encode_tier2_packetization, - submit_lossless_codestream_buffers_from_prepared_classic_batch, - submit_lossless_codestream_buffers_from_prepared_ht_batch, - }; - - #[cfg(target_os = "macos")] - pub(crate) use crate::compute::decode_cleanup::{ - decode_classic_cleanup_code_block, decode_classic_cleanup_sub_band, - decode_ht_cleanup_code_block, decode_ht_cleanup_sub_band, - }; - }; -} - -pub(super) use wire_compute_symbols; diff --git a/crates/j2k-metal/src/compute/test_counters.rs b/crates/j2k-metal/src/compute/test_counters.rs index 72ff536e..09e6347d 100644 --- a/crates/j2k-metal/src/compute/test_counters.rs +++ b/crates/j2k-metal/src/compute/test_counters.rs @@ -47,15 +47,101 @@ test_atomic_counter!( reset_flattened_hybrid_cpu_decode_batches_for_test, flattened_hybrid_cpu_decode_batches_for_test ); - std::thread_local! { + static STACKED_COMPONENT_BATCHES: Cell = const { Cell::new(0) }; static RESIDENT_GPU_TIMESTAMP_QUERIES: Cell = const { Cell::new(0) }; static RESIDENT_CODESTREAM_COMMAND_BUFFER_WAITS: Cell = const { Cell::new(0) }; static DIRECT_TIER1_INPUT_BUFFER_PREPARES: Cell = const { Cell::new(0) }; + static DIRECT_TIER1_INPUT_BUFFER_RUNTIME: Cell = const { Cell::new(0) }; static HYBRID_CPU_DECODE_INPUTS_FOR_THREAD: Cell = const { Cell::new(0) }; static LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES: Cell = const { Cell::new(0) }; static CLASSIC_GPU_TOKEN_PACK_DISPATCHES: Cell = const { Cell::new(0) }; static CLASSIC_SPLIT_MQ_BYTE_GPU_TOKEN_PACK_DISPATCHES: Cell = const { Cell::new(0) }; + static HT_IMMUTABLE_PAYLOAD_UPLOADS: Cell = const { Cell::new(0) }; + static HT_IMMUTABLE_JOB_UPLOADS: Cell = const { Cell::new(0) }; + static METAL_COMMAND_BUFFERS: Cell = const { Cell::new(0) }; + static METAL_COMPUTE_ENCODERS: Cell = const { Cell::new(0) }; + static DIRECT_DESTINATION_EVENT_ALLOCATIONS: Cell = const { Cell::new(0) }; + static DIRECT_DESTINATION_EVENT_SIGNALS: Cell = const { Cell::new(0) }; + static DIRECT_DESTINATION_EVENT_WAITS: Cell = const { Cell::new(0) }; +} + +pub(crate) fn reset_direct_destination_event_bridge_for_test() { + DIRECT_DESTINATION_EVENT_ALLOCATIONS.with(|count| count.set(0)); + DIRECT_DESTINATION_EVENT_SIGNALS.with(|count| count.set(0)); + DIRECT_DESTINATION_EVENT_WAITS.with(|count| count.set(0)); +} + +pub(crate) fn direct_destination_event_bridge_for_test() -> (usize, usize, usize) { + ( + DIRECT_DESTINATION_EVENT_ALLOCATIONS.with(Cell::get), + DIRECT_DESTINATION_EVENT_SIGNALS.with(Cell::get), + DIRECT_DESTINATION_EVENT_WAITS.with(Cell::get), + ) +} + +pub(crate) fn record_direct_destination_event_allocation() { + DIRECT_DESTINATION_EVENT_ALLOCATIONS.with(|count| count.set(count.get().saturating_add(1))); +} + +pub(crate) fn record_direct_destination_event_signal() { + DIRECT_DESTINATION_EVENT_SIGNALS.with(|count| count.set(count.get().saturating_add(1))); +} + +pub(crate) fn record_direct_destination_event_wait() { + DIRECT_DESTINATION_EVENT_WAITS.with(|count| count.set(count.get().saturating_add(1))); +} + +pub(crate) fn reset_metal_command_buffers_for_test() { + METAL_COMMAND_BUFFERS.with(|count| count.set(0)); +} + +pub(crate) fn metal_command_buffers_for_test() -> usize { + METAL_COMMAND_BUFFERS.with(Cell::get) +} + +pub(crate) fn record_metal_command_buffer() { + METAL_COMMAND_BUFFERS.with(|count| count.set(count.get().saturating_add(1))); +} + +pub(crate) fn reset_metal_compute_encoders_for_test() { + METAL_COMPUTE_ENCODERS.with(|count| count.set(0)); +} + +pub(crate) fn metal_compute_encoders_for_test() -> usize { + METAL_COMPUTE_ENCODERS.with(Cell::get) +} + +pub(crate) fn record_metal_compute_encoder() { + METAL_COMPUTE_ENCODERS.with(|count| count.set(count.get().saturating_add(1))); +} + +pub(crate) fn reset_ht_immutable_payload_uploads_for_test() { + HT_IMMUTABLE_PAYLOAD_UPLOADS.with(|uploads| uploads.set(0)); +} + +pub(crate) fn ht_immutable_payload_uploads_for_test() -> usize { + HT_IMMUTABLE_PAYLOAD_UPLOADS.with(Cell::get) +} + +pub(crate) fn reset_ht_immutable_job_uploads_for_test() { + HT_IMMUTABLE_JOB_UPLOADS.with(|uploads| uploads.set(0)); +} + +pub(crate) fn ht_immutable_job_uploads_for_test() -> usize { + HT_IMMUTABLE_JOB_UPLOADS.with(Cell::get) +} + +pub(crate) fn reset_stacked_component_batches_for_test() { + STACKED_COMPONENT_BATCHES.with(|counter| counter.set(0)); +} + +pub(crate) fn stacked_component_batches_for_test() -> usize { + STACKED_COMPONENT_BATCHES.with(Cell::get) +} + +pub(crate) fn record_stacked_component_batch() { + STACKED_COMPONENT_BATCHES.with(|counter| counter.set(counter.get().saturating_add(1))); } pub(crate) fn reset_resident_gpu_timestamp_queries_for_test() { @@ -84,14 +170,22 @@ pub(crate) fn record_resident_codestream_command_buffer_wait() { pub(crate) fn reset_direct_tier1_input_buffer_prepares_for_test() { DIRECT_TIER1_INPUT_BUFFER_PREPARES.with(|counter| counter.set(0)); + DIRECT_TIER1_INPUT_BUFFER_RUNTIME.with(|identity| identity.set(0)); } pub(crate) fn direct_tier1_input_buffer_prepares_for_test() -> usize { DIRECT_TIER1_INPUT_BUFFER_PREPARES.with(Cell::get) } -pub(crate) fn record_direct_tier1_input_buffer_prepare() { +pub(crate) fn direct_tier1_input_buffer_runtime_for_test() -> usize { + DIRECT_TIER1_INPUT_BUFFER_RUNTIME.with(Cell::get) +} + +pub(crate) fn record_direct_tier1_input_buffer_prepare(runtime: &super::MetalRuntime) { DIRECT_TIER1_INPUT_BUFFER_PREPARES.with(|counter| counter.set(counter.get() + 1)); + DIRECT_TIER1_INPUT_BUFFER_RUNTIME.with(|identity| { + identity.set(std::ptr::from_ref(runtime).addr()); + }); } pub(crate) fn reset_thread_hybrid_cpu_decode_inputs_for_test() { @@ -166,3 +260,11 @@ pub(crate) fn record_hybrid_cpu_decode_inputs(count: usize) { pub(crate) fn record_flattened_hybrid_cpu_decode_batch() { FLATTENED_HYBRID_CPU_DECODE_BATCHES.fetch_add(1, Ordering::Relaxed); } + +pub(crate) fn record_ht_immutable_payload_upload() { + HT_IMMUTABLE_PAYLOAD_UPLOADS.with(|uploads| uploads.set(uploads.get().saturating_add(1))); +} + +pub(crate) fn record_ht_immutable_job_upload() { + HT_IMMUTABLE_JOB_UPLOADS.with(|uploads| uploads.set(uploads.get().saturating_add(1))); +} diff --git a/crates/j2k-metal/src/compute/tests.rs b/crates/j2k-metal/src/compute/tests.rs index 3f439392..f948cf89 100644 --- a/crates/j2k-metal/src/compute/tests.rs +++ b/crates/j2k-metal/src/compute/tests.rs @@ -1,1714 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::{ - checked_metal_surface_len, classic_batch_uses_plain_fast_path, - classic_repeated_uses_plain_fast_path, crop_prepared_direct_grayscale_plan_to_output_region, - decode_prepared_classic_sub_band_on_cpu, decode_scaled_to_surface, - direct_tier1_input_buffer_prepares_for_test, - execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test, - execute_hybrid_cpu_tier1_direct_color_plan_batch, flattened_hybrid_cpu_decode_batches_for_test, - hybrid_cpu_decode_inputs_for_test, hybrid_cpu_decode_worker_count, - hybrid_cpu_decode_worker_inits_for_test, hybrid_repeated_output_blits_for_test, - hybrid_stacked_component_batches_for_test, j2k_pack_kernel_name_for, j2k_pack_scale_arrays, - output_shape_for, prepare_direct_color_plan, prepare_direct_color_plan_for_cpu_upload, - prepare_direct_grayscale_plan, prepare_sub_band_groups, - prepared_direct_color_tier1_input_count, prepared_direct_grayscale_plan_compute_encoder_count, - prepared_idwt_output_len, prepared_repeated_direct_ht_cleanup_dispatch_count, - repeated_gray_store_is_contiguous_full_surface, - reset_direct_tier1_input_buffer_prepares_for_test, - reset_flattened_hybrid_cpu_decode_batches_for_test, reset_hybrid_cpu_decode_inputs_for_test, - reset_hybrid_cpu_decode_worker_inits_for_test, reset_hybrid_repeated_output_blits_for_test, - reset_hybrid_stacked_component_batches_for_test, reset_shared_buffer_pool_misses_for_test, - reset_thread_hybrid_cpu_decode_inputs_for_test, retain_ht_jobs_for_required_region, - runtime_initialization_error, shared_buffer_pool_misses_for_test, - should_flatten_hybrid_cpu_tier1_color_batch, supports_stacked_direct_component_plane_batch, - thread_hybrid_cpu_decode_inputs_for_test, with_runtime_for_device, DirectTier1Mode, - J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob, J2kRepeatedGrayStoreParams, - MetalRuntime, MetalSupportError, PreparedClassicSubBand, PreparedDirectColorPlan, - PreparedDirectGrayscaleStep, PreparedHtSubBand, -}; -use j2k_core::PixelFormat; -use j2k_native::{ - decode_j2k_sub_band_scalar, encode, encode_htj2k, ColorSpace as NativeColorSpace, - DecodeSettings, DecoderContext, EncodeOptions, Image, J2kCodeBlockBatchJob, - J2kCodeBlockDecodeJob, J2kDirectGrayscaleStep as NativeDirectGrayscaleStep, - J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, J2kSubBandDecodeJob, J2kWaveletTransform, -}; -use metal::foreign_types::ForeignType; -use metal::Device; -use std::sync::{Arc, Mutex}; - -static HYBRID_COUNTER_TEST_LOCK: Mutex<()> = Mutex::new(()); - -fn should_run_metal_runtime() -> bool { - j2k_test_support::metal_runtime_gate(module_path!()) -} - -#[test] -fn rgb16_with_alpha_is_rejected() { - if !should_run_metal_runtime() { - return; - } - - let runtime = MetalRuntime::new().expect("Metal runtime"); - let result = output_shape_for( - &NativeColorSpace::RGB, - true, - 4, - PixelFormat::Rgb16, - &runtime, - ); - assert!(result.is_err(), "RGBA input must not silently map to Rgb16"); -} - -#[test] -fn runtime_initialization_error_classifies_null_queue_as_unavailable() { - assert!(matches!( - runtime_initialization_error(&MetalSupportError::CommandQueueUnavailable), - crate::Error::MetalUnavailable - )); -} - -#[test] -fn classic_encode_output_capacity_keeps_conservative_default() { - let capacity = - super::classic_encode_output_capacity(64, 64, 11).expect("classic output capacity"); - - assert_eq!(capacity, 64 * 64 * 11 * 8 + 4097); -} - -#[test] -fn classic_encode_segment_capacity_uses_coding_style_bound() { - assert_eq!(super::classic_encode_segment_capacity(0, 16), 1); - assert_eq!( - super::classic_encode_segment_capacity( - super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, - 9, - ), - 11 - ); - assert_eq!( - super::classic_encode_segment_capacity( - super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, - 16, - ), - 25 - ); - assert_eq!( - super::classic_encode_segment_capacity( - super::J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, - 16, - ), - 46 - ); -} - -#[test] -fn checked_metal_surface_len_accepts_valid_surface() { - assert_eq!( - checked_metal_surface_len((13, 7), PixelFormat::Rgb8.bytes_per_pixel(), "test surface") - .unwrap(), - (39, 273) - ); -} - -#[test] -fn checked_metal_surface_len_reports_overflow_as_metal_error() { - let error = checked_metal_surface_len((u32::MAX, 1), usize::MAX, "test surface").unwrap_err(); - - assert!( - matches!(error, crate::Error::MetalKernel { message } if message.contains("surface row byte count")) - ); -} - -#[test] -fn two_d_threads_per_group_clamps_empty_pipeline_limits() { - let threads = j2k_metal_support::two_d_threads_per_group(0, 0); - - assert_eq!((threads.width, threads.height, threads.depth), (1, 1, 1)); -} - -#[test] -fn one_d_threads_per_group_clamps_empty_pipeline_width() { - let threads = j2k_metal_support::one_d_threads_per_group(0); - - assert_eq!((threads.width, threads.height, threads.depth), (1, 1, 1)); -} - -#[test] -fn two_d_threads_per_group_preserves_simd_width_and_derives_height() { - let threads = j2k_metal_support::two_d_threads_per_group(32, 1024); - - assert_eq!((threads.width, threads.height, threads.depth), (32, 32, 1)); -} - -#[test] -fn classic_tier1_pass_class_counts_split_bypass_pass_types() { - let counts = super::classic_tier1_pass_class_counts( - 23, - super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, - ); - - assert_eq!(counts.arithmetic, 14); - assert_eq!(counts.raw, 9); - assert_eq!(counts.cleanup, 8); - assert_eq!(counts.sigprop, 8); - assert_eq!(counts.magref, 7); - assert_eq!(counts.arithmetic_cleanup, 8); - assert_eq!(counts.arithmetic_sigprop, 3); - assert_eq!(counts.arithmetic_magref, 3); - assert_eq!(counts.raw_sigprop, 5); - assert_eq!(counts.raw_magref, 4); -} - -#[test] -fn classic_tier1_pass_class_counts_style0_stays_arithmetic() { - let counts = super::classic_tier1_pass_class_counts(5, 0); - - assert_eq!(counts.arithmetic, 5); - assert_eq!(counts.raw, 0); - assert_eq!(counts.cleanup, 2); - assert_eq!(counts.sigprop, 2); - assert_eq!(counts.magref, 1); - assert_eq!(counts.arithmetic_cleanup, 2); - assert_eq!(counts.arithmetic_sigprop, 2); - assert_eq!(counts.arithmetic_magref, 1); - assert_eq!(counts.raw_sigprop, 0); - assert_eq!(counts.raw_magref, 0); -} - -#[test] -fn classic_tier1_scan_estimates_multiply_passes_by_block_area() { - let pass_counts = super::classic_tier1_pass_class_counts( - 23, - super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, - ); - let mut stats = super::J2kResidentEncodeStageStats::default(); - - super::accumulate_classic_tier1_scan_estimates(&mut stats, pass_counts, 32 * 32); - - assert_eq!(stats.tier1_full_scan_coeff_visit_count_total, 23 * 1024); - assert_eq!( - stats.tier1_arithmetic_scan_coeff_visit_count_total, - 14 * 1024 - ); - assert_eq!(stats.tier1_raw_scan_coeff_visit_count_total, 9 * 1024); - assert_eq!(stats.tier1_cleanup_scan_coeff_visit_count_total, 8 * 1024); - assert_eq!(stats.tier1_sigprop_scan_coeff_visit_count_total, 8 * 1024); - assert_eq!(stats.tier1_magref_scan_coeff_visit_count_total, 7 * 1024); - assert_eq!(stats.max_tier1_full_scan_coeff_visits_per_block, 23 * 1024); -} - -#[test] -fn classic_packet_output_capacity_uses_raw_sample_bound_when_smaller() { - let codestream = super::J2kLosslessCodestreamAssemblyJob { - width: 512, - height: 512, - component_count: 3, - bit_depth: 8, - signed: false, - num_decomposition_levels: 3, - use_mct: true, - guard_bits: 2, - code_block_width_exp: 4, - code_block_height_exp: 4, - progression_order: j2k_native::EncodeProgressionOrder::Lrcp, - write_tlm: false, - block_coding_mode: super::J2kLosslessCodestreamBlockCodingMode::Classic, - }; - let header_capacity = 1024 * 256 + 4096; - let conservative_capacity = 12 * 1024 * 1024; - let packet_descriptor_count = 3; - - let capacity = super::classic_packet_output_capacity( - conservative_capacity, - header_capacity, - packet_descriptor_count, - codestream, - ) - .expect("classic packet capacity"); - - let raw_bytes = 512 * 512 * 3; - let descriptor_slack = packet_descriptor_count * 256; - assert_eq!( - capacity, - raw_bytes + header_capacity + descriptor_slack + 64 * 1024 - ); - - let tiny_tier1_capacity = 4096; - let clamped = super::classic_packet_output_capacity( - tiny_tier1_capacity, - header_capacity, - packet_descriptor_count, - codestream, - ) - .expect("classic packet capacity"); - let conservative_packet_capacity = - tiny_tier1_capacity + header_capacity * packet_descriptor_count + 1024; - assert_eq!(clamped, conservative_packet_capacity); -} - -#[test] -fn ht_encode_output_capacity_scales_with_code_block_area() { - let max_block = super::ht_encode_output_capacity(128, 128).expect("max HT output capacity"); - assert_eq!(max_block, super::J2K_HT_ENCODE_BASE_OUTPUT_SIZE); - - let smaller_block = - super::ht_encode_output_capacity(32, 32).expect("scaled HT output capacity"); - assert!(smaller_block < max_block / 2); - assert!(smaller_block >= 8192); -} - -#[test] -fn classic_encode_pipeline_kind_prefers_style0_32_for_resident_jobs() { - let jobs = [super::J2kClassicEncodeBatchJob { - width: 32, - height: 32, - style_flags: 0, - ..super::J2kClassicEncodeBatchJob::default() - }]; - - assert_eq!( - super::classic_encode_code_blocks_pipeline_kind(&jobs), - super::J2kClassicEncodePipelineKind::Style0_32 - ); -} - -#[test] -fn classic_encode_pipeline_kind_prefers_bypass_32_for_resident_jobs() { - let jobs = [super::J2kClassicEncodeBatchJob { - width: 32, - height: 32, - style_flags: super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, - total_bitplanes: 31, - ..super::J2kClassicEncodeBatchJob::default() - }]; - - assert_eq!( - super::classic_encode_code_blocks_pipeline_kind(&jobs), - super::J2kClassicEncodePipelineKind::Bypass32 - ); -} - -#[test] -fn classic_encode_pipeline_kind_prefers_bypass_u16_32_for_low_bitplane_resident_jobs() { - let jobs = [super::J2kClassicEncodeBatchJob { - width: 32, - height: 32, - style_flags: super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, - total_bitplanes: 16, - ..super::J2kClassicEncodeBatchJob::default() - }]; - - assert_eq!( - super::classic_encode_code_blocks_pipeline_kind(&jobs), - super::J2kClassicEncodePipelineKind::BypassU16_32 - ); -} - -#[test] -fn with_runtime_for_device_scopes_runtime_to_requested_device() { - if !should_run_metal_runtime() { - return; - } - - let Some(device) = Device::system_default() else { - j2k_test_support::metal_device_unavailable_is_skip(module_path!()); - return; - }; - - let runtime_device = - with_runtime_for_device(&device, |runtime| Ok(runtime.device.as_ptr() as usize)) - .expect("Metal runtime"); - - assert_eq!(runtime_device, device.as_ptr() as usize); -} - -#[test] -fn runtime_reuses_recycled_shared_buffers() -> Result<(), crate::Error> { - if !should_run_metal_runtime() { - return Ok(()); - } - - let Some(device) = Device::system_default() else { - j2k_test_support::metal_device_unavailable_is_skip(module_path!()); - return Ok(()); - }; - let runtime = MetalRuntime::new_with_device(&device).expect("Metal runtime"); - - reset_shared_buffer_pool_misses_for_test(); - let first = runtime.take_shared_buffer(64)?; - runtime.recycle_shared_buffer(first)?; - let _second = runtime.take_shared_buffer(64)?; - - assert_eq!( - shared_buffer_pool_misses_for_test(), - 1, - "recycled shared metadata buffers should be reused instead of allocating again" - ); - Ok(()) -} - -#[test] -fn j2k_pack_selects_specialized_kernels_for_wsi_formats() { - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::Gray, false, 1, PixelFormat::Gray8), - Some("j2k_pack_gray8") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgb8), - Some("j2k_pack_rgb8") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgba8), - Some("j2k_pack_rgb_opaque_rgba8") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, true, 4, PixelFormat::Rgba8), - Some("j2k_pack_rgba8") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::Gray, false, 1, PixelFormat::Gray16), - Some("j2k_pack_gray16") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgb16), - Some("j2k_pack_rgb16") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, true, 4, PixelFormat::Rgb16), - None, - "RGBA input must not silently drop alpha when packing RGB16" - ); -} - -#[test] -fn j2k_pack_precomputes_scale_factors_on_cpu() { - let (max_values, u8_scales, u16_scales) = j2k_pack_scale_arrays([8, 12, 16, 0]); - - assert_f32_near(max_values[0], 255.0); - assert_f32_near(max_values[1], 4095.0); - assert_f32_near(max_values[2], 65_535.0); - assert_f32_near(max_values[3], 1.0); - assert_f32_near(u8_scales[0], 1.0); - assert_f32_near(u8_scales[1], 255.0 / 4095.0); - assert_f32_near(u16_scales[0], 257.0); - assert_f32_near(u16_scales[1], 1.0); - assert_f32_near(u16_scales[2], 1.0); - assert_f32_near(u16_scales[3], 65_535.0); -} - -fn assert_f32_near(actual: f32, expected: f32) { - assert!( - (actual - expected).abs() <= f32::EPSILON, - "expected {actual} to be within f32 epsilon of {expected}" - ); -} - -#[test] -fn scaled_htj2k_decode_runs_through_metal_compute_path() { - if !should_run_metal_runtime() { - return; - } - - let pixels: Vec = (0..16).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode ht gray8"); - - let image = Image::new( - &bytes, - &DecodeSettings { - target_resolution: Some((2, 2)), - ..DecodeSettings::default() - }, - ) - .expect("image"); - let host = image.decode().expect("host scaled decode"); - - let surface = decode_scaled_to_surface( - &bytes, - (4, 4), - PixelFormat::Gray8, - j2k_core::Downscale::Half, - ) - .expect("metal scaled decode"); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -fn test_ht_job(output_x: u32, output_y: u32, width: u32, height: u32) -> J2kHtCleanupBatchJob { - J2kHtCleanupBatchJob { - coded_offset: output_y - .checked_mul(32) - .and_then(|base| base.checked_add(output_x)) - .expect("test coded offset"), - width, - height, - coded_len: 1, - cleanup_length: 1, - refinement_length: 0, - missing_msbs: 0, - num_bitplanes: 8, - roi_shift: 0, - number_of_coding_passes: 1, - output_stride: 8, - output_offset: output_y - .checked_mul(8) - .and_then(|base| base.checked_add(output_x)) - .expect("test output offset"), - dequantization_step: 1.0, - stripe_causal: 0, - } -} - -#[expect( - clippy::cast_possible_truncation, - reason = "test helper receives small synthetic band identifiers" -)] -fn test_ht_sub_band(band_id: u32) -> PreparedHtSubBand { - PreparedHtSubBand { - band_id, - width: 8, - height: 8, - coded_data: vec![band_id as u8], - coded_buffer: None, - jobs: vec![test_ht_job(0, 0, 2, 2)], - jobs_buffer: None, - } -} - -fn separator_store_step() -> PreparedDirectGrayscaleStep { - PreparedDirectGrayscaleStep::Store(j2k_native::J2kDirectStoreStep { - input_band_id: 99, - input_rect: j2k_native::J2kRect { - x0: 0, - y0: 0, - x1: 1, - y1: 1, - }, - source_x: 0, - source_y: 0, - copy_width: 1, - copy_height: 1, - output_width: 1, - output_height: 1, - output_x: 0, - output_y: 0, - addend: 0.0, - }) -} - -#[test] -fn direct_sub_band_grouping_groups_adjacent_ht_runs_without_runtime() { - let steps = vec![ - PreparedDirectGrayscaleStep::HtSubBand(test_ht_sub_band(1)), - PreparedDirectGrayscaleStep::HtSubBand(test_ht_sub_band(2)), - separator_store_step(), - PreparedDirectGrayscaleStep::HtSubBand(test_ht_sub_band(3)), - PreparedDirectGrayscaleStep::HtSubBand(test_ht_sub_band(4)), - ]; - - let groups = prepare_sub_band_groups( - &steps, - DirectTier1Mode::CpuUpload, - |step| match step { - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => Some(sub_band), - _ => None, - }, - |start, end, sub_bands, _| { - Ok(( - start, - end, - sub_bands - .iter() - .map(|sub_band| sub_band.band_id) - .collect::>(), - )) - }, - ) - .expect("group adjacent HT sub-bands"); - - assert_eq!(groups, vec![(0, 2, vec![1, 2]), (3, 5, vec![3, 4])]); -} - -#[test] -fn direct_roi_prunes_ht_jobs_without_runtime() { - let mut jobs = vec![ - test_ht_job(0, 0, 2, 2), - test_ht_job(6, 0, 2, 2), - test_ht_job(2, 2, 3, 3), - ]; - let required = - j2k_native::J2kRequiredBandRegion::new(0, 0, 4, 4).expect("required test region"); - - retain_ht_jobs_for_required_region(&mut jobs, Some(required)); - - let retained_offsets = jobs.iter().map(|job| job.output_offset).collect::>(); - assert_eq!(retained_offsets, vec![0, 18]); - - retain_ht_jobs_for_required_region(&mut jobs, None); - assert!(jobs.is_empty()); -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn prepared_ht_direct_plan_groups_cleanup_subbands_before_idwt() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let ht_subband_steps = plan - .steps - .iter() - .filter(|step| matches!(step, j2k_native::J2kDirectGrayscaleStep::HtSubBand(_))) - .count(); - assert!( - ht_subband_steps > 1, - "fixture must exercise multiple HT sub-band cleanup steps" - ); - - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - assert_eq!( - prepared.ht_groups.len(), - 1, - "single-tile HTJ2K direct decode should group adjacent HT sub-bands into one cleanup dispatch" - ); - assert_eq!(prepared.ht_groups[0].members.len(), ht_subband_steps); - assert!(matches!( - prepared.steps[prepared.ht_groups[0].start_step], - PreparedDirectGrayscaleStep::HtSubBand(_) - )); -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn grouped_ht_direct_plan_uses_one_group_coded_arena() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - - reset_direct_tier1_input_buffer_prepares_for_test(); - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - assert_eq!( - prepared.ht_groups.len(), - 1, - "fixture must exercise one grouped HT dispatch" - ); - let group = &prepared.ht_groups[0]; - assert!(!group.coded_arena.data.is_empty()); - assert_eq!( - direct_tier1_input_buffer_prepares_for_test(), - 2, - "grouped HT dispatch should prepare one coded arena buffer and one job buffer" - ); - - for step in &prepared.steps[group.start_step..group.end_step] { - let PreparedDirectGrayscaleStep::HtSubBand(sub_band) = step else { - panic!("HT group should only span HT sub-band steps"); - }; - assert!(sub_band.coded_buffer.is_none()); - assert!(sub_band.jobs_buffer.is_none()); - } -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn prepared_classic_sub_band_decodes_on_cpu_for_hybrid_upload() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode classic gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - let native_sub_band = first_native_classic_sub_band(&plan); - let prepared_sub_band = first_prepared_classic_sub_band(&prepared); - - let expected = decode_native_classic_sub_band(native_sub_band); - let actual = - decode_prepared_classic_sub_band_on_cpu(prepared_sub_band).expect("prepared CPU decode"); - - assert_eq!(actual, expected); -} - -#[test] -fn cpu_upload_color_prepare_skips_tier1_metal_input_buffers() { - if !should_run_metal_runtime() { - return; - } - - if Device::system_default().is_none() { - j2k_test_support::metal_device_unavailable_is_skip(module_path!()); - return; - } - - let pixels = j2k_test_support::gradient_u8(32, 32, 3); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 2, - ..EncodeOptions::default() - }; - let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_color_plan_with_context(&mut context) - .expect("direct color plan"); - - reset_direct_tier1_input_buffer_prepares_for_test(); - let metal_prepared = prepare_direct_color_plan(&plan).expect("Metal prepared color plan"); - assert_eq!(metal_prepared.component_plans.len(), 3); - assert!( - direct_tier1_input_buffer_prepares_for_test() > 0, - "normal Metal preparation should build Tier-1 input buffers" - ); - - reset_direct_tier1_input_buffer_prepares_for_test(); - let cpu_upload_prepared = - prepare_direct_color_plan_for_cpu_upload(&plan).expect("CPUUpload prepared color plan"); - assert_eq!(cpu_upload_prepared.component_plans.len(), 3); - assert_eq!( - direct_tier1_input_buffer_prepares_for_test(), - 0, - "CPUUpload preparation should keep coded Tier-1 payloads on CPU and skip Metal input buffers" - ); -} - -fn first_native_classic_sub_band( - plan: &j2k_native::J2kDirectGrayscalePlan, -) -> &J2kOwnedSubBandPlan { - plan.steps - .iter() - .find_map(|step| match step { - NativeDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), - _ => None, - }) - .expect("classic sub-band step") -} - -fn first_prepared_classic_sub_band( - plan: &super::PreparedDirectGrayscalePlan, -) -> &PreparedClassicSubBand { - plan.steps - .iter() - .find_map(|step| match step { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), - _ => None, - }) - .expect("prepared classic sub-band step") -} - -fn cached_direct_color_tier1_input_count(plan: &PreparedDirectColorPlan) -> usize { - plan.component_plans - .iter() - .map(cached_direct_component_tier1_input_count) - .sum() -} - -fn cached_direct_component_tier1_input_count(plan: &super::PreparedDirectGrayscalePlan) -> usize { - let mut count = 0; - let mut step_idx = 0; - while step_idx < plan.steps.len() { - if let Some(group) = plan.classic_group_starting_at(step_idx) { - if has_cached_cpu_tier1_coefficients(plan, step_idx, group.total_coefficients) { - count += 1; - } - step_idx = group.end_step; - continue; - } - if let Some(group) = plan.ht_group_starting_at(step_idx) { - if has_cached_cpu_tier1_coefficients(plan, step_idx, group.total_coefficients) { - count += 1; - } - step_idx = group.end_step; - continue; - } - match &plan.steps[step_idx] { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - let output_len = sub_band.width as usize * sub_band.height as usize; - if has_cached_cpu_tier1_coefficients(plan, step_idx, output_len) { - count += 1; - } - } - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - let output_len = sub_band.width as usize * sub_band.height as usize; - if has_cached_cpu_tier1_coefficients(plan, step_idx, output_len) { - count += 1; - } - } - PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => {} - } - step_idx += 1; - } - count -} - -fn has_cached_cpu_tier1_coefficients( - plan: &super::PreparedDirectGrayscalePlan, - step_idx: usize, - output_len: usize, -) -> bool { - let mut budget = - crate::batch_allocation::BatchMetadataBudget::new("J2K Metal test CPU Tier-1 cache lookup"); - plan.cached_cpu_tier1_coefficients(&mut budget, step_idx, output_len) - .expect("CPU Tier-1 cache lookup") - .is_some() -} - -fn decode_native_classic_sub_band(plan: &J2kOwnedSubBandPlan) -> Vec { - let mut output = vec![0.0_f32; plan.width as usize * plan.height as usize]; - let jobs = plan - .jobs - .iter() - .map(|job| J2kCodeBlockBatchJob { - output_x: job.output_x, - output_y: job.output_y, - code_block: native_classic_job(job), - }) - .collect::>(); - decode_j2k_sub_band_scalar( - J2kSubBandDecodeJob { - width: plan.width, - height: plan.height, - jobs: &jobs, - }, - &mut output, - ) - .expect("native scalar classic sub-band decode"); - output -} - -fn native_classic_job(job: &J2kOwnedCodeBlockBatchJob) -> J2kCodeBlockDecodeJob<'_> { - J2kCodeBlockDecodeJob { - data: &job.data, - segments: &job.segments, - width: job.width, - height: job.height, - output_stride: job.output_stride, - missing_bit_planes: job.missing_bit_planes, - number_of_coding_passes: job.number_of_coding_passes, - total_bitplanes: job.total_bitplanes, - roi_shift: job.roi_shift, - sub_band_type: job.sub_band_type, - style: job.style, - strict: job.strict, - dequantization_step: job.dequantization_step, - } -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn prepared_ht_direct_plan_encodes_full_decode_in_one_compute_encoder() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - - assert_eq!( - prepared_direct_grayscale_plan_compute_encoder_count(&prepared, PixelFormat::Gray8), - 1, - "prepared single-tile direct decode should keep cleanup, IDWT, and grayscale store in one compute encoder" - ); -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn repeated_prepared_ht_direct_plan_groups_cleanup_subbands_before_idwt() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let ht_subband_steps = plan - .steps - .iter() - .filter(|step| matches!(step, j2k_native::J2kDirectGrayscaleStep::HtSubBand(_))) - .count(); - assert!( - ht_subband_steps > 1, - "fixture must exercise multiple HT sub-band cleanup steps" - ); - - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - assert_eq!( - prepared_repeated_direct_ht_cleanup_dispatch_count(&prepared), - 1, - "repeated HTJ2K WSI tile batches should group adjacent sub-band cleanups like the single-tile path" - ); -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn distinct_prepared_ht_direct_plans_support_stacked_component_batch() { - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes_a = encode_htj2k(&(0..64).collect::>(), 8, 8, 1, 8, false, &options) - .expect("encode first ht gray8"); - let bytes_b = encode_htj2k( - &(0..64).rev().collect::>(), - 8, - 8, - 1, - 8, - false, - &options, - ) - .expect("encode second ht gray8"); - let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); - let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); - let mut context_a = DecoderContext::default(); - let mut context_b = DecoderContext::default(); - let plan_a = image_a - .build_direct_grayscale_plan_with_context(&mut context_a) - .expect("first direct plan"); - let plan_b = image_b - .build_direct_grayscale_plan_with_context(&mut context_b) - .expect("second direct plan"); - let prepared_a = prepare_direct_grayscale_plan(&plan_a).expect("first prepared plan"); - let prepared_b = prepare_direct_grayscale_plan(&plan_b).expect("second prepared plan"); - - assert!( - supports_stacked_direct_component_plane_batch(&[&prepared_a, &prepared_b]), - "distinct same-shape HTJ2K grayscale plans should be eligible for one stacked batch graph" - ); -} - -#[test] -fn hybrid_rgb8_batch_uses_stacked_component_graph() { - if !should_run_metal_runtime() { - return; - } - - let pixels = j2k_test_support::gradient_u8(32, 32, 3); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 2, - ..EncodeOptions::default() - }; - let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_color_plan_with_context(&mut context) - .expect("direct color plan"); - let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); - let _guard = HYBRID_COUNTER_TEST_LOCK - .lock() - .expect("hybrid counter lock"); - reset_hybrid_stacked_component_batches_for_test(); - reset_hybrid_cpu_decode_worker_inits_for_test(); - - let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( - &[prepared.clone(), prepared], - PixelFormat::Rgb8, - ) - .expect("hybrid RGB8 batch"); - - assert_eq!(surfaces.len(), 2); - assert!( - hybrid_stacked_component_batches_for_test() >= 3, - "hybrid RGB batch should stack each component plane instead of encoding each tile/component serially" - ); - assert!( - hybrid_cpu_decode_worker_inits_for_test() > 0, - "hybrid RGB batch should use worker-local CPU decode scratch instead of per-input decode/flatten" - ); -} - -#[test] -fn hybrid_rgb8_repeated_batch_decodes_shared_tier1_inputs_once() { - if !should_run_metal_runtime() { - return; - } - - let pixels = j2k_test_support::gradient_u8(32, 32, 3); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 2, - ..EncodeOptions::default() - }; - let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_color_plan_with_context(&mut context) - .expect("direct color plan"); - let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); - let unique_tier1_inputs = prepared_direct_color_tier1_input_count(&prepared); - assert!( - unique_tier1_inputs > 0, - "fixture should have Tier-1 inputs to decode" - ); - let _guard = HYBRID_COUNTER_TEST_LOCK - .lock() - .expect("hybrid counter lock"); - reset_hybrid_cpu_decode_inputs_for_test(); - reset_thread_hybrid_cpu_decode_inputs_for_test(); - - let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( - &[prepared.clone(), prepared.clone(), prepared], - PixelFormat::Rgb8, - ) - .expect("hybrid repeated RGB8 batch"); - - assert_eq!(surfaces.len(), 3); - assert!( - hybrid_cpu_decode_inputs_for_test() >= unique_tier1_inputs, - "repeated RGB hybrid batches should decode the shared coefficient inputs" - ); -} - -#[test] -fn hybrid_rgb8_reused_plan_caches_cpu_tier1_inputs_across_calls() { - if !should_run_metal_runtime() { - return; - } - - let pixels = j2k_test_support::gradient_u8(32, 32, 3); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 2, - ..EncodeOptions::default() - }; - let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_color_plan_with_context(&mut context) - .expect("direct color plan"); - let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); - let unique_tier1_inputs = prepared_direct_color_tier1_input_count(&prepared); - assert!( - unique_tier1_inputs > 0, - "fixture should have Tier-1 inputs to decode" - ); - let _guard = HYBRID_COUNTER_TEST_LOCK - .lock() - .expect("hybrid counter lock"); - reset_hybrid_cpu_decode_inputs_for_test(); - reset_thread_hybrid_cpu_decode_inputs_for_test(); - - let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( - &[prepared.clone(), prepared.clone()], - PixelFormat::Rgb8, - ) - .expect("first hybrid repeated RGB8 batch"); - assert_eq!(surfaces.len(), 2); - assert_eq!( - cached_direct_color_tier1_input_count(&prepared), - unique_tier1_inputs, - "first RGB hybrid call should cache every decoded CPU Tier-1 input" - ); - assert_eq!( - thread_hybrid_cpu_decode_inputs_for_test(), - unique_tier1_inputs, - "first RGB hybrid call should decode each shared Tier-1 input once" - ); - - let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( - &[prepared.clone(), prepared.clone()], - PixelFormat::Rgb8, - ) - .expect("second hybrid repeated RGB8 batch"); - assert_eq!(surfaces.len(), 2); - assert_eq!( - cached_direct_color_tier1_input_count(&prepared), - unique_tier1_inputs, - "second RGB hybrid call should keep every decoded CPU Tier-1 input cached" - ); - assert_eq!( - thread_hybrid_cpu_decode_inputs_for_test(), - unique_tier1_inputs, - "second RGB hybrid call must reuse cached CPU Tier-1 coefficients without re-decoding" - ); -} - -#[test] -fn hybrid_rgb8_repeated_batch_decodes_once_and_blits_distinct_outputs() { - if !should_run_metal_runtime() { - return; - } - - let pixels = j2k_test_support::gradient_u8(32, 32, 3); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 2, - ..EncodeOptions::default() - }; - let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_color_plan_with_context(&mut context) - .expect("direct color plan"); - let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); - let _guard = HYBRID_COUNTER_TEST_LOCK - .lock() - .expect("hybrid counter lock"); - reset_hybrid_repeated_output_blits_for_test(); - - let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( - &[ - prepared.clone(), - prepared.clone(), - prepared.clone(), - prepared, - ], - PixelFormat::Rgb8, - ) - .expect("hybrid repeated RGB8 batch"); - - assert_eq!(surfaces.len(), 4); - let surface_bytes = surfaces[0].as_bytes().expect("surface byte access").len(); - let offsets = surfaces - .iter() - .map(|surface| { - surface - .metal_buffer_trusted() - .expect("resident Metal surface") - .1 - }) - .collect::>(); - assert_eq!( - offsets, - (0..surfaces.len()) - .map(|index| index * surface_bytes) - .collect::>(), - "repeated outputs must retain distinct Metal buffer offsets" - ); - for surface in &surfaces[1..] { - assert_eq!( - surface.as_bytes().expect("surface byte access"), - surfaces[0].as_bytes().expect("surface byte access"), - "repeated outputs should remain byte-identical" - ); - } - assert_eq!( - hybrid_repeated_output_blits_for_test(), - 2, - "repeated RGB hybrid batches should duplicate packed output surfaces with logarithmic Metal blit ranges" - ); -} - -#[test] -fn hybrid_rgb8_distinct_batch_keeps_tier1_inputs_separate() { - if !should_run_metal_runtime() { - return; - } - - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 2, - ..EncodeOptions::default() - }; - let bytes_a = encode( - &j2k_test_support::gradient_variant_u8(32, 32, 3, 0), - 32, - 32, - 3, - 8, - false, - &options, - ) - .expect("encode first rgb8"); - let bytes_b = encode( - &j2k_test_support::gradient_variant_u8(32, 32, 3, 7), - 32, - 32, - 3, - 8, - false, - &options, - ) - .expect("encode second rgb8"); - let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); - let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); - let mut context_a = DecoderContext::default(); - let mut context_b = DecoderContext::default(); - let plan_a = image_a - .build_direct_color_plan_with_context(&mut context_a) - .expect("first direct color plan"); - let plan_b = image_b - .build_direct_color_plan_with_context(&mut context_b) - .expect("second direct color plan"); - let prepared_a = Arc::new(prepare_direct_color_plan(&plan_a).expect("first prepared")); - let prepared_b = Arc::new(prepare_direct_color_plan(&plan_b).expect("second prepared")); - let expected_inputs = prepared_direct_color_tier1_input_count(&prepared_a) - + prepared_direct_color_tier1_input_count(&prepared_b); - let _guard = HYBRID_COUNTER_TEST_LOCK - .lock() - .expect("hybrid counter lock"); - reset_hybrid_cpu_decode_inputs_for_test(); - - let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( - &[prepared_a, prepared_b], - PixelFormat::Rgb8, - ) - .expect("hybrid distinct RGB8 batch"); - - assert_eq!(surfaces.len(), 2); - assert_ne!( - surfaces[0].as_bytes().expect("surface byte access"), - surfaces[1].as_bytes().expect("surface byte access"), - "distinct RGB inputs must not reuse the first tile's decoded coefficients" - ); - assert_eq!( - thread_hybrid_cpu_decode_inputs_for_test(), - expected_inputs, - "distinct RGB hybrid batches should decode each tile's own Tier-1 inputs" - ); -} - -#[test] -fn hybrid_rgb8_flattened_cpu_tier1_batch_uses_one_decode_queue() { - if !should_run_metal_runtime() { - return; - } - - let pixels_a = j2k_test_support::gradient_variant_u8(32, 32, 3, 0); - let pixels_b = j2k_test_support::gradient_variant_u8(32, 32, 3, 11); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 2, - ..EncodeOptions::default() - }; - let bytes_a = encode(&pixels_a, 32, 32, 3, 8, false, &options).expect("encode first rgb8"); - let bytes_b = encode(&pixels_b, 32, 32, 3, 8, false, &options).expect("encode second rgb8"); - let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); - let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); - let mut context_a = DecoderContext::default(); - let mut context_b = DecoderContext::default(); - let plan_a = image_a - .build_direct_color_plan_with_context(&mut context_a) - .expect("first direct color plan"); - let plan_b = image_b - .build_direct_color_plan_with_context(&mut context_b) - .expect("second direct color plan"); - let prepared_a = Arc::new(prepare_direct_color_plan(&plan_a).expect("first prepared")); - let prepared_b = Arc::new(prepare_direct_color_plan(&plan_b).expect("second prepared")); - let expected_inputs = prepared_direct_color_tier1_input_count(&prepared_a) - + prepared_direct_color_tier1_input_count(&prepared_b); - let _guard = HYBRID_COUNTER_TEST_LOCK - .lock() - .expect("hybrid counter lock"); - reset_hybrid_cpu_decode_inputs_for_test(); - reset_flattened_hybrid_cpu_decode_batches_for_test(); - - let surfaces = execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test( - &[prepared_a, prepared_b], - PixelFormat::Rgb8, - ) - .expect("flattened hybrid distinct RGB8 batch"); - - assert_eq!(surfaces.len(), 2); - assert_ne!( - surfaces[0].as_bytes().expect("surface byte access"), - surfaces[1].as_bytes().expect("surface byte access"), - "flattened distinct RGB hybrid batches must keep each tile's coefficients separate" - ); - assert!( - hybrid_cpu_decode_inputs_for_test() >= expected_inputs, - "flattened RGB hybrid batches should still decode every distinct Tier-1 input" - ); - assert!( - flattened_hybrid_cpu_decode_batches_for_test() >= 1, - "flattened RGB hybrid should collect Tier-1 work through the flattened CPU decode queue" - ); -} - -#[test] -fn flattened_cpu_tier1_default_gate_targets_large_distinct_batches_only() { - fn color_plan(width: u32, height: u32) -> Arc { - Arc::new(PreparedDirectColorPlan { - dimensions: (width, height), - bit_depths: [8, 8, 8], - mct: true, - transform: J2kWaveletTransform::Reversible53, - component_plans: Vec::new(), - }) - } - - let repeated = vec![color_plan(1024, 1024); 16]; - assert!( - !should_flatten_hybrid_cpu_tier1_color_batch(&repeated), - "repeated RGB batches already win through shared Tier-1 decode and should not use the flattened distinct scheduler" - ); - - let small_distinct = (0..16).map(|_| color_plan(256, 256)).collect::>(); - assert!( - !should_flatten_hybrid_cpu_tier1_color_batch(&small_distinct), - "small RGB batches measured slower with flattened Tier-1 and should stay on the grouped path" - ); - - let large_distinct = (0..16).map(|_| color_plan(1024, 1024)).collect::>(); - assert!( - should_flatten_hybrid_cpu_tier1_color_batch(&large_distinct), - "large distinct RGB explicit hybrid batches measured faster with flattened Tier-1" - ); -} - -#[test] -fn hybrid_cpu_decode_worker_count_allows_two_way_small_batch_parallelism() { - let available = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get); - if available < 2 { - return; - } - - assert_eq!( - hybrid_cpu_decode_worker_count(2), - 2, - "two independent hybrid CPU Tier-1 inputs should be able to use two workers" - ); -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn cropped_region_scaled_ht_direct_plan_prunes_codeblocks_outside_output_roi() { - let mut pixels = Vec::with_capacity(256 * 256); - for y in 0..256u32 { - for x in 0..256u32 { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - } - } - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new( - &bytes, - &DecodeSettings { - target_resolution: Some((64, 64)), - ..DecodeSettings::default() - }, - ) - .expect("scaled image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - let full_jobs = prepared_direct_grayscale_ht_job_count(&prepared); - assert!( - full_jobs > 8, - "fixture should have multiple HT code-block jobs" - ); - - crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - j2k_core::Rect { - x: 24, - y: 24, - w: 8, - h: 8, - }, - ) - .expect("crop direct plan"); - let cropped_jobs = prepared_direct_grayscale_ht_job_count(&prepared); - - assert!( - cropped_jobs > 0 && cropped_jobs < full_jobs, - "cropped ROI should prune HT code-block jobs; full={full_jobs}, cropped={cropped_jobs}" - ); -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn cropped_region_scaled_ht_direct_plan_compacts_coded_payloads() { - let mut pixels = Vec::with_capacity(256 * 256); - for y in 0..256u32 { - for x in 0..256u32 { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - } - } - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new( - &bytes, - &DecodeSettings { - target_resolution: Some((64, 64)), - ..DecodeSettings::default() - }, - ) - .expect("scaled image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - let full_bytes = prepared_direct_grayscale_ht_coded_byte_count(&prepared); - assert!(full_bytes > 0, "fixture should carry HT coded payloads"); - - crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - j2k_core::Rect { - x: 24, - y: 24, - w: 8, - h: 8, - }, - ) - .expect("crop direct plan"); - let cropped_bytes = prepared_direct_grayscale_ht_coded_byte_count(&prepared); - - assert!( - cropped_bytes > 0 && cropped_bytes < full_bytes, - "cropped ROI should compact HT coded bytes; full={full_bytes}, cropped={cropped_bytes}" - ); -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn cropped_region_scaled_ht_direct_plan_reduces_idwt_output_work() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - } - } - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new( - &bytes, - &DecodeSettings { - target_resolution: Some((32, 32)), - ..DecodeSettings::default() - }, - ) - .expect("scaled image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - let full_samples = prepared_direct_grayscale_idwt_output_sample_count(&prepared); - - crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - j2k_core::Rect { - x: 10, - y: 10, - w: 4, - h: 4, - }, - ) - .expect("crop direct plan"); - let cropped_samples = prepared_direct_grayscale_idwt_output_sample_count(&prepared); - - assert!( - cropped_samples > 0 && cropped_samples < full_samples, - "cropped ROI should reduce IDWT output work; full={full_samples}, cropped={cropped_samples}" - ); -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn cropped_region_ht_direct_plan_keeps_idwt_windows_bounded() { - let mut pixels = Vec::with_capacity(256 * 256); - for y in 0..256u32 { - for x in 0..256u32 { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - } - } - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - let idwt_levels = prepared_direct_grayscale_idwt_full_and_prepared_lens(&prepared); - assert!( - idwt_levels.len() >= 3, - "fixture should exercise a multi-level IDWT plan" - ); - - crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - j2k_core::Rect { - x: 112, - y: 112, - w: 32, - h: 32, - }, - ) - .expect("crop direct plan"); - let cropped_idwt_levels = prepared_direct_grayscale_idwt_full_and_prepared_lens(&prepared); - - assert_eq!(cropped_idwt_levels.len(), idwt_levels.len()); - for (level_idx, (full_len, cropped_len)) in cropped_idwt_levels.iter().copied().enumerate() { - assert!( - cropped_len > 0 && cropped_len <= full_len, - "cropped ROI should keep IDWT level {level_idx} bounded; full={full_len}, cropped={cropped_len}" - ); - } - assert!( - cropped_idwt_levels - .iter() - .any(|(full_len, cropped_len)| cropped_len < full_len), - "cropped ROI should reduce at least one IDWT level" - ); -} - -fn prepared_direct_grayscale_ht_job_count(plan: &super::PreparedDirectGrayscalePlan) -> usize { - plan.steps - .iter() - .map(|step| match step { - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => sub_band.jobs.len(), - _ => 0, - }) - .sum() -} - -fn prepared_direct_grayscale_ht_coded_byte_count( - plan: &super::PreparedDirectGrayscalePlan, -) -> usize { - plan.steps - .iter() - .map(|step| match step { - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => sub_band.coded_data.len(), - _ => 0, - }) - .sum() -} - -fn prepared_direct_grayscale_idwt_output_sample_count( - plan: &super::PreparedDirectGrayscalePlan, -) -> usize { - plan.steps - .iter() - .map(|step| match step { - PreparedDirectGrayscaleStep::Idwt(idwt) => prepared_idwt_output_len(idwt), - _ => 0, - }) - .sum() -} - -fn prepared_direct_grayscale_idwt_full_and_prepared_lens( - plan: &super::PreparedDirectGrayscalePlan, -) -> Vec<(usize, usize)> { - plan.steps - .iter() - .filter_map(|step| match step { - PreparedDirectGrayscaleStep::Idwt(idwt) => Some(( - idwt.step.rect.width() as usize * idwt.step.rect.height() as usize, - prepared_idwt_output_len(idwt), - )), - _ => None, - }) - .collect() -} - -#[test] -#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] -fn prepared_classic_direct_plan_groups_cleanup_subbands_before_idwt() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode j2k gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let classic_subband_steps = plan - .steps - .iter() - .filter(|step| matches!(step, j2k_native::J2kDirectGrayscaleStep::ClassicSubBand(_))) - .count(); - assert!( - classic_subband_steps > 1, - "fixture must exercise multiple classic sub-band cleanup steps" - ); - - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - assert_eq!( - prepared.classic_groups.len(), - 1, - "classic J2K direct decode should group adjacent sub-band cleanups before IDWT" - ); - assert_eq!( - prepared.classic_groups[0].members.len(), - classic_subband_steps - ); - assert!(matches!( - prepared.steps[prepared.classic_groups[0].start_step], - PreparedDirectGrayscaleStep::ClassicSubBand(_) - )); -} - -#[test] -fn classic_plain_fast_path_accepts_style_zero_arithmetic_jobs() { - let jobs = [J2kClassicCleanupBatchJob { - coded_offset: 0, - coded_len: 1, - segment_offset: 0, - segment_count: 1, - width: 64, - height: 64, - output_stride: 64, - output_offset: 0, - missing_msbs: 0, - total_bitplanes: 8, - roi_shift: 0, - number_of_coding_passes: 1, - sub_band_type: 0, - style_flags: 0, - strict: 1, - dequantization_step: 1.0, - }]; - let segments = [J2kClassicSegment { - data_offset: 0, - data_length: 1, - start_coding_pass: 0, - end_coding_pass: 1, - use_arithmetic: 1, - }]; - - assert!( - classic_batch_uses_plain_fast_path(&jobs, &segments), - "style-0 arithmetic-only classic J2K jobs should use the fused plain cleanup/store kernel" - ); -} - -#[test] -fn classic_repeated_plain_fast_path_stays_off_for_wsi_batch_size() { - let jobs = [J2kClassicCleanupBatchJob { - coded_offset: 0, - coded_len: 1, - segment_offset: 0, - segment_count: 1, - width: 64, - height: 64, - output_stride: 64, - output_offset: 0, - missing_msbs: 0, - total_bitplanes: 8, - roi_shift: 0, - number_of_coding_passes: 1, - sub_band_type: 0, - style_flags: 0, - strict: 1, - dequantization_step: 1.0, - }]; - let segments = [J2kClassicSegment { - data_offset: 0, - data_length: 1, - start_coding_pass: 0, - end_coding_pass: 1, - use_arithmetic: 1, - }]; - - assert!( - !classic_repeated_uses_plain_fast_path(16, &jobs, &segments), - "batch-16 WSI classic J2K should keep the device-state cleanup plus separate store path" - ); -} - -#[test] -fn repeated_gray_store_detects_contiguous_full_wsi_tiles() { - let full_tile = J2kRepeatedGrayStoreParams { - input_width: 1024, - input_height: 1024, - source_x: 0, - source_y: 0, - copy_width: 1024, - copy_height: 1024, - output_width: 1024, - output_height: 1024, - output_x: 0, - output_y: 0, - addend: 0.0, - batch_count: 16, - max_value: 255.0, - u8_scale: 1.0, - u16_scale: 257.0, - }; - assert!( - repeated_gray_store_is_contiguous_full_surface(full_tile), - "full repeated grayscale WSI stores should use the contiguous store kernel" - ); - - let windowed = J2kRepeatedGrayStoreParams { - source_x: 1, - copy_width: 1023, - ..full_tile - }; - assert!( - !repeated_gray_store_is_contiguous_full_surface(windowed), - "ROI/windowed repeated grayscale stores must stay on the generic store kernel" - ); -} +mod capacity; +mod classic; +mod grouping; +mod hybrid; +mod hybrid_support; +mod referenced_plan; +mod reuse; +mod roi; +mod runtime; diff --git a/crates/j2k-metal/src/compute/tests/capacity.rs b/crates/j2k-metal/src/compute/tests/capacity.rs new file mode 100644 index 00000000..39080d48 --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/capacity.rs @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::abi::{ + J2kClassicEncodeBatchJob, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, J2K_HT_ENCODE_BASE_OUTPUT_SIZE, +}; +use super::super::{ + accumulate_classic_tier1_scan_estimates, classic_encode_code_blocks_pipeline_kind, + classic_encode_output_capacity, classic_encode_segment_capacity, + classic_packet_output_capacity, classic_tier1_pass_class_counts, ht_encode_output_capacity, + J2kClassicEncodePipelineKind, J2kLosslessCodestreamAssemblyJob, + J2kLosslessCodestreamBlockCodingMode, J2kResidentEncodeStageStats, +}; + +#[test] +fn classic_encode_output_capacity_keeps_conservative_default() { + let capacity = classic_encode_output_capacity(64, 64, 11).expect("classic output capacity"); + + assert_eq!(capacity, 64 * 64 * 11 * 8 + 4097); +} + +#[test] +fn classic_encode_segment_capacity_uses_coding_style_bound() { + assert_eq!(classic_encode_segment_capacity(0, 16), 1); + assert_eq!( + classic_encode_segment_capacity(J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, 9), + 11 + ); + assert_eq!( + classic_encode_segment_capacity(J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, 16), + 25 + ); + assert_eq!( + classic_encode_segment_capacity(J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, 16), + 46 + ); +} + +#[test] +fn classic_tier1_pass_class_counts_split_bypass_pass_types() { + let counts = + classic_tier1_pass_class_counts(23, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS); + + assert_eq!(counts.arithmetic, 14); + assert_eq!(counts.raw, 9); + assert_eq!(counts.cleanup, 8); + assert_eq!(counts.sigprop, 8); + assert_eq!(counts.magref, 7); + assert_eq!(counts.arithmetic_cleanup, 8); + assert_eq!(counts.arithmetic_sigprop, 3); + assert_eq!(counts.arithmetic_magref, 3); + assert_eq!(counts.raw_sigprop, 5); + assert_eq!(counts.raw_magref, 4); +} + +#[test] +fn classic_tier1_pass_class_counts_style0_stays_arithmetic() { + let counts = classic_tier1_pass_class_counts(5, 0); + + assert_eq!(counts.arithmetic, 5); + assert_eq!(counts.raw, 0); + assert_eq!(counts.cleanup, 2); + assert_eq!(counts.sigprop, 2); + assert_eq!(counts.magref, 1); + assert_eq!(counts.arithmetic_cleanup, 2); + assert_eq!(counts.arithmetic_sigprop, 2); + assert_eq!(counts.arithmetic_magref, 1); + assert_eq!(counts.raw_sigprop, 0); + assert_eq!(counts.raw_magref, 0); +} + +#[test] +fn classic_tier1_scan_estimates_multiply_passes_by_block_area() { + let pass_counts = + classic_tier1_pass_class_counts(23, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS); + let mut stats = J2kResidentEncodeStageStats::default(); + + accumulate_classic_tier1_scan_estimates(&mut stats, pass_counts, 32 * 32); + + assert_eq!(stats.tier1_full_scan_coeff_visit_count_total, 23 * 1024); + assert_eq!( + stats.tier1_arithmetic_scan_coeff_visit_count_total, + 14 * 1024 + ); + assert_eq!(stats.tier1_raw_scan_coeff_visit_count_total, 9 * 1024); + assert_eq!(stats.tier1_cleanup_scan_coeff_visit_count_total, 8 * 1024); + assert_eq!(stats.tier1_sigprop_scan_coeff_visit_count_total, 8 * 1024); + assert_eq!(stats.tier1_magref_scan_coeff_visit_count_total, 7 * 1024); + assert_eq!(stats.max_tier1_full_scan_coeff_visits_per_block, 23 * 1024); +} + +#[test] +fn classic_packet_output_capacity_uses_raw_sample_bound_when_smaller() { + let codestream = J2kLosslessCodestreamAssemblyJob { + width: 512, + height: 512, + component_count: 3, + bit_depth: 8, + signed: false, + num_decomposition_levels: 3, + use_mct: true, + guard_bits: 2, + code_block_width_exp: 4, + code_block_height_exp: 4, + progression_order: j2k_native::EncodeProgressionOrder::Lrcp, + write_tlm: false, + block_coding_mode: J2kLosslessCodestreamBlockCodingMode::Classic, + }; + let header_capacity = 1024 * 256 + 4096; + let conservative_capacity = 12 * 1024 * 1024; + let packet_descriptor_count = 3; + + let capacity = classic_packet_output_capacity( + conservative_capacity, + header_capacity, + packet_descriptor_count, + codestream, + ) + .expect("classic packet capacity"); + + let raw_bytes = 512 * 512 * 3; + let descriptor_slack = packet_descriptor_count * 256; + assert_eq!( + capacity, + raw_bytes + header_capacity + descriptor_slack + 64 * 1024 + ); + + let tiny_tier1_capacity = 4096; + let clamped = classic_packet_output_capacity( + tiny_tier1_capacity, + header_capacity, + packet_descriptor_count, + codestream, + ) + .expect("classic packet capacity"); + let conservative_packet_capacity = + tiny_tier1_capacity + header_capacity * packet_descriptor_count + 1024; + assert_eq!(clamped, conservative_packet_capacity); +} + +#[test] +fn ht_encode_output_capacity_scales_with_code_block_area() { + let max_block = ht_encode_output_capacity(128, 128).expect("max HT output capacity"); + assert_eq!(max_block, J2K_HT_ENCODE_BASE_OUTPUT_SIZE); + + let smaller_block = ht_encode_output_capacity(32, 32).expect("scaled HT output capacity"); + assert!(smaller_block < max_block / 2); + assert!(smaller_block >= 8192); +} + +#[test] +fn classic_encode_pipeline_kind_prefers_style0_32_for_resident_jobs() { + let jobs = [J2kClassicEncodeBatchJob { + width: 32, + height: 32, + style_flags: 0, + ..J2kClassicEncodeBatchJob::default() + }]; + + assert_eq!( + classic_encode_code_blocks_pipeline_kind(&jobs), + J2kClassicEncodePipelineKind::Style0_32 + ); +} + +#[test] +fn classic_encode_pipeline_kind_prefers_bypass_32_for_resident_jobs() { + let jobs = [J2kClassicEncodeBatchJob { + width: 32, + height: 32, + style_flags: J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + total_bitplanes: 31, + ..J2kClassicEncodeBatchJob::default() + }]; + + assert_eq!( + classic_encode_code_blocks_pipeline_kind(&jobs), + J2kClassicEncodePipelineKind::Bypass32 + ); +} + +#[test] +fn classic_encode_pipeline_kind_prefers_bypass_u16_32_for_low_bitplane_resident_jobs() { + let jobs = [J2kClassicEncodeBatchJob { + width: 32, + height: 32, + style_flags: J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + total_bitplanes: 16, + ..J2kClassicEncodeBatchJob::default() + }]; + + assert_eq!( + classic_encode_code_blocks_pipeline_kind(&jobs), + J2kClassicEncodePipelineKind::BypassU16_32 + ); +} diff --git a/crates/j2k-metal/src/compute/tests/classic.rs b/crates/j2k-metal/src/compute/tests/classic.rs new file mode 100644 index 00000000..b429e77d --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/classic.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::abi::{J2kClassicCleanupBatchJob, J2kClassicSegment, J2kRepeatedGrayStoreParams}; +use super::super::decode_dispatch::store::repeated_gray_store_is_contiguous_full_surface; +use super::super::decode_dispatch::{ + classic_batch_uses_plain_fast_path, classic_repeated_uses_plain_fast_path, +}; +use super::super::{ + decode_prepared_classic_sub_band_on_cpu, direct_tier1_input_buffer_prepares_for_test, + prepare_direct_color_plan, prepare_direct_color_plan_for_cpu_upload, + prepare_direct_grayscale_plan, reset_direct_tier1_input_buffer_prepares_for_test, + PreparedClassicSubBand, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, +}; +use super::runtime::should_run_metal_runtime; +use j2k_native::{ + decode_j2k_sub_band_scalar, encode, DecodeSettings, DecoderContext, EncodeOptions, Image, + J2kCodeBlockBatchJob, J2kCodeBlockDecodeJob, + J2kDirectGrayscaleStep as NativeDirectGrayscaleStep, J2kOwnedCodeBlockBatchJob, + J2kOwnedSubBandPlan, J2kSubBandDecodeJob, +}; +use metal::Device; + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn prepared_classic_sub_band_decodes_on_cpu_for_hybrid_upload() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode classic gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let native_sub_band = first_native_classic_sub_band(&plan); + let prepared_sub_band = first_prepared_classic_sub_band(&prepared); + + let expected = decode_native_classic_sub_band(native_sub_band); + let actual = + decode_prepared_classic_sub_band_on_cpu(prepared_sub_band).expect("prepared CPU decode"); + + assert_eq!(actual, expected); +} + +#[test] +fn cpu_upload_color_prepare_skips_tier1_metal_input_buffers() { + if !should_run_metal_runtime() { + return; + } + + if Device::system_default().is_none() { + j2k_test_support::metal_device_unavailable_is_skip(module_path!()); + return; + } + + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + + reset_direct_tier1_input_buffer_prepares_for_test(); + let metal_prepared = prepare_direct_color_plan(&plan).expect("Metal prepared color plan"); + assert_eq!(metal_prepared.component_plans.len(), 3); + assert!( + direct_tier1_input_buffer_prepares_for_test() > 0, + "normal Metal preparation should build Tier-1 input buffers" + ); + + reset_direct_tier1_input_buffer_prepares_for_test(); + let cpu_upload_prepared = + prepare_direct_color_plan_for_cpu_upload(&plan).expect("CPUUpload prepared color plan"); + assert_eq!(cpu_upload_prepared.component_plans.len(), 3); + assert_eq!( + direct_tier1_input_buffer_prepares_for_test(), + 0, + "CPUUpload preparation should keep coded Tier-1 payloads on CPU and skip Metal input buffers" + ); +} + +fn first_native_classic_sub_band( + plan: &j2k_native::J2kDirectGrayscalePlan, +) -> &J2kOwnedSubBandPlan { + plan.steps + .iter() + .find_map(|step| match step { + NativeDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), + _ => None, + }) + .expect("classic sub-band step") +} + +fn first_prepared_classic_sub_band(plan: &PreparedDirectGrayscalePlan) -> &PreparedClassicSubBand { + plan.steps + .iter() + .find_map(|step| match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), + _ => None, + }) + .expect("prepared classic sub-band step") +} + +fn decode_native_classic_sub_band(plan: &J2kOwnedSubBandPlan) -> Vec { + let mut output = vec![0.0_f32; plan.width as usize * plan.height as usize]; + let jobs = plan + .jobs + .iter() + .map(|job| J2kCodeBlockBatchJob { + output_x: job.output_x, + output_y: job.output_y, + code_block: native_classic_job(job), + }) + .collect::>(); + decode_j2k_sub_band_scalar( + J2kSubBandDecodeJob { + width: plan.width, + height: plan.height, + jobs: &jobs, + }, + &mut output, + ) + .expect("native scalar classic sub-band decode"); + output +} + +fn native_classic_job(job: &J2kOwnedCodeBlockBatchJob) -> J2kCodeBlockDecodeJob<'_> { + J2kCodeBlockDecodeJob { + data: &job.data, + segments: &job.segments, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + total_bitplanes: job.total_bitplanes, + roi_shift: job.roi_shift, + sub_band_type: job.sub_band_type, + style: job.style, + strict: job.strict, + dequantization_step: job.dequantization_step, + } +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn prepared_classic_direct_plan_groups_cleanup_subbands_before_idwt() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode j2k gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let classic_subband_steps = plan + .steps + .iter() + .filter(|step| matches!(step, j2k_native::J2kDirectGrayscaleStep::ClassicSubBand(_))) + .count(); + assert!( + classic_subband_steps > 1, + "fixture must exercise multiple classic sub-band cleanup steps" + ); + + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + assert_eq!( + prepared.classic_groups.len(), + 1, + "classic J2K direct decode should group adjacent sub-band cleanups before IDWT" + ); + assert_eq!( + prepared.classic_groups[0].members.len(), + classic_subband_steps + ); + assert!(matches!( + prepared.steps[prepared.classic_groups[0].start_step], + PreparedDirectGrayscaleStep::ClassicSubBand(_) + )); +} + +#[test] +fn classic_plain_fast_path_accepts_style_zero_arithmetic_jobs() { + let jobs = [J2kClassicCleanupBatchJob { + coded_offset: 0, + coded_len: 1, + segment_offset: 0, + segment_count: 1, + width: 64, + height: 64, + output_stride: 64, + output_offset: 0, + missing_msbs: 0, + total_bitplanes: 8, + roi_shift: 0, + number_of_coding_passes: 1, + sub_band_type: 0, + style_flags: 0, + strict: 1, + dequantization_step: 1.0, + }]; + let segments = [J2kClassicSegment { + data_offset: 0, + data_length: 1, + start_coding_pass: 0, + end_coding_pass: 1, + use_arithmetic: 1, + }]; + + assert!( + classic_batch_uses_plain_fast_path(&jobs, &segments), + "style-0 arithmetic-only classic J2K jobs should use the fused plain cleanup/store kernel" + ); +} + +#[test] +fn classic_repeated_plain_fast_path_stays_off_for_wsi_batch_size() { + let jobs = [J2kClassicCleanupBatchJob { + coded_offset: 0, + coded_len: 1, + segment_offset: 0, + segment_count: 1, + width: 64, + height: 64, + output_stride: 64, + output_offset: 0, + missing_msbs: 0, + total_bitplanes: 8, + roi_shift: 0, + number_of_coding_passes: 1, + sub_band_type: 0, + style_flags: 0, + strict: 1, + dequantization_step: 1.0, + }]; + let segments = [J2kClassicSegment { + data_offset: 0, + data_length: 1, + start_coding_pass: 0, + end_coding_pass: 1, + use_arithmetic: 1, + }]; + + assert!( + !classic_repeated_uses_plain_fast_path(16, &jobs, &segments), + "batch-16 WSI classic J2K should keep the device-state cleanup plus separate store path" + ); +} + +#[test] +fn repeated_gray_store_detects_contiguous_full_wsi_tiles() { + let full_tile = J2kRepeatedGrayStoreParams { + input_width: 1024, + input_height: 1024, + source_x: 0, + source_y: 0, + copy_width: 1024, + copy_height: 1024, + output_width: 1024, + output_height: 1024, + output_x: 0, + output_y: 0, + addend: 0.0, + batch_count: 16, + max_value: 255.0, + u8_scale: 1.0, + u16_scale: 257.0, + }; + assert!( + repeated_gray_store_is_contiguous_full_surface(full_tile), + "full repeated grayscale WSI stores should use the contiguous store kernel" + ); + + let windowed = J2kRepeatedGrayStoreParams { + source_x: 1, + copy_width: 1023, + ..full_tile + }; + assert!( + !repeated_gray_store_is_contiguous_full_surface(windowed), + "ROI/windowed repeated grayscale stores must stay on the generic store kernel" + ); +} diff --git a/crates/j2k-metal/src/compute/tests/grouping.rs b/crates/j2k-metal/src/compute/tests/grouping.rs new file mode 100644 index 00000000..5dc3e35d --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/grouping.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::abi::J2kHtCleanupBatchJob; +use super::super::direct_prepare::prepare_sub_band_groups; +use super::super::direct_roi::retain_ht_jobs_for_required_region; +use super::super::{ + direct_tier1_input_buffer_prepares_for_test, prepare_direct_grayscale_plan, + prepared_direct_grayscale_plan_compute_encoder_count, + prepared_repeated_direct_ht_cleanup_dispatch_count, + reset_direct_tier1_input_buffer_prepares_for_test, + supports_stacked_direct_component_plane_batch, DirectTier1Mode, PreparedDirectGrayscaleStep, + PreparedHtExecutionOwner, PreparedHtPayloadSource, PreparedHtSubBand, +}; +use j2k_core::PixelFormat; +use j2k_native::{encode_htj2k, DecodeSettings, DecoderContext, EncodeOptions, Image}; +use std::sync::Arc; + +fn test_ht_job(output_x: u32, output_y: u32, width: u32, height: u32) -> J2kHtCleanupBatchJob { + J2kHtCleanupBatchJob { + coded_offset: output_y + .checked_mul(32) + .and_then(|base| base.checked_add(output_x)) + .expect("test coded offset"), + width, + height, + coded_len: 1, + cleanup_length: 1, + refinement_length: 0, + missing_msbs: 0, + num_bitplanes: 8, + roi_shift: 0, + number_of_coding_passes: 1, + output_stride: 8, + output_offset: output_y + .checked_mul(8) + .and_then(|base| base.checked_add(output_x)) + .expect("test output offset"), + dequantization_step: 1.0, + stripe_causal: 0, + } +} + +#[expect( + clippy::cast_possible_truncation, + reason = "test helper receives small synthetic band identifiers" +)] +fn test_ht_sub_band(band_id: u32) -> PreparedHtSubBand { + PreparedHtSubBand { + band_id, + width: 8, + height: 8, + payload_source: PreparedHtPayloadSource::Contiguous(vec![band_id as u8]), + jobs: vec![test_ht_job(0, 0, 2, 2)], + execution_owner: Arc::new(PreparedHtExecutionOwner), + } +} + +fn separator_store_step() -> PreparedDirectGrayscaleStep { + PreparedDirectGrayscaleStep::Store(j2k_native::J2kDirectStoreStep { + input_band_id: 99, + input_rect: j2k_native::J2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + source_x: 0, + source_y: 0, + copy_width: 1, + copy_height: 1, + output_width: 1, + output_height: 1, + output_x: 0, + output_y: 0, + addend: 0.0, + }) +} + +#[test] +fn direct_sub_band_grouping_groups_adjacent_ht_runs_without_runtime() { + let steps = vec![ + PreparedDirectGrayscaleStep::HtSubBand(test_ht_sub_band(1)), + PreparedDirectGrayscaleStep::HtSubBand(test_ht_sub_band(2)), + separator_store_step(), + PreparedDirectGrayscaleStep::HtSubBand(test_ht_sub_band(3)), + PreparedDirectGrayscaleStep::HtSubBand(test_ht_sub_band(4)), + ]; + + let groups = prepare_sub_band_groups( + &steps, + DirectTier1Mode::CpuUpload, + |step| match step { + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => Some(sub_band), + _ => None, + }, + |start, end, sub_bands, _| { + Ok(( + start, + end, + sub_bands + .iter() + .map(|sub_band| sub_band.band_id) + .collect::>(), + )) + }, + ) + .expect("group adjacent HT sub-bands"); + + assert_eq!(groups, vec![(0, 2, vec![1, 2]), (3, 5, vec![3, 4])]); +} + +#[test] +fn direct_roi_prunes_ht_jobs_without_runtime() { + let mut jobs = vec![ + test_ht_job(0, 0, 2, 2), + test_ht_job(6, 0, 2, 2), + test_ht_job(2, 2, 3, 3), + ]; + let required = + j2k_native::J2kRequiredBandRegion::new(0, 0, 4, 4).expect("required test region"); + + retain_ht_jobs_for_required_region(&mut jobs, Some(required)); + + let retained_offsets = jobs.iter().map(|job| job.output_offset).collect::>(); + assert_eq!(retained_offsets, vec![0, 18]); + + retain_ht_jobs_for_required_region(&mut jobs, None); + assert!(jobs.is_empty()); +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn prepared_ht_direct_plan_groups_cleanup_subbands_before_idwt() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let ht_subband_steps = plan + .steps + .iter() + .filter(|step| matches!(step, j2k_native::J2kDirectGrayscaleStep::HtSubBand(_))) + .count(); + assert!( + ht_subband_steps > 1, + "fixture must exercise multiple HT sub-band cleanup steps" + ); + + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + assert_eq!( + prepared.ht_groups.len(), + 1, + "single-tile HTJ2K direct decode should group adjacent HT sub-bands into one cleanup dispatch" + ); + assert_eq!(prepared.ht_groups[0].members.len(), ht_subband_steps); + assert!(matches!( + prepared.steps[prepared.ht_groups[0].start_step], + PreparedDirectGrayscaleStep::HtSubBand(_) + )); +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn grouped_ht_direct_plan_uses_one_group_coded_arena() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + + reset_direct_tier1_input_buffer_prepares_for_test(); + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + assert_eq!( + prepared.ht_groups.len(), + 1, + "fixture must exercise one grouped HT dispatch" + ); + let group = &prepared.ht_groups[0]; + assert!(matches!( + &group.payload_source, + PreparedHtPayloadSource::Contiguous(data) if !data.is_empty() + )); + assert_eq!( + direct_tier1_input_buffer_prepares_for_test(), + 0, + "HT payload and job buffers must be bounded and uploaded only by the chunk submission" + ); +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn prepared_ht_direct_plan_encodes_full_decode_in_one_compute_encoder() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + + assert_eq!( + prepared_direct_grayscale_plan_compute_encoder_count(&prepared, PixelFormat::Gray8), + 1, + "prepared single-tile direct decode should keep cleanup, IDWT, and grayscale store in one compute encoder" + ); +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn repeated_prepared_ht_direct_plan_groups_cleanup_subbands_before_idwt() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let ht_subband_steps = plan + .steps + .iter() + .filter(|step| matches!(step, j2k_native::J2kDirectGrayscaleStep::HtSubBand(_))) + .count(); + assert!( + ht_subband_steps > 1, + "fixture must exercise multiple HT sub-band cleanup steps" + ); + + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + assert_eq!( + prepared_repeated_direct_ht_cleanup_dispatch_count(&prepared), + 1, + "repeated HTJ2K WSI tile batches should group adjacent sub-band cleanups like the single-tile path" + ); +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn distinct_prepared_ht_direct_plans_support_stacked_component_batch() { + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes_a = encode_htj2k(&(0..64).collect::>(), 8, 8, 1, 8, false, &options) + .expect("encode first ht gray8"); + let bytes_b = encode_htj2k( + &(0..64).rev().collect::>(), + 8, + 8, + 1, + 8, + false, + &options, + ) + .expect("encode second ht gray8"); + let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); + let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); + let mut context_a = DecoderContext::default(); + let mut context_b = DecoderContext::default(); + let plan_a = image_a + .build_direct_grayscale_plan_with_context(&mut context_a) + .expect("first direct plan"); + let plan_b = image_b + .build_direct_grayscale_plan_with_context(&mut context_b) + .expect("second direct plan"); + let prepared_a = prepare_direct_grayscale_plan(&plan_a).expect("first prepared plan"); + let prepared_b = prepare_direct_grayscale_plan(&plan_b).expect("second prepared plan"); + + assert!( + supports_stacked_direct_component_plane_batch(&[&prepared_a, &prepared_b]), + "distinct same-shape HTJ2K grayscale plans should be eligible for one stacked batch graph" + ); +} diff --git a/crates/j2k-metal/src/compute/tests/hybrid.rs b/crates/j2k-metal/src/compute/tests/hybrid.rs new file mode 100644 index 00000000..559f7897 --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/hybrid.rs @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::direct_grayscale_execute::execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test; +use super::super::{ + execute_hybrid_cpu_tier1_direct_color_plan_batch, flattened_hybrid_cpu_decode_batches_for_test, + hybrid_cpu_decode_inputs_for_test, hybrid_cpu_decode_worker_count, + hybrid_cpu_decode_worker_inits_for_test, hybrid_stacked_component_batches_for_test, + prepare_direct_color_plan, reset_flattened_hybrid_cpu_decode_batches_for_test, + reset_hybrid_cpu_decode_inputs_for_test, reset_hybrid_cpu_decode_worker_inits_for_test, + reset_hybrid_stacked_component_batches_for_test, + reset_thread_hybrid_cpu_decode_inputs_for_test, should_flatten_hybrid_cpu_tier1_color_batch, + thread_hybrid_cpu_decode_inputs_for_test, PreparedDirectColorPlan, +}; +use super::{ + hybrid_support::{prepared_direct_color_tier1_input_count, HYBRID_COUNTER_TEST_LOCK}, + runtime::should_run_metal_runtime, +}; +use j2k_core::PixelFormat; +use j2k_native::{ + encode, DecodeSettings, DecoderContext, EncodeOptions, Image, J2kWaveletTransform, +}; +use std::sync::Arc; + +#[test] +fn hybrid_rgb8_batch_uses_stacked_component_graph() { + if !should_run_metal_runtime() { + return; + } + + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_stacked_component_batches_for_test(); + reset_hybrid_cpu_decode_worker_inits_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared.clone(), prepared], + PixelFormat::Rgb8, + ) + .expect("hybrid RGB8 batch"); + + assert_eq!(surfaces.len(), 2); + assert!( + hybrid_stacked_component_batches_for_test() >= 3, + "hybrid RGB batch should stack each component plane instead of encoding each tile/component serially" + ); + assert!( + hybrid_cpu_decode_worker_inits_for_test() > 0, + "hybrid RGB batch should use worker-local CPU decode scratch instead of per-input decode/flatten" + ); +} + +#[test] +fn hybrid_rgb8_repeated_batch_decodes_shared_tier1_inputs_once() { + if !should_run_metal_runtime() { + return; + } + + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let unique_tier1_inputs = prepared_direct_color_tier1_input_count(&prepared); + assert!( + unique_tier1_inputs > 0, + "fixture should have Tier-1 inputs to decode" + ); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + reset_thread_hybrid_cpu_decode_inputs_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared.clone(), prepared.clone(), prepared], + PixelFormat::Rgb8, + ) + .expect("hybrid repeated RGB8 batch"); + + assert_eq!(surfaces.len(), 3); + assert!( + hybrid_cpu_decode_inputs_for_test() >= unique_tier1_inputs, + "repeated RGB hybrid batches should decode the shared coefficient inputs" + ); +} + +#[test] +fn hybrid_rgb8_distinct_batch_keeps_tier1_inputs_separate() { + if !should_run_metal_runtime() { + return; + } + + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes_a = encode( + &j2k_test_support::gradient_variant_u8(32, 32, 3, 0), + 32, + 32, + 3, + 8, + false, + &options, + ) + .expect("encode first rgb8"); + let bytes_b = encode( + &j2k_test_support::gradient_variant_u8(32, 32, 3, 7), + 32, + 32, + 3, + 8, + false, + &options, + ) + .expect("encode second rgb8"); + let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); + let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); + let mut context_a = DecoderContext::default(); + let mut context_b = DecoderContext::default(); + let plan_a = image_a + .build_direct_color_plan_with_context(&mut context_a) + .expect("first direct color plan"); + let plan_b = image_b + .build_direct_color_plan_with_context(&mut context_b) + .expect("second direct color plan"); + let prepared_a = Arc::new(prepare_direct_color_plan(&plan_a).expect("first prepared")); + let prepared_b = Arc::new(prepare_direct_color_plan(&plan_b).expect("second prepared")); + let expected_inputs = prepared_direct_color_tier1_input_count(&prepared_a) + + prepared_direct_color_tier1_input_count(&prepared_b); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared_a, prepared_b], + PixelFormat::Rgb8, + ) + .expect("hybrid distinct RGB8 batch"); + + assert_eq!(surfaces.len(), 2); + assert_ne!( + surfaces[0].as_bytes().expect("surface byte access"), + surfaces[1].as_bytes().expect("surface byte access"), + "distinct RGB inputs must not reuse the first tile's decoded coefficients" + ); + assert_eq!( + thread_hybrid_cpu_decode_inputs_for_test(), + expected_inputs, + "distinct RGB hybrid batches should decode each tile's own Tier-1 inputs" + ); +} + +#[test] +fn hybrid_rgb8_flattened_cpu_tier1_batch_uses_one_decode_queue() { + if !should_run_metal_runtime() { + return; + } + + let pixels_a = j2k_test_support::gradient_variant_u8(32, 32, 3, 0); + let pixels_b = j2k_test_support::gradient_variant_u8(32, 32, 3, 11); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes_a = encode(&pixels_a, 32, 32, 3, 8, false, &options).expect("encode first rgb8"); + let bytes_b = encode(&pixels_b, 32, 32, 3, 8, false, &options).expect("encode second rgb8"); + let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); + let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); + let mut context_a = DecoderContext::default(); + let mut context_b = DecoderContext::default(); + let plan_a = image_a + .build_direct_color_plan_with_context(&mut context_a) + .expect("first direct color plan"); + let plan_b = image_b + .build_direct_color_plan_with_context(&mut context_b) + .expect("second direct color plan"); + let prepared_a = Arc::new(prepare_direct_color_plan(&plan_a).expect("first prepared")); + let prepared_b = Arc::new(prepare_direct_color_plan(&plan_b).expect("second prepared")); + let expected_inputs = prepared_direct_color_tier1_input_count(&prepared_a) + + prepared_direct_color_tier1_input_count(&prepared_b); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + reset_flattened_hybrid_cpu_decode_batches_for_test(); + + let surfaces = execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test( + &[prepared_a, prepared_b], + PixelFormat::Rgb8, + ) + .expect("flattened hybrid distinct RGB8 batch"); + + assert_eq!(surfaces.len(), 2); + assert_ne!( + surfaces[0].as_bytes().expect("surface byte access"), + surfaces[1].as_bytes().expect("surface byte access"), + "flattened distinct RGB hybrid batches must keep each tile's coefficients separate" + ); + assert!( + hybrid_cpu_decode_inputs_for_test() >= expected_inputs, + "flattened RGB hybrid batches should still decode every distinct Tier-1 input" + ); + assert!( + flattened_hybrid_cpu_decode_batches_for_test() >= 1, + "flattened RGB hybrid should collect Tier-1 work through the flattened CPU decode queue" + ); +} + +#[test] +fn flattened_cpu_tier1_default_gate_targets_large_distinct_batches_only() { + fn color_plan(width: u32, height: u32) -> Arc { + Arc::new(PreparedDirectColorPlan { + dimensions: (width, height), + bit_depths: [8, 8, 8], + alpha_bit_depth: None, + signed: false, + mct: true, + transform: J2kWaveletTransform::Reversible53, + component_plans: Vec::new(), + }) + } + + let repeated = vec![color_plan(1024, 1024); 16]; + assert!( + !should_flatten_hybrid_cpu_tier1_color_batch(&repeated), + "repeated RGB batches already win through shared Tier-1 decode and should not use the flattened distinct scheduler" + ); + + let small_distinct = (0..16).map(|_| color_plan(256, 256)).collect::>(); + assert!( + !should_flatten_hybrid_cpu_tier1_color_batch(&small_distinct), + "small RGB batches measured slower with flattened Tier-1 and should stay on the grouped path" + ); + + let large_distinct = (0..16).map(|_| color_plan(1024, 1024)).collect::>(); + assert!( + should_flatten_hybrid_cpu_tier1_color_batch(&large_distinct), + "large distinct RGB explicit hybrid batches measured faster with flattened Tier-1" + ); +} + +#[test] +fn hybrid_cpu_decode_worker_count_allows_two_way_small_batch_parallelism() { + let available = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get); + if available < 2 { + return; + } + + assert_eq!( + hybrid_cpu_decode_worker_count(2), + 2, + "two independent hybrid CPU Tier-1 inputs should be able to use two workers" + ); +} diff --git a/crates/j2k-metal/src/compute/tests/hybrid_support.rs b/crates/j2k-metal/src/compute/tests/hybrid_support.rs new file mode 100644 index 00000000..2d511229 --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/hybrid_support.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::{ + PreparedDirectColorPlan, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, +}; +use std::sync::Mutex; + +pub(super) static HYBRID_COUNTER_TEST_LOCK: Mutex<()> = Mutex::new(()); + +pub(super) fn prepared_direct_color_tier1_input_count(plan: &PreparedDirectColorPlan) -> usize { + plan.component_plans + .iter() + .map(prepared_direct_component_tier1_input_count) + .sum() +} + +fn prepared_direct_component_tier1_input_count(plan: &PreparedDirectGrayscalePlan) -> usize { + let mut count = 0; + let mut step_idx = 0; + while step_idx < plan.steps.len() { + if let Some(group) = plan.classic_group_starting_at(step_idx) { + count += 1; + step_idx = group.end_step; + continue; + } + if let Some(group) = plan.ht_group_starting_at(step_idx) { + count += 1; + step_idx = group.end_step; + continue; + } + if matches!( + &plan.steps[step_idx], + PreparedDirectGrayscaleStep::ClassicSubBand(_) + | PreparedDirectGrayscaleStep::HtSubBand(_) + ) { + count += 1; + } + step_idx += 1; + } + count +} + +pub(super) fn cached_direct_color_tier1_input_count(plan: &PreparedDirectColorPlan) -> usize { + plan.component_plans + .iter() + .map(cached_direct_component_tier1_input_count) + .sum() +} + +fn cached_direct_component_tier1_input_count(plan: &PreparedDirectGrayscalePlan) -> usize { + let mut count = 0; + let mut step_idx = 0; + while step_idx < plan.steps.len() { + if let Some(group) = plan.classic_group_starting_at(step_idx) { + if has_cached_cpu_tier1_coefficients(plan, step_idx, group.total_coefficients) { + count += 1; + } + step_idx = group.end_step; + continue; + } + if let Some(group) = plan.ht_group_starting_at(step_idx) { + if has_cached_cpu_tier1_coefficients(plan, step_idx, group.total_coefficients) { + count += 1; + } + step_idx = group.end_step; + continue; + } + match &plan.steps[step_idx] { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let output_len = sub_band.width as usize * sub_band.height as usize; + if has_cached_cpu_tier1_coefficients(plan, step_idx, output_len) { + count += 1; + } + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + let output_len = sub_band.width as usize * sub_band.height as usize; + if has_cached_cpu_tier1_coefficients(plan, step_idx, output_len) { + count += 1; + } + } + PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => {} + } + step_idx += 1; + } + count +} + +fn has_cached_cpu_tier1_coefficients( + plan: &PreparedDirectGrayscalePlan, + step_idx: usize, + output_len: usize, +) -> bool { + let mut budget = + crate::batch_allocation::BatchMetadataBudget::new("J2K Metal test CPU Tier-1 cache lookup"); + plan.cached_cpu_tier1_coefficients(&mut budget, step_idx, output_len) + .expect("CPU Tier-1 cache lookup") + .is_some() +} diff --git a/crates/j2k-metal/src/compute/tests/referenced_plan.rs b/crates/j2k-metal/src/compute/tests/referenced_plan.rs new file mode 100644 index 00000000..8b050677 --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/referenced_plan.rs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_native::{ + encode_htj2k, DecodeSettings, DecoderContext, EncodeOptions, Image, J2kDirectGrayscaleStep, +}; + +mod prepared; + +#[test] +fn referenced_htj2k_payload_ranges_reconstruct_owned_direct_plan_bytes() { + let pixels = (0_u8..64).collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode HT fixture"); + let image = Image::new(&bytes, &DecodeSettings::strict()).expect("strict HT image"); + let mut owned_context = DecoderContext::default(); + let owned = image + .build_direct_grayscale_plan_with_context(&mut owned_context) + .expect("owned direct plan"); + let mut referenced_context = DecoderContext::default(); + let referenced = image + .build_referenced_htj2k_plan_region_with_context(&mut referenced_context, (0, 0, 8, 8)) + .expect("referenced direct plan"); + let geometry = referenced + .grayscale_geometry() + .expect("grayscale referenced plan"); + let mut payload_cursor = 0usize; + + for (owned_step, referenced_step) in owned.steps.iter().zip(&geometry.steps) { + let ( + J2kDirectGrayscaleStep::HtSubBand(owned_sub_band), + J2kDirectGrayscaleStep::HtSubBand(referenced_sub_band), + ) = (owned_step, referenced_step) + else { + continue; + }; + for (owned_job, referenced_job) in owned_sub_band.jobs.iter().zip(&referenced_sub_band.jobs) + { + assert!(referenced_job.data.is_empty()); + let payload = referenced + .payloads() + .get(payload_cursor) + .expect("payload for referenced job"); + payload_cursor += 1; + let mut reconstructed = + bytes[payload.cleanup.offset..payload.cleanup.end().expect("cleanup end")].to_vec(); + if let Some(refinement) = payload.refinement { + reconstructed.extend_from_slice( + &bytes[refinement.offset..refinement.end().expect("refinement end")], + ); + } + assert_eq!(reconstructed, owned_job.data); + } + } + assert_eq!(payload_cursor, referenced.payloads().len()); +} diff --git a/crates/j2k-metal/src/compute/tests/referenced_plan/prepared.rs b/crates/j2k-metal/src/compute/tests/referenced_plan/prepared.rs new file mode 100644 index 00000000..66cc79c9 --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/referenced_plan/prepared.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k_native::{encode_htj2k, DecodeSettings, DecoderContext, EncodeOptions, Image}; + +use crate::compute::{ + decode_prepared_ht_sub_band_group_on_cpu_profile, prepare_direct_grayscale_plan, + prepare_referenced_htj2k_grayscale_plan, PreparedDirectGrayscaleStep, PreparedHtPayloadSource, +}; + +#[test] +fn referenced_metal_plan_retains_ranges_without_copying_compressed_payloads() { + let pixels = (0_u8..64).collect::>(); + let bytes = Arc::<[u8]>::from( + encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + .expect("encode referenced Metal fixture"), + ); + let image = Image::new(&bytes, &DecodeSettings::strict()).expect("strict HT image"); + let mut owned_context = DecoderContext::default(); + let owned = image + .build_direct_grayscale_plan_with_context(&mut owned_context) + .expect("owned direct plan"); + let owned_prepared = prepare_direct_grayscale_plan(&owned).expect("owned Metal plan"); + let mut context = DecoderContext::default(); + let referenced = image + .build_referenced_htj2k_plan_region_with_context(&mut context, (0, 0, 8, 8)) + .expect("referenced direct plan"); + + let prepared = prepare_referenced_htj2k_grayscale_plan(&referenced, &bytes) + .expect("referenced Metal plan"); + + for step in &prepared.steps { + let PreparedDirectGrayscaleStep::HtSubBand(sub_band) = step else { + continue; + }; + let PreparedHtPayloadSource::Referenced { input, ranges } = &sub_band.payload_source else { + panic!("referenced sub-band must retain input ranges"); + }; + assert!(Arc::ptr_eq(input, &bytes)); + assert_eq!(ranges.len(), sub_band.jobs.len()); + } + for group in &prepared.ht_groups { + let PreparedHtPayloadSource::Referenced { input, ranges } = &group.payload_source else { + panic!("referenced group must retain input ranges"); + }; + assert!(Arc::ptr_eq(input, &bytes)); + assert_eq!(ranges.len(), group.jobs.len()); + } + let step_capacity = prepared + .steps + .iter() + .map(|step| match step { + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + sub_band.payload_source.owned_payload_capacity() + } + _ => 0, + }) + .sum::(); + let group_capacity = prepared + .ht_groups + .iter() + .map(|group| group.payload_source.owned_payload_capacity()) + .sum::(); + assert_eq!( + step_capacity + group_capacity, + 0, + "prepared referenced HT sub-bands and groups must own no compressed payload copies", + ); + assert_eq!(prepared.ht_groups.len(), owned_prepared.ht_groups.len()); + for (referenced_group, owned_group) in prepared.ht_groups.iter().zip(&owned_prepared.ht_groups) + { + let referenced_coefficients = + decode_prepared_ht_sub_band_group_on_cpu_profile(referenced_group, None) + .expect("referenced CPU fallback decode"); + let owned_coefficients = + decode_prepared_ht_sub_band_group_on_cpu_profile(owned_group, None) + .expect("owned CPU fallback decode"); + assert_eq!(referenced_coefficients, owned_coefficients); + } +} diff --git a/crates/j2k-metal/src/compute/tests/reuse.rs b/crates/j2k-metal/src/compute/tests/reuse.rs new file mode 100644 index 00000000..e236a032 --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/reuse.rs @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::{ + execute_hybrid_cpu_tier1_direct_color_plan_batch, hybrid_repeated_output_blits_for_test, + prepare_direct_color_plan, reset_hybrid_cpu_decode_inputs_for_test, + reset_hybrid_repeated_output_blits_for_test, reset_thread_hybrid_cpu_decode_inputs_for_test, + thread_hybrid_cpu_decode_inputs_for_test, +}; +use super::{ + hybrid_support::{ + cached_direct_color_tier1_input_count, prepared_direct_color_tier1_input_count, + HYBRID_COUNTER_TEST_LOCK, + }, + runtime::should_run_metal_runtime, +}; +use j2k_core::PixelFormat; +use j2k_native::{encode, DecodeSettings, DecoderContext, EncodeOptions, Image}; +use std::sync::Arc; + +#[test] +fn hybrid_rgb8_reused_plan_caches_cpu_tier1_inputs_across_calls() { + if !should_run_metal_runtime() { + return; + } + + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let unique_tier1_inputs = prepared_direct_color_tier1_input_count(&prepared); + assert!( + unique_tier1_inputs > 0, + "fixture should have Tier-1 inputs to decode" + ); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + reset_thread_hybrid_cpu_decode_inputs_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared.clone(), prepared.clone()], + PixelFormat::Rgb8, + ) + .expect("first hybrid repeated RGB8 batch"); + assert_eq!(surfaces.len(), 2); + assert_eq!( + cached_direct_color_tier1_input_count(&prepared), + unique_tier1_inputs, + "first RGB hybrid call should cache every decoded CPU Tier-1 input" + ); + assert_eq!( + thread_hybrid_cpu_decode_inputs_for_test(), + unique_tier1_inputs, + "first RGB hybrid call should decode each shared Tier-1 input once" + ); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared.clone(), prepared.clone()], + PixelFormat::Rgb8, + ) + .expect("second hybrid repeated RGB8 batch"); + assert_eq!(surfaces.len(), 2); + assert_eq!( + cached_direct_color_tier1_input_count(&prepared), + unique_tier1_inputs, + "second RGB hybrid call should keep every decoded CPU Tier-1 input cached" + ); + assert_eq!( + thread_hybrid_cpu_decode_inputs_for_test(), + unique_tier1_inputs, + "second RGB hybrid call must reuse cached CPU Tier-1 coefficients without re-decoding" + ); +} + +#[test] +fn hybrid_rgb8_repeated_batch_decodes_once_and_blits_distinct_outputs() { + if !should_run_metal_runtime() { + return; + } + + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_repeated_output_blits_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[ + prepared.clone(), + prepared.clone(), + prepared.clone(), + prepared, + ], + PixelFormat::Rgb8, + ) + .expect("hybrid repeated RGB8 batch"); + + assert_eq!(surfaces.len(), 4); + let surface_bytes = surfaces[0].as_bytes().expect("surface byte access").len(); + let offsets = surfaces + .iter() + .map(|surface| { + surface + .metal_buffer_trusted() + .expect("resident Metal surface") + .1 + }) + .collect::>(); + assert_eq!( + offsets, + (0..surfaces.len()) + .map(|index| index * surface_bytes) + .collect::>(), + "repeated outputs must retain distinct Metal buffer offsets" + ); + for surface in &surfaces[1..] { + assert_eq!( + surface.as_bytes().expect("surface byte access"), + surfaces[0].as_bytes().expect("surface byte access"), + "repeated outputs should remain byte-identical" + ); + } + assert_eq!( + hybrid_repeated_output_blits_for_test(), + 2, + "repeated RGB hybrid batches should duplicate packed output surfaces with logarithmic Metal blit ranges" + ); +} diff --git a/crates/j2k-metal/src/compute/tests/roi.rs b/crates/j2k-metal/src/compute/tests/roi.rs new file mode 100644 index 00000000..8760c3b1 --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/roi.rs @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::direct_roi::prepared_idwt_output_len; +use super::super::{ + crop_prepared_direct_grayscale_plan_to_output_region, prepare_direct_grayscale_plan, + PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, PreparedHtPayloadSource, +}; +use j2k_native::{encode_htj2k, DecodeSettings, DecoderContext, EncodeOptions, Image}; + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn cropped_region_scaled_ht_direct_plan_prunes_codeblocks_outside_output_roi() { + let mut pixels = Vec::with_capacity(256 * 256); + for y in 0..256u32 { + for x in 0..256u32 { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((64, 64)), + ..DecodeSettings::default() + }, + ) + .expect("scaled image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let full_jobs = prepared_direct_grayscale_ht_job_count(&prepared); + assert!( + full_jobs > 8, + "fixture should have multiple HT code-block jobs" + ); + + crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + j2k_core::Rect { + x: 24, + y: 24, + w: 8, + h: 8, + }, + ) + .expect("crop direct plan"); + let cropped_jobs = prepared_direct_grayscale_ht_job_count(&prepared); + + assert!( + cropped_jobs > 0 && cropped_jobs < full_jobs, + "cropped ROI should prune HT code-block jobs; full={full_jobs}, cropped={cropped_jobs}" + ); +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn cropped_region_scaled_ht_direct_plan_compacts_coded_payloads() { + let mut pixels = Vec::with_capacity(256 * 256); + for y in 0..256u32 { + for x in 0..256u32 { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((64, 64)), + ..DecodeSettings::default() + }, + ) + .expect("scaled image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let full_bytes = prepared_direct_grayscale_ht_coded_byte_count(&prepared); + assert!(full_bytes > 0, "fixture should carry HT coded payloads"); + + crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + j2k_core::Rect { + x: 24, + y: 24, + w: 8, + h: 8, + }, + ) + .expect("crop direct plan"); + let cropped_bytes = prepared_direct_grayscale_ht_coded_byte_count(&prepared); + + assert!( + cropped_bytes > 0 && cropped_bytes < full_bytes, + "cropped ROI should compact HT coded bytes; full={full_bytes}, cropped={cropped_bytes}" + ); +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn cropped_region_scaled_ht_direct_plan_reduces_idwt_output_work() { + let mut pixels = Vec::with_capacity(128 * 128); + for y in 0..128u32 { + for x in 0..128u32 { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((32, 32)), + ..DecodeSettings::default() + }, + ) + .expect("scaled image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let full_samples = prepared_direct_grayscale_idwt_output_sample_count(&prepared); + + crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + j2k_core::Rect { + x: 10, + y: 10, + w: 4, + h: 4, + }, + ) + .expect("crop direct plan"); + let cropped_samples = prepared_direct_grayscale_idwt_output_sample_count(&prepared); + + assert!( + cropped_samples > 0 && cropped_samples < full_samples, + "cropped ROI should reduce IDWT output work; full={full_samples}, cropped={cropped_samples}" + ); +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn cropped_region_ht_direct_plan_keeps_idwt_windows_bounded() { + let mut pixels = Vec::with_capacity(256 * 256); + for y in 0..256u32 { + for x in 0..256u32 { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let idwt_levels = prepared_direct_grayscale_idwt_full_and_prepared_lens(&prepared); + assert!( + idwt_levels.len() >= 3, + "fixture should exercise a multi-level IDWT plan" + ); + + crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + j2k_core::Rect { + x: 112, + y: 112, + w: 32, + h: 32, + }, + ) + .expect("crop direct plan"); + let cropped_idwt_levels = prepared_direct_grayscale_idwt_full_and_prepared_lens(&prepared); + + assert_eq!(cropped_idwt_levels.len(), idwt_levels.len()); + for (level_idx, (full_len, cropped_len)) in cropped_idwt_levels.iter().copied().enumerate() { + assert!( + cropped_len > 0 && cropped_len <= full_len, + "cropped ROI should keep IDWT level {level_idx} bounded; full={full_len}, cropped={cropped_len}" + ); + } + assert!( + cropped_idwt_levels + .iter() + .any(|(full_len, cropped_len)| cropped_len < full_len), + "cropped ROI should reduce at least one IDWT level" + ); +} + +fn prepared_direct_grayscale_ht_job_count(plan: &PreparedDirectGrayscalePlan) -> usize { + plan.steps + .iter() + .map(|step| match step { + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => sub_band.jobs.len(), + _ => 0, + }) + .sum() +} + +fn prepared_direct_grayscale_ht_coded_byte_count(plan: &PreparedDirectGrayscalePlan) -> usize { + plan.steps + .iter() + .map(|step| match step { + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => match &sub_band.payload_source { + PreparedHtPayloadSource::Contiguous(data) => data.len(), + PreparedHtPayloadSource::Referenced { .. } => 0, + }, + _ => 0, + }) + .sum() +} + +fn prepared_direct_grayscale_idwt_output_sample_count(plan: &PreparedDirectGrayscalePlan) -> usize { + plan.steps + .iter() + .map(|step| match step { + PreparedDirectGrayscaleStep::Idwt(idwt) => prepared_idwt_output_len(idwt) + .expect("prepared IDWT output dimensions must fit Metal f32 storage"), + _ => 0, + }) + .sum() +} + +fn prepared_direct_grayscale_idwt_full_and_prepared_lens( + plan: &PreparedDirectGrayscalePlan, +) -> Vec<(usize, usize)> { + plan.steps + .iter() + .filter_map(|step| match step { + PreparedDirectGrayscaleStep::Idwt(idwt) => Some(( + idwt.step.rect.width() as usize * idwt.step.rect.height() as usize, + prepared_idwt_output_len(idwt) + .expect("prepared IDWT output dimensions must fit Metal f32 storage"), + )), + _ => None, + }) + .collect() +} diff --git a/crates/j2k-metal/src/compute/tests/runtime.rs b/crates/j2k-metal/src/compute/tests/runtime.rs new file mode 100644 index 00000000..ca448218 --- /dev/null +++ b/crates/j2k-metal/src/compute/tests/runtime.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::{ + checked_metal_surface_len, decode_scaled_to_surface, j2k_pack_kernel_name_for, + j2k_pack_scale_arrays, output_shape_for, reset_shared_buffer_pool_misses_for_test, + runtime_initialization_error, shared_buffer_pool_misses_for_test, with_runtime_for_device, + MetalRuntime, MetalSupportError, +}; +use j2k_core::PixelFormat; +use j2k_native::{ + encode_htj2k, ColorSpace as NativeColorSpace, DecodeSettings, EncodeOptions, Image, +}; +use metal::{foreign_types::ForeignType, Device}; + +pub(super) fn should_run_metal_runtime() -> bool { + j2k_test_support::metal_runtime_gate(module_path!()) +} + +#[test] +fn rgb16_with_alpha_is_rejected() { + if !should_run_metal_runtime() { + return; + } + + let runtime = MetalRuntime::new().expect("Metal runtime"); + let result = output_shape_for( + &NativeColorSpace::RGB, + true, + 4, + PixelFormat::Rgb16, + &runtime, + ); + assert!(result.is_err(), "RGBA input must not silently map to Rgb16"); +} + +#[test] +fn runtime_initialization_error_classifies_null_queue_as_unavailable() { + assert!(matches!( + runtime_initialization_error(&MetalSupportError::CommandQueueUnavailable), + crate::Error::MetalUnavailable + )); +} + +#[test] +fn checked_metal_surface_len_accepts_valid_surface() { + assert_eq!( + checked_metal_surface_len((13, 7), PixelFormat::Rgb8.bytes_per_pixel(), "test surface") + .unwrap(), + (39, 273) + ); +} + +#[test] +fn checked_metal_surface_len_reports_overflow_as_metal_error() { + let error = checked_metal_surface_len((u32::MAX, 1), usize::MAX, "test surface").unwrap_err(); + + assert!( + matches!(error, crate::Error::MetalKernel { message } if message.contains("surface row byte count")) + ); +} + +#[test] +fn two_d_threads_per_group_clamps_empty_pipeline_limits() { + let threads = j2k_metal_support::two_d_threads_per_group(0, 0); + + assert_eq!((threads.width, threads.height, threads.depth), (1, 1, 1)); +} + +#[test] +fn one_d_threads_per_group_clamps_empty_pipeline_width() { + let threads = j2k_metal_support::one_d_threads_per_group(0); + + assert_eq!((threads.width, threads.height, threads.depth), (1, 1, 1)); +} + +#[test] +fn two_d_threads_per_group_preserves_simd_width_and_derives_height() { + let threads = j2k_metal_support::two_d_threads_per_group(32, 1024); + + assert_eq!((threads.width, threads.height, threads.depth), (32, 32, 1)); +} + +#[test] +fn with_runtime_for_device_scopes_runtime_to_requested_device() { + if !should_run_metal_runtime() { + return; + } + + let Some(device) = Device::system_default() else { + j2k_test_support::metal_device_unavailable_is_skip(module_path!()); + return; + }; + + let runtime_device = + with_runtime_for_device(&device, |runtime| Ok(runtime.device.as_ptr() as usize)) + .expect("Metal runtime"); + + assert_eq!(runtime_device, device.as_ptr() as usize); +} + +#[test] +fn runtime_reuses_recycled_shared_buffers() -> Result<(), crate::Error> { + if !should_run_metal_runtime() { + return Ok(()); + } + + let Some(device) = Device::system_default() else { + j2k_test_support::metal_device_unavailable_is_skip(module_path!()); + return Ok(()); + }; + let runtime = MetalRuntime::new_with_device(&device).expect("Metal runtime"); + + reset_shared_buffer_pool_misses_for_test(); + let first = runtime.take_shared_buffer(64)?; + runtime.recycle_shared_buffer(first)?; + let _second = runtime.take_shared_buffer(64)?; + + assert_eq!( + shared_buffer_pool_misses_for_test(), + 1, + "recycled shared metadata buffers should be reused instead of allocating again" + ); + Ok(()) +} + +#[test] +fn j2k_pack_selects_specialized_kernels_for_wsi_formats() { + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::Gray, false, 1, PixelFormat::Gray8), + Some("j2k_pack_gray8") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgb8), + Some("j2k_pack_rgb8") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgba8), + Some("j2k_pack_rgb_opaque_rgba8") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, true, 4, PixelFormat::Rgba8), + Some("j2k_pack_rgba8") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::Gray, false, 1, PixelFormat::Gray16), + Some("j2k_pack_gray16") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgb16), + Some("j2k_pack_rgb16") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, true, 4, PixelFormat::Rgb16), + None, + "RGBA input must not silently drop alpha when packing RGB16" + ); +} + +#[test] +fn j2k_pack_precomputes_scale_factors_on_cpu() { + let (max_values, u8_scales, u16_scales) = j2k_pack_scale_arrays([8, 12, 16, 0]); + + assert_f32_near(max_values[0], 255.0); + assert_f32_near(max_values[1], 4095.0); + assert_f32_near(max_values[2], 65_535.0); + assert_f32_near(max_values[3], 1.0); + assert_f32_near(u8_scales[0], 1.0); + assert_f32_near(u8_scales[1], 255.0 / 4095.0); + assert_f32_near(u16_scales[0], 257.0); + assert_f32_near(u16_scales[1], 1.0); + assert_f32_near(u16_scales[2], 1.0); + assert_f32_near(u16_scales[3], 65_535.0); +} + +fn assert_f32_near(actual: f32, expected: f32) { + assert!( + (actual - expected).abs() <= f32::EPSILON, + "expected {actual} to be within f32 epsilon of {expected}" + ); +} + +#[test] +fn scaled_htj2k_decode_runs_through_metal_compute_path() { + if !should_run_metal_runtime() { + return; + } + + let pixels: Vec = (0..16).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode ht gray8"); + + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((2, 2)), + ..DecodeSettings::default() + }, + ) + .expect("image"); + let host = image.decode().expect("host scaled decode"); + + let surface = decode_scaled_to_surface( + &bytes, + (4, 4), + PixelFormat::Gray8, + j2k_core::Downscale::Half, + ) + .expect("metal scaled decode"); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} diff --git a/crates/j2k-metal/src/compute/tier1_encode.rs b/crates/j2k-metal/src/compute/tier1_encode.rs index 278e3789..5a0a75a1 100644 --- a/crates/j2k-metal/src/compute/tier1_encode.rs +++ b/crates/j2k-metal/src/compute/tier1_encode.rs @@ -1,19 +1,27 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use std::mem::size_of; + +use j2k_metal_support::{dispatch_1d_pipeline, dispatch_single_thread}; + +use crate::profile_env::{label_command_buffer, label_compute_encoder}; + +use super::abi::{ + J2kClassicEncodeBatchJob, J2kClassicEncodeParams, J2kClassicEncodeStatus, J2kClassicSegment, + J2kHtEncodeBatchJob, J2kHtEncodeParams, J2kHtEncodeStatus, J2kPacketEncodeStatus, + J2K_ENCODE_STATUS_FAIL, J2K_ENCODE_STATUS_OK, J2K_ENCODE_STATUS_UNSUPPORTED, +}; +use super::resident_tier1::{J2kResidentLosslessHtCodeBlocks, J2kResidentLosslessTier1CodeBlocks}; use super::{ checked_buffer_read, checked_buffer_slice, checked_buffer_slice_at, classic_encode_code_blocks_pipeline, classic_encode_output_capacity, classic_encode_segment_capacity, classic_style_flags, commit_and_wait_metal, - copied_slice_buffer, dispatch_1d_pipeline, dispatch_single_thread, ht_encode_output_capacity, - label_command_buffer, label_compute_encoder, new_blit_command_encoder, new_command_buffer, - new_compute_command_encoder, new_private_buffer, new_shared_buffer, size_of, with_runtime, + copied_slice_buffer, ht_encode_output_capacity, new_blit_command_encoder, new_command_buffer, + new_compute_command_encoder, new_private_buffer, new_shared_buffer, with_runtime, with_runtime_for_session, zeroed_shared_buffer, Buffer, EncodedHtJ2kCodeBlock, - EncodedJ2kCodeBlock, Error, J2kClassicEncodeBatchJob, J2kClassicEncodeParams, - J2kClassicEncodeStatus, J2kClassicSegment, J2kCodeBlockSegment, J2kHtCodeBlockEncodeJob, - J2kHtEncodeBatchJob, J2kHtEncodeParams, J2kHtEncodeStatus, J2kLosslessDeviceCodeBlock, - J2kPacketEncodeStatus, J2kPreparedLosslessDeviceCodeBlocks, J2kResidentLosslessHtCodeBlocks, - J2kResidentLosslessTier1CodeBlocks, J2kTier1CodeBlockEncodeJob, MetalRuntime, - J2K_ENCODE_STATUS_FAIL, J2K_ENCODE_STATUS_OK, J2K_ENCODE_STATUS_UNSUPPORTED, + EncodedJ2kCodeBlock, Error, J2kCodeBlockSegment, J2kHtCodeBlockEncodeJob, + J2kLosslessDeviceCodeBlock, J2kPreparedLosslessDeviceCodeBlocks, J2kTier1CodeBlockEncodeJob, + MetalRuntime, }; fn checked_type_buffer_bytes(count: usize, context: &'static str) -> Result { diff --git a/crates/j2k-metal/src/compute/tier1_encode/test_support.rs b/crates/j2k-metal/src/compute/tier1_encode/test_support.rs index 8ce649e8..3e042439 100644 --- a/crates/j2k-metal/src/compute/tier1_encode/test_support.rs +++ b/crates/j2k-metal/src/compute/tier1_encode/test_support.rs @@ -1,13 +1,16 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::super::{ - classic_tier1_gpu_token_pack_supported, dispatch_classic_tier1_split_token_emit_for_cpu_pack, +use super::super::abi::{J2kClassicTier1SymbolPlanCounters, J2kClassicTier1TokenSegment}; +use super::super::resident_tier1::{ + dispatch_classic_tier1_split_token_emit_for_cpu_pack, dispatch_classic_tier1_split_token_emit_for_gpu_pack, dispatch_classic_tier1_split_token_pack_from_gpu_tokens, dispatch_classic_tier1_token_emit_for_gpu_pack, dispatch_classic_tier1_token_pack_from_gpu_tokens, - pack_j2k_code_block_scalar_from_tier1_tokens, J2kClassicTier1SymbolPlanCounters, - J2kClassicTier1TokenSegment, J2kTier1TokenSegment, +}; +use super::super::{ + classic_tier1_gpu_token_pack_supported, pack_j2k_code_block_scalar_from_tier1_tokens, + J2kTier1TokenSegment, }; use super::*; diff --git a/crates/j2k-metal/src/decoder.rs b/crates/j2k-metal/src/decoder.rs index 08ae5e1c..d6222347 100644 --- a/crates/j2k-metal/src/decoder.rs +++ b/crates/j2k-metal/src/decoder.rs @@ -9,6 +9,15 @@ mod request; mod routes; mod surface; +#[cfg(target_os = "macos")] +use std::sync::Arc; + +#[cfg(target_os = "macos")] +use j2k_core::PixelFormat; + +#[cfg(target_os = "macos")] +use crate::{Error, MetalBackendSession, MetalDirectFallbackReason}; + pub use adapters::Codec; pub use core::J2kDecoder; pub use request::{ @@ -17,9 +26,25 @@ pub use request::{ #[cfg(target_os = "macos")] pub(crate) use direct_paths::{ - decode_full_color_batch_direct_to_device, decode_full_grayscale_batch_direct_to_device, - is_direct_runtime_fallback_error, + decode_full_color_batch_direct_to_device_routed, + decode_full_grayscale_batch_direct_to_device_routed, is_direct_runtime_fallback_error, }; +#[cfg(target_os = "macos")] +pub(crate) fn prepare_full_grayscale_direct_plan_with_session( + input: &[u8], + fmt: PixelFormat, + session: &MetalBackendSession, +) -> Result, Error> { + let mut decoder = J2kDecoder::new(input)?; + decoder + .ensure_prepared_direct_gray_plan_with_session(fmt, session)? + .ok_or_else(|| Error::MetalDirectFallback { + message: format!( + "explicit J2K MetalDirect destination could not build a full grayscale plan; fmt={fmt:?}" + ), + reason: MetalDirectFallbackReason::UnsupportedPlan, + }) +} #[cfg(test)] mod tests; diff --git a/crates/j2k-metal/src/decoder/direct_paths.rs b/crates/j2k-metal/src/decoder/direct_paths.rs index 568261d2..2588018f 100644 --- a/crates/j2k-metal/src/decoder/direct_paths.rs +++ b/crates/j2k-metal/src/decoder/direct_paths.rs @@ -9,7 +9,6 @@ use j2k_core::{BackendRequest, PixelFormat}; use j2k_native::{ DecodeSettings as NativeDecodeSettings, Image as NativeImage, J2kDirectGrayscalePlan, }; -#[cfg(target_os = "macos")] use metal::Device; #[cfg(target_os = "macos")] @@ -20,7 +19,7 @@ use crate::direct; #[cfg(target_os = "macos")] use crate::error::{adapter_backend_error, native_decode_error}; #[cfg(target_os = "macos")] -use crate::session::{ +use crate::session::direct_plan_cache::{ cached_session_direct_color_plan, cached_session_direct_gray_plan, direct_gray_plan_cache_key, direct_plan_cache_key, store_session_direct_color_plan, store_session_direct_gray_plan, }; @@ -42,7 +41,7 @@ macro_rules! define_ensure_prepared_direct_plan { prepare: $prepare:path ) => { #[cfg(target_os = "macos")] - fn $with_session( + pub(super) fn $with_session( &mut self, fmt: PixelFormat, session: &MetalBackendSession, @@ -83,7 +82,13 @@ macro_rules! define_ensure_prepared_direct_plan { } Err(error) => return Err(native_decode_error(error)), }; - let prepared = Arc::new($prepare(plan.as_ref())?); + let prepared = if let Some((session, _)) = &session_cache { + Arc::new(crate::compute::with_runtime_for_session(session, |_| { + $prepare(plan.as_ref()) + })?) + } else { + Arc::new($prepare(plan.as_ref())?) + }; if let Some((session, fmt)) = session_cache { let cache_key = $cache_key(self.bytes, fmt); $store(session, cache_key, plan.clone(), prepared.clone())?; @@ -164,7 +169,7 @@ impl J2kDecoder<'_> { }; return match crate::compute::execute_prepared_direct_color_plan(plan, fmt) { Ok(surface) => Ok(Some(surface)), - Err(error) if is_direct_color_runtime_fallback_error(&error) => Ok(None), + Err(error) if is_direct_runtime_fallback_error(&error) => Ok(None), Err(error) => Err(error), }; } @@ -206,7 +211,7 @@ impl J2kDecoder<'_> { session.device_handle(), ) { Ok(surface) => Ok(Some(surface)), - Err(error) if is_direct_color_runtime_fallback_error(&error) => Ok(None), + Err(error) if is_direct_runtime_fallback_error(&error) => Ok(None), Err(error) => Err(error), }; } @@ -281,34 +286,51 @@ impl J2kDecoder<'_> { &mut self, fmt: PixelFormat, count: usize, + ) -> Result, Error> { + self.decode_repeated_grayscale_direct_to_device_routed(fmt, count, None) + } + + #[cfg(target_os = "macos")] + #[doc(hidden)] + pub fn decode_repeated_grayscale_direct_to_device_with_session( + &mut self, + fmt: PixelFormat, + count: usize, + session: &MetalBackendSession, + ) -> Result, Error> { + self.decode_repeated_grayscale_direct_to_device_routed(fmt, count, Some(session)) + } + + #[cfg(target_os = "macos")] + pub(crate) fn decode_repeated_grayscale_direct_to_device_routed( + &mut self, + fmt: PixelFormat, + count: usize, + session: Option<&MetalBackendSession>, ) -> Result, Error> { if count == 0 { return Ok(Vec::new()); } - if self.native_direct_gray_plan.is_none() { - self.ensure_native_image()?; - let (Some(image), native_context) = - (self.native_image.as_ref(), &mut self.native_context) - else { - return Err(Error::Decode(adapter_backend_error( - "native image cache missing".to_string(), - ))); - }; - let plan = Arc::new( - image - .build_direct_grayscale_plan_with_context(native_context) - .map_err(native_decode_error)?, - ); - let prepared = Arc::new(crate::compute::prepare_direct_grayscale_plan( - plan.as_ref(), - )?); - self.native_direct_gray_plan = Some(plan); - self.native_prepared_direct_gray_plan = Some(prepared); - } - let Some(plan) = self.native_prepared_direct_gray_plan.as_ref() else { - return Ok(Vec::new()); + let plan = match session { + Some(session) => self.ensure_prepared_direct_gray_plan_with_session(fmt, session)?, + None => self.ensure_prepared_direct_gray_plan()?, }; - crate::compute::execute_repeated_prepared_direct_grayscale_plan(plan, fmt, count) + let Some(plan) = plan else { + return Err(Error::MetalDirectFallback { + message: format!( + "explicit J2K MetalDirect repeated batch does not support {fmt:?}" + ), + reason: MetalDirectFallbackReason::UnsupportedPlan, + }); + }; + match session { + Some(session) => crate::compute::with_runtime_for_session(session, |_| { + crate::compute::execute_repeated_prepared_direct_grayscale_plan(&plan, fmt, count) + }), + None => { + crate::compute::execute_repeated_prepared_direct_grayscale_plan(&plan, fmt, count) + } + } } #[cfg(target_os = "macos")] @@ -317,11 +339,38 @@ impl J2kDecoder<'_> { &mut self, fmt: PixelFormat, count: usize, + ) -> Result, Error> { + self.decode_repeated_color_direct_to_device_routed(fmt, count, None) + } + + #[cfg(target_os = "macos")] + #[doc(hidden)] + pub fn decode_repeated_color_direct_to_device_with_session( + &mut self, + fmt: PixelFormat, + count: usize, + session: &MetalBackendSession, + ) -> Result, Error> { + self.decode_repeated_color_direct_to_device_routed(fmt, count, Some(session)) + } + + #[cfg(target_os = "macos")] + pub(crate) fn decode_repeated_color_direct_to_device_routed( + &mut self, + fmt: PixelFormat, + count: usize, + session: Option<&MetalBackendSession>, ) -> Result, Error> { if count == 0 { return Ok(Vec::new()); } - let surface = self.decode_to_surface_impl(fmt, BackendRequest::Metal)?; + let surface = match session { + Some(session) => self.decode_request_to_device_with_session( + crate::MetalDecodeRequest::full(fmt, BackendRequest::Metal), + session, + )?, + None => self.decode_to_surface_impl(fmt, BackendRequest::Metal)?, + }; let mut budget = crate::batch_allocation::BatchMetadataBudget::new( "J2K Metal repeated color surface collection", ); @@ -380,20 +429,16 @@ impl J2kDecoder<'_> { } } -#[cfg(target_os = "macos")] -fn is_direct_color_runtime_fallback_error(error: &Error) -> bool { - is_direct_runtime_fallback_error(error) -} - #[cfg(target_os = "macos")] pub(crate) fn is_direct_runtime_fallback_error(error: &Error) -> bool { error.is_direct_fallback() } #[cfg(target_os = "macos")] -pub(crate) fn decode_full_grayscale_batch_direct_to_device( +pub(crate) fn decode_full_grayscale_batch_direct_to_device_routed( inputs: &[Arc<[u8]>], fmt: PixelFormat, + session: Option<&MetalBackendSession>, ) -> Result, Error> { if inputs.is_empty() { return Ok(Vec::new()); @@ -409,7 +454,11 @@ pub(crate) fn decode_full_grayscale_batch_direct_to_device( let mut plans = budget.try_vec(inputs.len(), "J2K Metal direct grayscale plans")?; for input in inputs { let mut decoder = J2kDecoder::new(input.as_ref())?; - let Some(plan) = decoder.ensure_prepared_direct_gray_plan()? else { + let plan = match session { + Some(session) => decoder.ensure_prepared_direct_gray_plan_with_session(fmt, session)?, + None => decoder.ensure_prepared_direct_gray_plan()?, + }; + let Some(plan) = plan else { return Err(Error::MetalDirectFallback { message: format!( "explicit J2K MetalDirect batch currently supports full grayscale Gray8/Gray16 only; fmt={fmt:?}" @@ -419,13 +468,19 @@ pub(crate) fn decode_full_grayscale_batch_direct_to_device( }; plans.push(plan); } - crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) + match session { + Some(session) => crate::compute::with_runtime_for_session(session, |_| { + crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) + }), + None => crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt), + } } #[cfg(target_os = "macos")] -pub(crate) fn decode_full_color_batch_direct_to_device( +pub(crate) fn decode_full_color_batch_direct_to_device_routed( inputs: &[Arc<[u8]>], fmt: PixelFormat, + session: Option<&MetalBackendSession>, ) -> Result, Error> { if inputs.is_empty() { return Ok(Vec::new()); @@ -444,7 +499,13 @@ pub(crate) fn decode_full_color_batch_direct_to_device( let mut plans = budget.try_vec(inputs.len(), "J2K Metal direct color plans")?; for input in inputs { let mut decoder = J2kDecoder::new(input.as_ref())?; - let Some(plan) = decoder.ensure_prepared_direct_color_plan()? else { + let plan = match session { + Some(session) => { + decoder.ensure_prepared_direct_color_plan_with_session(fmt, session)? + } + None => decoder.ensure_prepared_direct_color_plan()?, + }; + let Some(plan) = plan else { return Err(Error::MetalDirectFallback { message: format!( "explicit J2K MetalDirect batch currently supports full RGB color only; fmt={fmt:?}" @@ -454,9 +515,15 @@ pub(crate) fn decode_full_color_batch_direct_to_device( }; plans.push(plan); } - match crate::compute::execute_prepared_direct_color_plan_batch(&plans, fmt) { + let result = match session { + Some(session) => crate::compute::with_runtime_for_session(session, |_| { + crate::compute::execute_prepared_direct_color_plan_batch(&plans, fmt) + }), + None => crate::compute::execute_prepared_direct_color_plan_batch(&plans, fmt), + }; + match result { Ok(surfaces) => Ok(surfaces), - Err(error) if is_direct_color_runtime_fallback_error(&error) => { + Err(error) if is_direct_runtime_fallback_error(&error) => { Err(Error::UnsupportedMetalRequest { reason: CPU_STAGED_METAL_REQUIRES_EXPLICIT_API, }) diff --git a/crates/j2k-metal/src/decoder/tests.rs b/crates/j2k-metal/src/decoder/tests.rs index 2cf5777c..86105903 100644 --- a/crates/j2k-metal/src/decoder/tests.rs +++ b/crates/j2k-metal/src/decoder/tests.rs @@ -157,8 +157,58 @@ fn metal_backend_sessions_own_distinct_direct_plan_caches() { let second = MetalBackendSession::new(device); assert_ne!( - first.direct_cache_ids_for_test(), - second.direct_cache_ids_for_test() + crate::session::direct_plan_cache::direct_cache_ids_for_test(&first), + crate::session::direct_plan_cache::direct_cache_ids_for_test(&second) + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn fresh_direct_plan_preparation_uses_the_explicit_session_runtime() { + if !should_run_metal_runtime() { + return; + } + + let Some(device) = Device::system_default() else { + j2k_test_support::metal_device_unavailable_is_skip(module_path!()); + return; + }; + let pixels = j2k_test_support::gradient_u8(32, 32, 1); + let bytes = j2k_native::encode( + &pixels, + 32, + 32, + 1, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode classic grayscale session-runtime fixture"); + let session = MetalBackendSession::new(device.clone()); + let session_runtime = session.runtime().expect("explicit session runtime"); + + crate::compute::reset_direct_tier1_input_buffer_prepares_for_test(); + crate::compute::with_isolated_runtime_for_device_for_test(&device, || { + let mut decoder = J2kDecoder::new(&bytes)?; + let prepared = decoder + .ensure_prepared_direct_gray_plan_with_session(PixelFormat::Gray8, &session)?; + assert!(prepared.is_some()); + Ok(()) + }) + .expect("prepare direct plan with explicit session"); + + assert!( + crate::compute::direct_tier1_input_buffer_prepares_for_test() > 0, + "fixture must allocate classic Tier-1 input buffers" + ); + assert_eq!( + crate::compute::direct_tier1_input_buffer_runtime_for_test(), + Arc::as_ptr(&session_runtime).addr(), + "fresh cached buffers must be prepared by the explicit session runtime" ); } diff --git a/crates/j2k-metal/src/error.rs b/crates/j2k-metal/src/error.rs index 99273ae9..d0e4df3b 100644 --- a/crates/j2k-metal/src/error.rs +++ b/crates/j2k-metal/src/error.rs @@ -172,6 +172,23 @@ impl MetalKernelRetryClass { } impl Error { + /// Whether this failure invalidates retained Metal session state or an + /// operation-wide ownership boundary. + #[doc(hidden)] + #[must_use] + pub fn session_is_unusable(&self) -> bool { + matches!( + self, + Self::BatchInfrastructure(_) + | Self::PreparedPlanCacheAllocation { .. } + | Self::PreparedPlanCacheInvariant { .. } + | Self::MetalUnavailable + | Self::MetalRuntime { .. } + | Self::MetalStatePoisoned { .. } + | Self::MetalStateInvariant { .. } + ) + } + #[cfg(target_os = "macos")] pub(crate) fn is_conservative_retry_candidate(&self, requested: MetalKernelRetryClass) -> bool { match self { @@ -434,4 +451,26 @@ mod tests { assert!(matches!(error, Error::MetalUnavailable)); assert!(error.is_unsupported()); } + + #[test] + fn persistent_session_classification_is_owned_by_metal_error() { + assert!(Error::MetalUnavailable.session_is_unusable()); + assert!(Error::MetalRuntime { + message: "test runtime failure".to_string(), + } + .session_is_unusable()); + assert!(Error::MetalStateInvariant { + state: "test state", + reason: "test invariant", + } + .session_is_unusable()); + assert!(!Error::UnsupportedMetalRequest { + reason: "test unsupported request", + } + .session_is_unusable()); + assert!(!Error::MetalKernel { + message: "test group status".to_string(), + } + .session_is_unusable()); + } } diff --git a/crates/j2k-metal/src/ht.rs b/crates/j2k-metal/src/ht.rs index b9339268..5d49e32d 100644 --- a/crates/j2k-metal/src/ht.rs +++ b/crates/j2k-metal/src/ht.rs @@ -512,7 +512,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_ht_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); - let mut actual = vec![0.0f32; job.output_len()]; + let mut actual = vec![13_579.0f32; job.output_len()]; compute::decode_ht_cleanup_code_block(job.as_job(), &mut actual).expect("metal decode"); assert_eq!(actual, expected); diff --git a/crates/j2k-metal/src/ht_cleanup.metal b/crates/j2k-metal/src/ht_cleanup.metal index d9d6f099..55af93b7 100644 --- a/crates/j2k-metal/src/ht_cleanup.metal +++ b/crates/j2k-metal/src/ht_cleanup.metal @@ -194,10 +194,12 @@ inline ForwardBitReader forward_reader_new(device const uchar *data, uint data_l inline void forward_reader_fill(thread ForwardBitReader &reader) { while (reader.bits <= 32u) { - const uchar byte = reader.pos < reader.data_len ? reader.data[reader.pos++] : reader.pad; - reader.tmp |= (ulong(byte) << reader.bits); + const uchar raw_byte = + reader.pos < reader.data_len ? reader.data[reader.pos++] : reader.pad; + const uchar value = reader.unstuff ? raw_byte & uchar(0x7F) : raw_byte; + reader.tmp |= (ulong(value) << reader.bits); reader.bits += 8u - uint(reader.unstuff); - reader.unstuff = byte == uchar(0xFF); + reader.unstuff = raw_byte == uchar(0xFF); } } @@ -257,15 +259,18 @@ inline ReverseBitReader reverse_reader_new_mrp( inline void reverse_reader_fill(thread ReverseBitReader &reader) { while (reader.bits <= 32u) { - const uchar byte = reader.remaining > 0u ? reader.data[reader.pos] : uchar(0u); + const uchar raw_byte = reader.remaining > 0u ? reader.data[reader.pos] : uchar(0u); if (reader.remaining > 0u) { reader.pos -= 1; reader.remaining -= 1u; } - const uint d_bits = 8u - uint(reader.unstuff && (byte & uchar(0x7F)) == uchar(0x7F)); - reader.tmp |= (ulong(byte) << reader.bits); + const bool stuffed = + reader.unstuff && (raw_byte & uchar(0x7F)) == uchar(0x7F); + const uint d_bits = 8u - uint(stuffed); + const uchar value = stuffed ? raw_byte & uchar(0x7F) : raw_byte; + reader.tmp |= (ulong(value) << reader.bits); reader.bits += d_bits; - reader.unstuff = byte > uchar(0x8F); + reader.unstuff = raw_byte > uchar(0x8F); } } @@ -331,7 +336,7 @@ inline void decode_mag_sgn_sample_with_vn( value |= (v_n + 2u) << (p - 1u); } -inline void decode_ht_cleanup_impl( +inline void decode_ht_cleanup_common( device const uchar *coded_data, device uint *decoded_data, J2kHtCleanupParams params, @@ -339,10 +344,16 @@ inline void decode_ht_cleanup_impl( constant ushort *vlc_table1, constant ushort *uvlc_table0, constant ushort *uvlc_table1, - device J2kHtStatus *status + device J2kHtStatus *status, + thread ushort *scratch, + thread uint *v_n_scratch ) { set_ht_status(status, J2K_HT_STATUS_OK, 0u); + if (params.number_of_coding_passes > 3u) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 3u); + return; + } uint num_passes = params.number_of_coding_passes; if (num_passes > 1u && params.refinement_length == 0u) { num_passes = 1u; @@ -360,7 +371,7 @@ inline void decode_ht_cleanup_impl( set_ht_status(status, J2K_HT_STATUS_FAIL, 2u); return; } - if (num_passes > 3u || params.missing_msbs > 30u || params.missing_msbs == 30u) { + if (params.missing_msbs > 30u || params.missing_msbs == 30u) { set_ht_status(status, J2K_HT_STATUS_FAIL, 3u); return; } @@ -390,9 +401,6 @@ inline void decode_ht_cleanup_impl( return; } - thread ushort scratch[J2K_HT_MAX_SCRATCH]; - thread uint v_n_scratch[J2K_HT_MAX_VN]; - { thread MelDecoder mel = mel_decoder_new(coded_data, lcup, scup); thread ReverseBitReader vlc = reverse_reader_new_vlc(coded_data, lcup, scup); @@ -668,19 +676,35 @@ inline void decode_ht_cleanup_impl( } } - if (num_passes > 1u) { - const uint sigma_rows = ((height + 3u) / 4u) + 1u; - const uint mstr = ((((width + 3u) / 4u) + 2u + 7u) & ~7u); - const uint prev_row_len = ((width + 3u) / 4u) + 8u; - if (mstr > J2K_HT_MAX_MSTR || sigma_rows * mstr > J2K_HT_MAX_SIGMA) { - set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 15u); - return; - } - if (prev_row_len > J2K_HT_MAX_PREV_ROW_SIG) { - set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 16u); - return; - } +} + +inline bool decode_ht_refinement_impl( + device const uchar *coded_data, + device uint *decoded_data, + J2kHtCleanupParams params, + uint num_passes, + device J2kHtStatus *status, + thread const ushort *scratch +) { + const uint width = params.width; + const uint height = params.height; + const uint stride = params.output_stride; + const uint sstr = (width + 9u) & ~7u; + const uint lcup = params.cleanup_length; + const uint p = 30u - params.missing_msbs; + const uint sigma_rows = ((height + 3u) / 4u) + 1u; + const uint mstr = ((((width + 3u) / 4u) + 2u + 7u) & ~7u); + const uint prev_row_len = ((width + 3u) / 4u) + 8u; + if (mstr > J2K_HT_MAX_MSTR || sigma_rows * mstr > J2K_HT_MAX_SIGMA) { + set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 15u); + return false; + } + if (prev_row_len > J2K_HT_MAX_PREV_ROW_SIG) { + set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 16u); + return false; + } + { thread ushort sigma[J2K_HT_MAX_SIGMA]; thread ushort prev_row_sig[J2K_HT_MAX_PREV_ROW_SIG]; @@ -873,11 +897,8 @@ inline void decode_ht_cleanup_impl( forward_reader_advance(sigprop, cnt); } - const uint combined_sig = new_sig | cs; + const uint combined_sig = new_sig | (cs & 0xFFFFu); prev_row_sig[idx] = ushort(combined_sig); - if (idx + 1u < prev_row_len) { - prev_row_sig[idx + 1u] = ushort(combined_sig >> 16u); - } const uint combined = combined_sig; uint next_prev = combined_sig; @@ -959,7 +980,16 @@ inline void decode_ht_cleanup_impl( } } } + return true; +} +inline void convert_ht_cleanup_coefficients( + device uint *decoded_data, + J2kHtCleanupParams params +) { + const uint width = params.width; + const uint height = params.height; + const uint stride = params.output_stride; for (uint y = 0u; y < height; ++y) { uint row_offset = params.output_offset + y * stride; for (uint x = 0u; x < width; ++x) { @@ -973,6 +1003,63 @@ inline void decode_ht_cleanup_impl( } } +inline void decode_ht_cleanup_only_impl( + device const uchar *coded_data, + device uint *decoded_data, + J2kHtCleanupParams params, + constant ushort *vlc_table0, + constant ushort *vlc_table1, + constant ushort *uvlc_table0, + constant ushort *uvlc_table1, + device J2kHtStatus *status +) { + thread ushort scratch[J2K_HT_MAX_SCRATCH]; + thread uint v_n_scratch[J2K_HT_MAX_VN]; + decode_ht_cleanup_common( + coded_data, decoded_data, params, vlc_table0, vlc_table1, + uvlc_table0, uvlc_table1, status, scratch, v_n_scratch + ); + if (status->code != J2K_HT_STATUS_OK) { + return; + } + convert_ht_cleanup_coefficients(decoded_data, params); +} + +inline void decode_ht_cleanup_impl( + device const uchar *coded_data, + device uint *decoded_data, + J2kHtCleanupParams params, + constant ushort *vlc_table0, + constant ushort *vlc_table1, + constant ushort *uvlc_table0, + constant ushort *uvlc_table1, + device J2kHtStatus *status +) { + thread ushort scratch[J2K_HT_MAX_SCRATCH]; + thread uint v_n_scratch[J2K_HT_MAX_VN]; + decode_ht_cleanup_common( + coded_data, decoded_data, params, vlc_table0, vlc_table1, + uvlc_table0, uvlc_table1, status, scratch, v_n_scratch + ); + if (status->code != J2K_HT_STATUS_OK) { + return; + } + + uint num_passes = params.number_of_coding_passes; + if (num_passes > 1u && params.refinement_length == 0u) { + num_passes = 1u; + } + if (params.missing_msbs == 29u && num_passes > 1u) { + num_passes = 1u; + } + if (num_passes > 1u && !decode_ht_refinement_impl( + coded_data, decoded_data, params, num_passes, status, scratch + )) { + return; + } + convert_ht_cleanup_coefficients(decoded_data, params); +} + kernel void j2k_decode_ht_cleanup( device const uchar *coded_data [[buffer(0)]], device uint *decoded_data [[buffer(1)]], @@ -1039,24 +1126,11 @@ kernel void j2k_decode_ht_cleanup_batched( ); } -kernel void j2k_decode_ht_cleanup_repeated_batched( - device const uchar *coded_data [[buffer(0)]], - device uint *decoded_data [[buffer(1)]], - constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], - constant J2kHtRepeatedBatchParams &repeated [[buffer(3)]], - constant ushort *vlc_table0 [[buffer(4)]], - constant ushort *vlc_table1 [[buffer(5)]], - constant ushort *uvlc_table0 [[buffer(6)]], - constant ushort *uvlc_table1 [[buffer(7)]], - device J2kHtStatus *status [[buffer(8)]], - uint2 gid [[thread_position_in_grid]] +inline J2kHtCleanupParams ht_cleanup_params_from_job( + const constant J2kHtCleanupBatchJob &job, + uint coding_passes, + uint output_offset ) { - if (gid.x >= repeated.job_count || gid.y >= repeated.batch_count) { - return; - } - - const constant J2kHtCleanupBatchJob &job = jobs[gid.x]; - J2kHtCleanupParams params; params.width = job.width; params.height = job.height; @@ -1065,11 +1139,120 @@ kernel void j2k_decode_ht_cleanup_repeated_batched( params.refinement_length = job.refinement_length; params.missing_msbs = job.missing_msbs; params.num_bitplanes = job.num_bitplanes; - params.number_of_coding_passes = job.number_of_coding_passes; + params.number_of_coding_passes = coding_passes; params.output_stride = job.output_stride; - params.output_offset = job.output_offset + gid.y * repeated.output_plane_len; + params.output_offset = output_offset; params.dequantization_step = job.dequantization_step; params.stripe_causal = job.stripe_causal; + return params; +} + +template +inline void decode_ht_cleanup_batched_specialized( + device const uchar *coded_data, + device uint *decoded_data, + constant J2kHtCleanupBatchJob *jobs, + constant ushort *vlc_table0, + constant ushort *vlc_table1, + constant ushort *uvlc_table0, + constant ushort *uvlc_table1, + device J2kHtStatus *status, + uint gid +) { + const constant J2kHtCleanupBatchJob &job = jobs[gid]; + + const J2kHtCleanupParams params = + ht_cleanup_params_from_job(job, CodingPasses, job.output_offset); + + decode_ht_cleanup_impl( + coded_data + job.coded_offset, + decoded_data, + params, + vlc_table0, + vlc_table1, + uvlc_table0, + uvlc_table1, + status + gid + ); +} + +kernel void j2k_decode_ht_cleanup_batched_cleanup_only( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + constant ushort *vlc_table0 [[buffer(3)]], + constant ushort *vlc_table1 [[buffer(4)]], + constant ushort *uvlc_table0 [[buffer(5)]], + constant ushort *uvlc_table1 [[buffer(6)]], + device J2kHtStatus *status [[buffer(7)]], + uint gid [[thread_position_in_grid]] +) { + const constant J2kHtCleanupBatchJob &job = jobs[gid]; + const J2kHtCleanupParams params = + ht_cleanup_params_from_job(job, 1u, job.output_offset); + decode_ht_cleanup_only_impl( + coded_data + job.coded_offset, decoded_data, params, + vlc_table0, vlc_table1, uvlc_table0, uvlc_table1, status + gid + ); +} + +kernel void j2k_decode_ht_cleanup_batched_sigprop( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + constant ushort *vlc_table0 [[buffer(3)]], + constant ushort *vlc_table1 [[buffer(4)]], + constant ushort *uvlc_table0 [[buffer(5)]], + constant ushort *uvlc_table1 [[buffer(6)]], + device J2kHtStatus *status [[buffer(7)]], + uint gid [[thread_position_in_grid]] +) { + decode_ht_cleanup_batched_specialized<2u>( + coded_data, decoded_data, jobs, vlc_table0, vlc_table1, + uvlc_table0, uvlc_table1, status, gid + ); +} + +kernel void j2k_decode_ht_cleanup_batched_magref( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + constant ushort *vlc_table0 [[buffer(3)]], + constant ushort *vlc_table1 [[buffer(4)]], + constant ushort *uvlc_table0 [[buffer(5)]], + constant ushort *uvlc_table1 [[buffer(6)]], + device J2kHtStatus *status [[buffer(7)]], + uint gid [[thread_position_in_grid]] +) { + decode_ht_cleanup_batched_specialized<3u>( + coded_data, decoded_data, jobs, vlc_table0, vlc_table1, + uvlc_table0, uvlc_table1, status, gid + ); +} + +template +inline void decode_ht_cleanup_repeated_batched_specialized( + device const uchar *coded_data, + device uint *decoded_data, + constant J2kHtCleanupBatchJob *jobs, + constant J2kHtRepeatedBatchParams &repeated, + constant ushort *vlc_table0, + constant ushort *vlc_table1, + constant ushort *uvlc_table0, + constant ushort *uvlc_table1, + device J2kHtStatus *status, + uint2 gid +) { + if (gid.x >= repeated.job_count || gid.y >= repeated.batch_count) { + return; + } + + const constant J2kHtCleanupBatchJob &job = jobs[gid.x]; + const J2kHtCleanupParams params = ht_cleanup_params_from_job( + job, + CodingPasses, + job.output_offset + gid.y * repeated.output_plane_len + ); decode_ht_cleanup_impl( coded_data + job.coded_offset, @@ -1082,3 +1265,67 @@ kernel void j2k_decode_ht_cleanup_repeated_batched( status + gid.y * repeated.job_count + gid.x ); } + +kernel void j2k_decode_ht_cleanup_repeated_batched_cleanup_only( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + constant J2kHtRepeatedBatchParams &repeated [[buffer(3)]], + constant ushort *vlc_table0 [[buffer(4)]], + constant ushort *vlc_table1 [[buffer(5)]], + constant ushort *uvlc_table0 [[buffer(6)]], + constant ushort *uvlc_table1 [[buffer(7)]], + device J2kHtStatus *status [[buffer(8)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= repeated.job_count || gid.y >= repeated.batch_count) { + return; + } + const constant J2kHtCleanupBatchJob &job = jobs[gid.x]; + const J2kHtCleanupParams params = ht_cleanup_params_from_job( + job, + 1u, + job.output_offset + gid.y * repeated.output_plane_len + ); + decode_ht_cleanup_only_impl( + coded_data + job.coded_offset, decoded_data, params, + vlc_table0, vlc_table1, uvlc_table0, uvlc_table1, + status + gid.y * repeated.job_count + gid.x + ); +} + +kernel void j2k_decode_ht_cleanup_repeated_batched_sigprop( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + constant J2kHtRepeatedBatchParams &repeated [[buffer(3)]], + constant ushort *vlc_table0 [[buffer(4)]], + constant ushort *vlc_table1 [[buffer(5)]], + constant ushort *uvlc_table0 [[buffer(6)]], + constant ushort *uvlc_table1 [[buffer(7)]], + device J2kHtStatus *status [[buffer(8)]], + uint2 gid [[thread_position_in_grid]] +) { + decode_ht_cleanup_repeated_batched_specialized<2u>( + coded_data, decoded_data, jobs, repeated, vlc_table0, vlc_table1, + uvlc_table0, uvlc_table1, status, gid + ); +} + +kernel void j2k_decode_ht_cleanup_repeated_batched_magref( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + constant J2kHtRepeatedBatchParams &repeated [[buffer(3)]], + constant ushort *vlc_table0 [[buffer(4)]], + constant ushort *vlc_table1 [[buffer(5)]], + constant ushort *uvlc_table0 [[buffer(6)]], + constant ushort *uvlc_table1 [[buffer(7)]], + device J2kHtStatus *status [[buffer(8)]], + uint2 gid [[thread_position_in_grid]] +) { + decode_ht_cleanup_repeated_batched_specialized<3u>( + coded_data, decoded_data, jobs, repeated, vlc_table0, vlc_table1, + uvlc_table0, uvlc_table1, status, gid + ); +} diff --git a/crates/j2k-metal/src/hybrid.rs b/crates/j2k-metal/src/hybrid.rs index 77d38a53..e762a36f 100644 --- a/crates/j2k-metal/src/hybrid.rs +++ b/crates/j2k-metal/src/hybrid.rs @@ -18,7 +18,11 @@ use crate::error::native_decode_error; use crate::profile_env::{ decode_profile_label, elapsed_since_us, MetalDirectProfileRow, MetalProfileFormat, }; -use crate::session::{prepared_plan_cache_error, PreparedPlanCache, PreparedPlanCacheKey}; +use crate::session::direct_plan_cache::{ + cached_session_region_scaled_color_plan, prepared_plan_cache_error, + store_session_region_scaled_color_plan, +}; +use crate::session::{PreparedPlanCache, PreparedPlanCacheKey}; use crate::{direct, Error, J2kDecoder, Surface}; pub(crate) const RGB_REGION_SCALED_METAL_DIRECT_UNSUPPORTED: &str = @@ -93,9 +97,7 @@ impl RegionScaledColorPlanCache<'_> { match self { Self::Uncached => Ok(None), Self::Global => cached_region_scaled_direct_color_plan(key), - Self::Session(session) => { - cached_region_scaled_direct_color_plan_with_session(session, key) - } + Self::Session(session) => cached_session_region_scaled_color_plan(session, key), } } @@ -112,7 +114,7 @@ impl RegionScaledColorPlanCache<'_> { } Self::Session(session) => { plan.disable_dynamic_cpu_tier1_retention()?; - store_region_scaled_direct_color_plan_with_session(session, key, plan) + store_session_region_scaled_color_plan(session, key, plan) } } } @@ -274,9 +276,10 @@ fn execute_region_scaled_direct_plan_with_device( } } -pub(crate) fn decode_region_scaled_grayscale_batch_direct_to_device( +pub(crate) fn decode_region_scaled_grayscale_batch_direct_to_device_routed( requests: &[(Arc<[u8]>, Rect, Downscale)], fmt: PixelFormat, + session: Option<&crate::MetalBackendSession>, ) -> Result, Error> { if requests.is_empty() { return Ok(Vec::new()); @@ -297,12 +300,26 @@ pub(crate) fn decode_region_scaled_grayscale_batch_direct_to_device( let plan = build_region_scaled_direct_gray_plan(input.as_ref(), *roi, *scale)?; plans.push(Arc::new(plan)); } - crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) + match session { + Some(session) => crate::compute::with_runtime_for_session(session, |_| { + crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) + }), + None => crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt), + } } +#[cfg(test)] pub(crate) fn decode_region_scaled_color_batch_direct_to_device( requests: &[(Arc<[u8]>, Rect, Downscale)], fmt: PixelFormat, +) -> Result, Error> { + decode_region_scaled_color_batch_direct_to_device_routed(requests, fmt, None) +} + +pub(crate) fn decode_region_scaled_color_batch_direct_to_device_routed( + requests: &[(Arc<[u8]>, Rect, Downscale)], + fmt: PixelFormat, + session: Option<&crate::MetalBackendSession>, ) -> Result, Error> { if requests.is_empty() { return Ok(Vec::new()); @@ -316,31 +333,65 @@ pub(crate) fn decode_region_scaled_color_batch_direct_to_device( }); } - if let Some((input, roi, scale)) = repeated_region_scaled_request(requests) { - let plan = build_region_scaled_direct_color_plan_cached(input.as_ref(), fmt, roi, scale)?; + let cache = match session { + Some(session) => RegionScaledColorPlanCache::Session(session), + None => RegionScaledColorPlanCache::Global, + }; + let plans = if let Some((input, roi, scale)) = repeated_region_scaled_request(requests) { + let plan = build_region_scaled_direct_color_plan_cached_with_cache( + input.as_ref(), + fmt, + roi, + scale, + cache, + )?; let mut budget = crate::batch_allocation::BatchMetadataBudget::new( "J2K Metal repeated region-scaled color batch plan", ); - let plans = budget.try_filled( + budget.try_filled( requests.len(), plan, "J2K Metal repeated region-scaled color plans", - )?; - return crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt); + )? + } else { + try_resolve_plans_in_order(requests, |(input, roi, scale)| { + build_region_scaled_direct_color_plan_cached_with_cache( + input.as_ref(), + fmt, + *roi, + *scale, + cache, + ) + })? + }; + match session { + Some(session) => crate::compute::with_runtime_for_session(session, |_| { + crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt) + }), + None => crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt), } - - let plans = try_resolve_plans_in_order(requests, |(input, roi, scale)| { - build_region_scaled_direct_color_plan_cached(input.as_ref(), fmt, *roi, *scale) - })?; - crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt) } +#[cfg(test)] pub(crate) fn decode_repeated_region_scaled_color_batch_direct_to_device( input: &[u8], roi: Rect, scale: Downscale, fmt: PixelFormat, count: usize, +) -> Result, Error> { + decode_repeated_region_scaled_color_batch_direct_to_device_routed( + input, roi, scale, fmt, count, None, + ) +} + +pub(crate) fn decode_repeated_region_scaled_color_batch_direct_to_device_routed( + input: &[u8], + roi: Rect, + scale: Downscale, + fmt: PixelFormat, + count: usize, + session: Option<&crate::MetalBackendSession>, ) -> Result, Error> { if count == 0 { return Err(Error::MetalKernel { @@ -357,12 +408,22 @@ pub(crate) fn decode_repeated_region_scaled_color_batch_direct_to_device( }); } - let plan = build_region_scaled_direct_color_plan_cached(input, fmt, roi, scale)?; + let cache = match session { + Some(session) => RegionScaledColorPlanCache::Session(session), + None => RegionScaledColorPlanCache::Global, + }; + let plan = + build_region_scaled_direct_color_plan_cached_with_cache(input, fmt, roi, scale, cache)?; let mut budget = crate::batch_allocation::BatchMetadataBudget::new( "J2K Metal repeated region-scaled color batch plan", ); let plans = budget.try_filled(count, plan, "J2K Metal repeated region-scaled color plans")?; - crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt) + match session { + Some(session) => crate::compute::with_runtime_for_session(session, |_| { + crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt) + }), + None => crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt), + } } fn repeated_region_scaled_request( @@ -471,21 +532,6 @@ fn build_region_scaled_direct_color_plan( Ok(prepared) } -fn build_region_scaled_direct_color_plan_cached( - input: &[u8], - fmt: PixelFormat, - roi: Rect, - scale: Downscale, -) -> Result, Error> { - build_region_scaled_direct_color_plan_cached_with_cache( - input, - fmt, - roi, - scale, - RegionScaledColorPlanCache::Global, - ) -} - fn build_region_scaled_direct_color_plan_cached_with_cache( input: &[u8], fmt: PixelFormat, @@ -503,35 +549,6 @@ fn build_region_scaled_direct_color_plan_cached_with_cache( Ok(plan) } -fn cached_region_scaled_direct_color_plan_with_session( - session: &crate::MetalBackendSession, - key: PreparedPlanCacheKey<'_>, -) -> Result>, Error> { - let mut guard = - session - .region_scaled_color_plan_cache - .lock() - .map_err(|_| Error::MetalStatePoisoned { - state: "session region-scaled color prepared-plan cache", - })?; - Ok(guard.get(key).cloned()) -} - -fn store_region_scaled_direct_color_plan_with_session( - session: &crate::MetalBackendSession, - key: PreparedPlanCacheKey<'_>, - plan: Arc, -) -> Result<(), Error> { - let mut guard = - session - .region_scaled_color_plan_cache - .lock() - .map_err(|_| Error::MetalStatePoisoned { - state: "session region-scaled color prepared-plan cache", - })?; - evict_one_region_scaled_color_plan_if_needed(&mut guard, key, plan) -} - fn cached_region_scaled_direct_color_plan( key: PreparedPlanCacheKey<'_>, ) -> Result>, Error> { diff --git a/crates/j2k-metal/src/idwt.metal b/crates/j2k-metal/src/idwt.metal index 196e69df..82b61edc 100644 --- a/crates/j2k-metal/src/idwt.metal +++ b/crates/j2k-metal/src/idwt.metal @@ -145,10 +145,11 @@ kernel void j2k_idwt_interleave( const uint global_x = params.x0 + params.output_x + gid.x; const uint global_y = params.y0 + params.output_y + gid.y; - const uint low_x_parity = params.x0 & 1u; - const uint low_y_parity = params.y0 & 1u; - const bool low_x = (global_x & 1u) == low_x_parity; - const bool low_y = (global_y & 1u) == low_y_parity; + // JPEG 2000 assigns low-pass samples to even reference-grid + // coordinates. The decomposition origin only determines which band is + // encountered first in this tile; it must not redefine that parity. + const bool low_x = (global_x & 1u) == 0u; + const bool low_y = (global_y & 1u) == 0u; const uint full_band_x = low_x ? low_index(global_x, params.x0) : high_index(global_x, params.x0); const uint full_band_y = low_y ? low_index(global_y, params.y0) : high_index(global_y, params.y0); const uint out_idx = gid.y * params.width + gid.x; @@ -195,10 +196,10 @@ kernel void j2k_idwt_interleave_batched( const uint global_x = params.x0 + params.output_x + gid.x; const uint global_y = params.y0 + params.output_y + gid.y; - const uint low_x_parity = params.x0 & 1u; - const uint low_y_parity = params.y0 & 1u; - const bool low_x = (global_x & 1u) == low_x_parity; - const bool low_y = (global_y & 1u) == low_y_parity; + // Low/high assignment is defined in the reference grid. For an odd + // tile origin, the first sample is therefore high-pass. + const bool low_x = (global_x & 1u) == 0u; + const bool low_y = (global_y & 1u) == 0u; const uint full_band_x = low_x ? low_index(global_x, params.x0) : high_index(global_x, params.x0); const uint full_band_y = low_y ? low_index(global_y, params.y0) : high_index(global_y, params.y0); const uint out_plane_len = params.width * params.height; diff --git a/crates/j2k-metal/src/lib.rs b/crates/j2k-metal/src/lib.rs index a226115a..0497325b 100644 --- a/crates/j2k-metal/src/lib.rs +++ b/crates/j2k-metal/src/lib.rs @@ -12,6 +12,7 @@ mod batch; mod batch_allocation; +mod batch_decoder; #[cfg(target_os = "macos")] mod buffer_pool; #[cfg(any(test, target_os = "macos"))] @@ -54,7 +55,19 @@ use j2k_metal_support::{ use metal::Buffer; pub use j2k_core::SurfaceResidency; +#[cfg(target_os = "macos")] +pub use j2k_metal_support::{MetalImageDestination, MetalImageLayout}; +#[doc(hidden)] +pub use self::batch::MetalSubmission; +pub use self::batch_decoder::{ + MetalBatchDecodeResult, MetalBatchDecoder, MetalBatchGroup, MetalBatchGroupCompletion, + MetalBatchGroupError, MetalBatchGroupParts, +}; +#[cfg(target_os = "macos")] +pub use self::batch_decoder::{ + MetalResidentBatch, SubmittedMetalGroupDecodeInto, SubmittedMetalPreparedBatch, +}; pub use self::decoder::{ Codec, DecodeOperation, DecodeRouteReport, DecodeSurfaceWithReport, J2kDecoder, MetalDecodeOp, MetalDecodeRequest, diff --git a/crates/j2k-metal/src/session.rs b/crates/j2k-metal/src/session.rs index 9504f42e..4c771e18 100644 --- a/crates/j2k-metal/src/session.rs +++ b/crates/j2k-metal/src/session.rs @@ -4,192 +4,96 @@ use std::sync::{Arc, Mutex}; #[cfg(target_os = "macos")] -use j2k_core::{BackendKind, PixelFormat}; +use j2k_core::BackendKind; #[cfg(target_os = "macos")] use j2k_metal_support::{MetalRuntimeSession, MetalSupportError}; #[cfg(target_os = "macos")] -use j2k_native::{J2kDirectColorPlan, J2kDirectGrayscalePlan}; -#[cfg(target_os = "macos")] -use metal::Device; +use metal::{ + foreign_types::{ForeignType, ForeignTypeRef}, + Device, +}; use crate::{batch, Error}; #[cfg(any(test, target_os = "macos"))] mod cache; +#[cfg(target_os = "macos")] +pub(crate) mod direct_plan_cache; #[cfg(target_os = "macos")] pub(crate) use cache::{ - PreparedPlanCache, PreparedPlanCacheError, PreparedPlanCacheKey, PreparedPlanCacheValue, + PreparedPlanCache, PreparedPlanCacheKey, PreparedPlanCacheValue, + PREPARED_PLAN_CACHE_MAX_DEVICE_BYTES, PREPARED_PLAN_CACHE_MAX_HOST_BYTES, }; #[cfg(target_os = "macos")] -#[derive(Clone)] -struct DirectGrayPlanCacheEntry { - plan: Arc, - prepared: Arc, -} - -#[cfg(target_os = "macos")] -#[derive(Clone)] -struct DirectColorPlanCacheEntry { - plan: Arc, - prepared: Arc, -} - -#[cfg(target_os = "macos")] -type CachedDirectGrayPlan = ( - Arc, - Arc, -); - -#[cfg(target_os = "macos")] -type CachedDirectColorPlan = ( - Arc, - Arc, -); - -#[cfg(target_os = "macos")] -impl cache::PreparedPlanCacheValue for DirectGrayPlanCacheEntry { - fn retained_cache_weight( - &self, - ) -> Result { - let prepared = self - .prepared - .retained_cache_bytes() - .map_err(cache::PreparedPlanCacheError::Invariant)?; - direct_plan_cache_weight( - self.plan.retained_allocation_bytes().map_err(|_| { - cache::PreparedPlanCacheError::Invariant( - "native grayscale direct-plan retained-byte accounting failed", - ) - })?, - core::mem::size_of::(), - prepared.host, - prepared.device, - core::mem::size_of::(), - ) - } -} - -#[cfg(target_os = "macos")] -impl cache::PreparedPlanCacheValue for DirectColorPlanCacheEntry { - fn retained_cache_weight( - &self, - ) -> Result { - let prepared = self - .prepared - .retained_cache_bytes() - .map_err(cache::PreparedPlanCacheError::Invariant)?; - direct_plan_cache_weight( - self.plan.retained_allocation_bytes().map_err(|_| { - cache::PreparedPlanCacheError::Invariant( - "native color direct-plan retained-byte accounting failed", - ) - })?, - core::mem::size_of::(), - prepared.host, - prepared.device, - core::mem::size_of::(), - ) - } +pub(crate) struct MetalConsumerEventTimeline { + pub(crate) event: Option, + pub(crate) next_value: u64, } #[cfg(target_os = "macos")] -impl cache::PreparedPlanCacheValue for Arc { - fn retained_cache_weight( - &self, - ) -> Result { - let prepared = self - .retained_cache_bytes() - .map_err(cache::PreparedPlanCacheError::Invariant)?; - let host_bytes = arc_owner_bytes( - core::mem::size_of::(), - prepared.host, - )?; - Ok(cache::PreparedPlanCacheWeight::new( - host_bytes, - prepared.device, - )) +impl MetalConsumerEventTimeline { + const fn new() -> Self { + Self { + event: None, + next_value: 0, + } } } -#[cfg(target_os = "macos")] -fn direct_plan_cache_weight( - native_nested_host: usize, - native_root_bytes: usize, - prepared_host_bytes: usize, - prepared_device_bytes: usize, - prepared_root_bytes: usize, -) -> Result { - let native_host = arc_owner_bytes(native_root_bytes, native_nested_host)?; - let prepared_host = arc_owner_bytes(prepared_root_bytes, prepared_host_bytes)?; - let host_bytes = - native_host - .checked_add(prepared_host) - .ok_or(cache::PreparedPlanCacheError::Invariant( - "native and prepared direct-plan cache weight overflow", - ))?; - Ok(cache::PreparedPlanCacheWeight::new( - host_bytes, - prepared_device_bytes, - )) -} - -#[cfg(target_os = "macos")] -fn arc_owner_bytes( - root_bytes: usize, - nested_bytes: usize, -) -> Result { - root_bytes - .checked_add(2 * core::mem::size_of::()) - .and_then(|bytes| bytes.checked_add(nested_bytes)) - .ok_or(cache::PreparedPlanCacheError::Invariant( - "prepared-plan Arc owner byte count overflow", - )) -} - -#[cfg(target_os = "macos")] -const DIRECT_PLAN_CACHE_CAP: usize = 128; - #[cfg(target_os = "macos")] #[derive(Clone)] /// Reusable Metal device session for J2K decode and encode submissions. pub struct MetalBackendSession { runtime_session: MetalRuntimeSession, MetalSupportError>, - direct_gray_plan_cache: Arc>>, - direct_color_plan_cache: Arc>>, - pub(crate) region_scaled_color_plan_cache: - Arc>>>, + command_queue: Option, + direct_plan_caches: direct_plan_cache::DirectPlanCaches, + consumer_event_timeline: Arc>, } #[cfg(target_os = "macos")] impl MetalBackendSession { /// Create a session bound to an existing Metal device. pub fn new(device: Device) -> Self { - Self::with_runtime_session(MetalRuntimeSession::new(device)) + Self::with_runtime_session(MetalRuntimeSession::new(device), None) + } + + /// Create a session that submits on an existing queue from the same device. + /// + /// Sharing the framework's exact queue provides producer and consumer + /// ordering without host waits or an additional event bridge. + pub fn with_command_queue( + device: Device, + command_queue: metal::CommandQueue, + ) -> Result { + if command_queue.device().as_ptr() != device.as_ptr() { + return Err(Error::UnsupportedMetalRequest { + reason: "command queue belongs to a different Metal device", + }); + } + Ok(Self::with_runtime_session( + MetalRuntimeSession::new(device), + Some(command_queue), + )) } fn with_runtime_session( runtime_session: MetalRuntimeSession, MetalSupportError>, + command_queue: Option, ) -> Self { Self { runtime_session, - direct_gray_plan_cache: Arc::new(Mutex::new(PreparedPlanCache::new( - DIRECT_PLAN_CACHE_CAP, - ))), - direct_color_plan_cache: Arc::new(Mutex::new(PreparedPlanCache::new( - DIRECT_PLAN_CACHE_CAP, - ))), - region_scaled_color_plan_cache: Arc::new(Mutex::new(PreparedPlanCache::new( - crate::hybrid::REGION_SCALED_COLOR_PLAN_CACHE_CAP, - ))), + command_queue, + direct_plan_caches: direct_plan_cache::DirectPlanCaches::new(), + consumer_event_timeline: Arc::new(Mutex::new(MetalConsumerEventTimeline::new())), } } /// Create a session from the system default Metal device. pub fn system_default() -> Result { MetalRuntimeSession::system_default() - .map(Self::with_runtime_session) + .map(|runtime_session| Self::with_runtime_session(runtime_session, None)) .map_err(|error| crate::compute::runtime_initialization_error(&error)) } @@ -203,8 +107,15 @@ impl MetalBackendSession { } pub(crate) fn runtime(&self) -> Result, Error> { - match self.runtime_session.get_or_init_runtime(|device| { - crate::compute::MetalRuntime::new_with_device(device).map(Arc::new) + let command_queue = self.command_queue.clone(); + match self.runtime_session.get_or_init_runtime(move |device| { + match command_queue { + Some(queue) => { + crate::compute::MetalRuntime::new_with_device_and_queue(device, queue) + } + None => crate::compute::MetalRuntime::new_with_device(device), + } + .map(Arc::new) }) { Ok(runtime) => Ok(runtime.clone()), Err(error) => Err(crate::compute::runtime_initialization_error(error)), @@ -221,13 +132,17 @@ impl MetalBackendSession { self.runtime()?.buffer_pool_diagnostics() } - #[cfg(test)] - pub(crate) fn direct_cache_ids_for_test(&self) -> (usize, usize, usize) { - ( - Arc::as_ptr(&self.direct_gray_plan_cache) as usize, - Arc::as_ptr(&self.direct_color_plan_cache) as usize, - Arc::as_ptr(&self.region_scaled_color_plan_cache) as usize, - ) + /// Return whether this session submits on the exact supplied command queue. + #[doc(hidden)] + pub fn uses_command_queue( + &self, + command_queue: &metal::CommandQueueRef, + ) -> Result { + Ok(self.runtime()?.queue.as_ptr() == command_queue.as_ptr()) + } + + pub(crate) fn consumer_event_timeline(&self) -> Arc> { + self.consumer_event_timeline.clone() } } @@ -267,8 +182,6 @@ impl MetalBackendSession { /// Shared batching session used by J2K Metal submit APIs. pub struct MetalSession { pub(crate) shared: batch::SharedSession, - #[cfg(target_os = "macos")] - backend: Option, } impl MetalSession { @@ -276,15 +189,14 @@ impl MetalSession { #[cfg(target_os = "macos")] pub fn with_backend_session(backend: MetalBackendSession) -> Self { Self { - shared: batch::SharedSession::default(), - backend: Some(backend), + shared: batch::SharedSession::with_backend_session(backend), } } /// Metal backend session owned by this batching session, if any. #[cfg(target_os = "macos")] pub fn backend_session(&self) -> Option<&MetalBackendSession> { - self.backend.as_ref() + self.shared.backend_session() } /// Number of Metal or emulated submissions flushed through this session. @@ -306,171 +218,3 @@ impl core::fmt::Debug for MetalSession { .finish() } } - -#[cfg(target_os = "macos")] -pub(crate) fn direct_gray_plan_cache_key( - bytes: &[u8], - format: PixelFormat, -) -> PreparedPlanCacheKey<'_> { - PreparedPlanCacheKey::direct_gray(bytes, format) -} - -#[cfg(target_os = "macos")] -pub(crate) fn cached_session_direct_gray_plan( - session: &MetalBackendSession, - key: PreparedPlanCacheKey<'_>, -) -> Result, Error> { - let mut guard = - session - .direct_gray_plan_cache - .lock() - .map_err(|_| Error::MetalStatePoisoned { - state: "direct grayscale prepared-plan cache", - })?; - Ok(guard - .get(key) - .map(|entry| (entry.plan.clone(), entry.prepared.clone()))) -} - -#[cfg(target_os = "macos")] -pub(crate) fn store_session_direct_gray_plan( - session: &MetalBackendSession, - key: PreparedPlanCacheKey<'_>, - plan: Arc, - prepared: Arc, -) -> Result<(), Error> { - prepared.disable_cpu_tier1_retention()?; - let mut guard = - session - .direct_gray_plan_cache - .lock() - .map_err(|_| Error::MetalStatePoisoned { - state: "direct grayscale prepared-plan cache", - })?; - evict_one_direct_plan_if_needed(&mut guard, key, DirectGrayPlanCacheEntry { plan, prepared }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn cached_session_direct_color_plan( - session: &MetalBackendSession, - key: PreparedPlanCacheKey<'_>, -) -> Result, Error> { - let mut guard = - session - .direct_color_plan_cache - .lock() - .map_err(|_| Error::MetalStatePoisoned { - state: "direct color prepared-plan cache", - })?; - Ok(guard - .get(key) - .map(|entry| (entry.plan.clone(), entry.prepared.clone()))) -} - -#[cfg(target_os = "macos")] -pub(crate) fn store_session_direct_color_plan( - session: &MetalBackendSession, - key: PreparedPlanCacheKey<'_>, - plan: Arc, - prepared: Arc, -) -> Result<(), Error> { - prepared.disable_dynamic_cpu_tier1_retention()?; - let mut guard = - session - .direct_color_plan_cache - .lock() - .map_err(|_| Error::MetalStatePoisoned { - state: "direct color prepared-plan cache", - })?; - evict_one_direct_plan_if_needed( - &mut guard, - key, - DirectColorPlanCacheEntry { plan, prepared }, - ) -} - -#[cfg(target_os = "macos")] -pub(crate) fn direct_plan_cache_key(bytes: &[u8], format: PixelFormat) -> PreparedPlanCacheKey<'_> { - PreparedPlanCacheKey::direct_color(bytes, format) -} - -#[cfg(target_os = "macos")] -fn evict_one_direct_plan_if_needed( - cache: &mut PreparedPlanCache, - key: PreparedPlanCacheKey<'_>, - value: T, -) -> Result<(), Error> { - cache.insert(key, value).map(|_| ()).map_err(|error| { - prepared_plan_cache_error("Metal prepared-plan cache update failed", error) - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn prepared_plan_cache_error( - context: &'static str, - error: PreparedPlanCacheError, -) -> Error { - match error { - PreparedPlanCacheError::Allocation(source) => { - Error::PreparedPlanCacheAllocation { context, source } - } - PreparedPlanCacheError::Invariant(reason) => { - Error::PreparedPlanCacheInvariant { context, reason } - } - } -} - -#[cfg(all(test, target_os = "macos"))] -mod error_tests { - use j2k_core::CodecError; - - use super::{prepared_plan_cache_error, PreparedPlanCacheError}; - use crate::Error; - - #[test] - fn prepared_plan_cache_allocation_keeps_its_source_and_classification() { - let source = Vec::::new() - .try_reserve(usize::MAX) - .expect_err("capacity overflow must fail before allocation"); - let source_message = source.to_string(); - let error = prepared_plan_cache_error( - "Metal prepared-plan cache update failed", - PreparedPlanCacheError::Allocation(source), - ); - - assert!(matches!( - &error, - Error::PreparedPlanCacheAllocation { - context: "Metal prepared-plan cache update failed", - .. - } - )); - let chained = std::error::Error::source(&error).expect("cache allocation source"); - assert_eq!(chained.to_string(), source_message); - assert!(!error.is_unsupported()); - assert!(!error.is_buffer_error()); - } - - #[test] - fn prepared_plan_cache_invariant_keeps_static_reason_without_source() { - let error = prepared_plan_cache_error( - "Metal region-scaled prepared-plan cache update failed", - PreparedPlanCacheError::Invariant("test cache invariant"), - ); - - assert_eq!( - error.to_string(), - "Metal kernel error: Metal region-scaled prepared-plan cache update failed: cache invariant failed: test cache invariant" - ); - assert!(matches!( - &error, - Error::PreparedPlanCacheInvariant { - context: "Metal region-scaled prepared-plan cache update failed", - reason: "test cache invariant", - } - )); - assert!(std::error::Error::source(&error).is_none()); - assert!(!error.is_unsupported()); - assert!(!error.is_buffer_error()); - } -} diff --git a/crates/j2k-metal/src/session/cache.rs b/crates/j2k-metal/src/session/cache.rs index 55d1384a..fe494317 100644 --- a/crates/j2k-metal/src/session/cache.rs +++ b/crates/j2k-metal/src/session/cache.rs @@ -71,9 +71,10 @@ struct PreparedPlanCacheEntry { /// Flat LRU prepared-plan cache with randomized digests and full-key validation. /// /// Digests are only lookup accelerators. Every hit compares the owned input and -/// all semantic key fields. Host accounting includes allocator-returned key and -/// entry-vector capacities plus each value's nested owners; device accounting -/// uses each retained Metal buffer's reported length. +/// all semantic key fields. Host accounting includes copied-key capacities or +/// retained input `Arc` allocations, entry-vector capacity, and each value's +/// nested owners; device accounting uses each retained Metal buffer's reported +/// length. pub(crate) struct PreparedPlanCache { entries: Vec>, digest_builder: S, @@ -406,8 +407,7 @@ where self.device_bytes = 0; } - #[cfg(test)] - fn len(&self) -> usize { + pub(crate) fn len(&self) -> usize { self.entries.len() } diff --git a/crates/j2k-metal/src/session/cache/key.rs b/crates/j2k-metal/src/session/cache/key.rs index 4ec57a89..b02c947b 100644 --- a/crates/j2k-metal/src/session/cache/key.rs +++ b/crates/j2k-metal/src/session/cache/key.rs @@ -1,5 +1,12 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use core::{ + hash::{Hash, Hasher}, + mem::size_of, +}; +use std::sync::Arc; + +use j2k::DecodeRequest; use j2k_core::{Downscale, PixelFormat, Rect}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -7,34 +14,66 @@ enum PreparedPlanKind { DirectGray, DirectColor, RegionScaledColor, + PreparedGray, + PreparedColor, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, Debug)] +enum PreparedPlanInput<'a> { + Contents(&'a [u8]), + ArcIdentity(&'a Arc<[u8]>), +} + +#[derive(Clone, Copy, Debug)] pub(crate) struct PreparedPlanCacheKey<'a> { - input: &'a [u8], + input: PreparedPlanInput<'a>, format: PixelFormat, roi: Option, scale: Downscale, + request: Option, kind: PreparedPlanKind, } +impl Hash for PreparedPlanCacheKey<'_> { + fn hash(&self, state: &mut H) { + self.kind.hash(state); + self.format.hash(state); + self.roi.hash(state); + self.scale.hash(state); + self.request.hash(state); + match self.input { + PreparedPlanInput::Contents(input) => { + 0_u8.hash(state); + input.hash(state); + } + PreparedPlanInput::ArcIdentity(input) => { + 1_u8.hash(state); + Arc::as_ptr(input).addr().hash(state); + input.len().hash(state); + } + } + } +} + impl<'a> PreparedPlanCacheKey<'a> { pub(crate) const fn direct_gray(input: &'a [u8], format: PixelFormat) -> Self { Self { - input, + input: PreparedPlanInput::Contents(input), format, roi: None, scale: Downscale::None, + request: None, kind: PreparedPlanKind::DirectGray, } } pub(crate) const fn direct_color(input: &'a [u8], format: PixelFormat) -> Self { Self { - input, + input: PreparedPlanInput::Contents(input), format, roi: None, scale: Downscale::None, + request: None, kind: PreparedPlanKind::DirectColor, } } @@ -46,24 +85,64 @@ impl<'a> PreparedPlanCacheKey<'a> { scale: Downscale, ) -> Self { Self { - input, + input: PreparedPlanInput::Contents(input), format, roi: Some(roi), scale, + request: None, kind: PreparedPlanKind::RegionScaledColor, } } - pub(super) const fn input_len(self) -> usize { - self.input.len() + pub(crate) const fn prepared_gray( + input: &'a Arc<[u8]>, + request: DecodeRequest, + format: PixelFormat, + ) -> Self { + Self { + input: PreparedPlanInput::ArcIdentity(input), + format, + roi: None, + scale: Downscale::None, + request: Some(request), + kind: PreparedPlanKind::PreparedGray, + } } + + pub(crate) const fn prepared_color( + input: &'a Arc<[u8]>, + request: DecodeRequest, + format: PixelFormat, + ) -> Self { + Self { + input: PreparedPlanInput::ArcIdentity(input), + format, + roi: None, + scale: Downscale::None, + request: Some(request), + kind: PreparedPlanKind::PreparedColor, + } + } + + pub(super) fn input_len(self) -> usize { + match self.input { + PreparedPlanInput::Contents(input) => input.len(), + PreparedPlanInput::ArcIdentity(input) => retained_arc_bytes(input), + } + } +} + +enum OwnedPreparedPlanInput { + Contents(Vec), + ArcIdentity(Arc<[u8]>), } pub(super) struct OwnedPreparedPlanCacheKey { - input: Vec, + input: OwnedPreparedPlanInput, format: PixelFormat, roi: Option, scale: Downscale, + request: Option, kind: PreparedPlanKind, } @@ -71,28 +150,51 @@ impl OwnedPreparedPlanCacheKey { pub(super) fn try_from_borrowed( key: PreparedPlanCacheKey<'_>, ) -> Result { - let mut input = Vec::new(); - input.try_reserve_exact(key.input.len())?; - input.extend_from_slice(key.input); + let input = match key.input { + PreparedPlanInput::Contents(source) => { + let mut owned = Vec::new(); + owned.try_reserve_exact(source.len())?; + owned.extend_from_slice(source); + OwnedPreparedPlanInput::Contents(owned) + } + PreparedPlanInput::ArcIdentity(source) => { + OwnedPreparedPlanInput::ArcIdentity(source.clone()) + } + }; Ok(Self { input, format: key.format, roi: key.roi, scale: key.scale, + request: key.request, kind: key.kind, }) } pub(super) fn matches(&self, key: PreparedPlanCacheKey<'_>) -> bool { - self.input.as_slice() == key.input + let input_matches = match (&self.input, key.input) { + (OwnedPreparedPlanInput::Contents(owned), PreparedPlanInput::Contents(input)) => { + owned.as_slice() == input + } + (OwnedPreparedPlanInput::ArcIdentity(owned), PreparedPlanInput::ArcIdentity(input)) => { + Arc::ptr_eq(owned, input) + } + (OwnedPreparedPlanInput::Contents(_), PreparedPlanInput::ArcIdentity(_)) + | (OwnedPreparedPlanInput::ArcIdentity(_), PreparedPlanInput::Contents(_)) => false, + }; + input_matches && self.format == key.format && self.roi == key.roi && self.scale == key.scale + && self.request == key.request && self.kind == key.kind } - pub(super) const fn input_capacity(&self) -> usize { - self.input.capacity() + pub(super) fn input_capacity(&self) -> usize { + match &self.input { + OwnedPreparedPlanInput::Contents(input) => input.capacity(), + OwnedPreparedPlanInput::ArcIdentity(input) => retained_arc_bytes(input), + } } #[cfg(test)] @@ -103,11 +205,16 @@ impl OwnedPreparedPlanCacheKey { .expect("test key capacity reservation"); owned.extend_from_slice(input); Self { - input: owned, + input: OwnedPreparedPlanInput::Contents(owned), format: PixelFormat::Rgb8, roi: None, scale: Downscale::None, + request: None, kind: PreparedPlanKind::DirectColor, } } } + +fn retained_arc_bytes(input: &Arc<[u8]>) -> usize { + input.len().saturating_add(2 * size_of::()) +} diff --git a/crates/j2k-metal/src/session/cache/tests.rs b/crates/j2k-metal/src/session/cache/tests.rs index dcf59d95..497001c7 100644 --- a/crates/j2k-metal/src/session/cache/tests.rs +++ b/crates/j2k-metal/src/session/cache/tests.rs @@ -6,6 +6,7 @@ use std::{ sync::Arc, }; +use j2k::DecodeRequest; use j2k_core::{Downscale, PixelFormat, Rect}; use super::*; @@ -65,6 +66,104 @@ fn id(value: Option<&TestValue>) -> Option { value.map(|value| value.id) } +#[test] +fn prepared_image_keys_match_only_the_exact_arc_request_and_format() { + let bytes = Arc::<[u8]>::from(b"same codestream".as_slice()); + let alias = bytes.clone(); + let distinct_owner = Arc::<[u8]>::from(b"same codestream".as_slice()); + let mut cache = PreparedPlanCache::new(8); + cache + .insert( + PreparedPlanCacheKey::prepared_gray(&bytes, DecodeRequest::Full, PixelFormat::Gray16), + value(41, 0, 0), + ) + .expect("insert exact prepared-image identity"); + + assert_eq!( + id(cache.get(PreparedPlanCacheKey::prepared_gray( + &alias, + DecodeRequest::Full, + PixelFormat::Gray16, + ))), + Some(41) + ); + for key in [ + PreparedPlanCacheKey::prepared_gray( + &distinct_owner, + DecodeRequest::Full, + PixelFormat::Gray16, + ), + PreparedPlanCacheKey::prepared_gray( + &bytes, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + PixelFormat::Gray16, + ), + PreparedPlanCacheKey::prepared_gray(&bytes, DecodeRequest::Full, PixelFormat::GrayI16), + PreparedPlanCacheKey::prepared_color(&bytes, DecodeRequest::Full, PixelFormat::Gray16), + ] { + assert_eq!(id(cache.get(key)), None); + } +} + +#[test] +fn prepared_image_arc_bytes_drive_eviction_and_oversized_admission() { + let first = Arc::<[u8]>::from(vec![1_u8; 32]); + let second = Arc::<[u8]>::from(vec![2_u8; 32]); + let first_key = + PreparedPlanCacheKey::prepared_color(&first, DecodeRequest::Full, PixelFormat::Rgb8); + let second_key = + PreparedPlanCacheKey::prepared_color(&second, DecodeRequest::Full, PixelFormat::Rgb8); + + let mut probe = PreparedPlanCache::with_limits_and_digest_builder( + 2, + usize::MAX, + usize::MAX, + ConstantDigestBuilder, + ); + probe + .insert(first_key, value(1, 0, 0)) + .expect("probe one prepared Arc entry"); + let one_entry_limit = probe.retained_host_bytes().expect("probe retained bytes"); + + let mut bounded = PreparedPlanCache::with_limits_and_digest_builder( + 2, + one_entry_limit, + usize::MAX, + ConstantDigestBuilder, + ); + assert_eq!( + bounded.insert(first_key, value(1, 0, 0)).unwrap(), + PreparedPlanCacheInsert::Cached + ); + assert_eq!( + bounded.insert(second_key, value(2, 0, 0)).unwrap(), + PreparedPlanCacheInsert::Cached + ); + assert_eq!(id(bounded.get(first_key)), None); + assert_eq!(id(bounded.get(second_key)), Some(2)); + assert!(bounded.retained_host_bytes().unwrap() <= one_entry_limit); + + let oversized = Arc::<[u8]>::from(vec![3_u8; 33]); + let oversized_key = + PreparedPlanCacheKey::prepared_color(&oversized, DecodeRequest::Full, PixelFormat::Rgb8); + let mut rejecting = PreparedPlanCache::with_limits_and_digest_builder( + 2, + one_entry_limit, + usize::MAX, + ConstantDigestBuilder, + ); + assert_eq!( + rejecting + .insert(oversized_key, value(3, 0, 0)) + .expect("oversized prepared Arc admission"), + PreparedPlanCacheInsert::SkippedOversized + ); + assert_eq!(rejecting.len(), 0); + assert_eq!(Arc::strong_count(&oversized), 1); +} + #[test] fn forced_digest_collision_does_not_cross_hit_distinct_inputs() { let mut cache = PreparedPlanCache::with_digest_builder(4, ConstantDigestBuilder); diff --git a/crates/j2k-metal/src/session/direct_plan_cache.rs b/crates/j2k-metal/src/session/direct_plan_cache.rs new file mode 100644 index 00000000..9eb35a68 --- /dev/null +++ b/crates/j2k-metal/src/session/direct_plan_cache.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Session-owned native and prepared direct-plan caches. + +use std::sync::{Arc, Mutex}; + +use j2k_core::PixelFormat; +use j2k_native::{J2kDirectColorPlan, J2kDirectGrayscalePlan}; + +use super::{cache, MetalBackendSession}; +use crate::Error; + +const DIRECT_PLAN_CACHE_CAP: usize = 128; + +#[derive(Clone)] +struct DirectGrayPlanCacheEntry { + plan: Arc, + prepared: Arc, +} + +#[derive(Clone)] +struct DirectColorPlanCacheEntry { + plan: Arc, + prepared: Arc, +} + +type CachedDirectGrayPlan = ( + Arc, + Arc, +); + +type CachedDirectColorPlan = ( + Arc, + Arc, +); + +#[derive(Clone)] +pub(super) struct DirectPlanCaches { + gray: Arc>>, + color: Arc>>, + region_scaled_color: + Arc>>>, +} + +impl DirectPlanCaches { + pub(super) fn new() -> Self { + Self { + gray: Arc::new(Mutex::new(cache::PreparedPlanCache::new( + DIRECT_PLAN_CACHE_CAP, + ))), + color: Arc::new(Mutex::new(cache::PreparedPlanCache::new( + DIRECT_PLAN_CACHE_CAP, + ))), + region_scaled_color: Arc::new(Mutex::new(cache::PreparedPlanCache::new( + crate::hybrid::REGION_SCALED_COLOR_PLAN_CACHE_CAP, + ))), + } + } + + #[cfg(test)] + fn ids(&self) -> (usize, usize, usize) { + ( + Arc::as_ptr(&self.gray) as usize, + Arc::as_ptr(&self.color) as usize, + Arc::as_ptr(&self.region_scaled_color) as usize, + ) + } +} + +impl cache::PreparedPlanCacheValue for DirectGrayPlanCacheEntry { + fn retained_cache_weight( + &self, + ) -> Result { + let prepared = self + .prepared + .retained_cache_bytes() + .map_err(cache::PreparedPlanCacheError::Invariant)?; + direct_plan_cache_weight( + self.plan.retained_allocation_bytes().map_err(|_| { + cache::PreparedPlanCacheError::Invariant( + "native grayscale direct-plan retained-byte accounting failed", + ) + })?, + core::mem::size_of::(), + prepared.host, + prepared.device, + core::mem::size_of::(), + ) + } +} + +impl cache::PreparedPlanCacheValue for DirectColorPlanCacheEntry { + fn retained_cache_weight( + &self, + ) -> Result { + let prepared = self + .prepared + .retained_cache_bytes() + .map_err(cache::PreparedPlanCacheError::Invariant)?; + direct_plan_cache_weight( + self.plan.retained_allocation_bytes().map_err(|_| { + cache::PreparedPlanCacheError::Invariant( + "native color direct-plan retained-byte accounting failed", + ) + })?, + core::mem::size_of::(), + prepared.host, + prepared.device, + core::mem::size_of::(), + ) + } +} + +impl cache::PreparedPlanCacheValue for Arc { + fn retained_cache_weight( + &self, + ) -> Result { + let prepared = self + .retained_cache_bytes() + .map_err(cache::PreparedPlanCacheError::Invariant)?; + let host_bytes = arc_owner_bytes( + core::mem::size_of::(), + prepared.host, + )?; + Ok(cache::PreparedPlanCacheWeight::new( + host_bytes, + prepared.device, + )) + } +} + +impl cache::PreparedPlanCacheValue for Arc { + fn retained_cache_weight( + &self, + ) -> Result { + let prepared = self + .retained_cache_bytes() + .map_err(cache::PreparedPlanCacheError::Invariant)?; + let host_bytes = arc_owner_bytes( + core::mem::size_of::(), + prepared.host, + )?; + Ok(cache::PreparedPlanCacheWeight::new( + host_bytes, + prepared.device, + )) + } +} + +fn direct_plan_cache_weight( + native_nested_host: usize, + native_root_bytes: usize, + prepared_host_bytes: usize, + prepared_device_bytes: usize, + prepared_root_bytes: usize, +) -> Result { + let native_host = arc_owner_bytes(native_root_bytes, native_nested_host)?; + let prepared_host = arc_owner_bytes(prepared_root_bytes, prepared_host_bytes)?; + let host_bytes = + native_host + .checked_add(prepared_host) + .ok_or(cache::PreparedPlanCacheError::Invariant( + "native and prepared direct-plan cache weight overflow", + ))?; + Ok(cache::PreparedPlanCacheWeight::new( + host_bytes, + prepared_device_bytes, + )) +} + +fn arc_owner_bytes( + root_bytes: usize, + nested_bytes: usize, +) -> Result { + root_bytes + .checked_add(2 * core::mem::size_of::()) + .and_then(|bytes| bytes.checked_add(nested_bytes)) + .ok_or(cache::PreparedPlanCacheError::Invariant( + "prepared-plan Arc owner byte count overflow", + )) +} + +pub(crate) fn direct_gray_plan_cache_key( + bytes: &[u8], + format: PixelFormat, +) -> cache::PreparedPlanCacheKey<'_> { + cache::PreparedPlanCacheKey::direct_gray(bytes, format) +} + +pub(crate) fn direct_plan_cache_key( + bytes: &[u8], + format: PixelFormat, +) -> cache::PreparedPlanCacheKey<'_> { + cache::PreparedPlanCacheKey::direct_color(bytes, format) +} + +pub(crate) fn cached_session_direct_gray_plan( + session: &MetalBackendSession, + key: cache::PreparedPlanCacheKey<'_>, +) -> Result, Error> { + let mut guard = + session + .direct_plan_caches + .gray + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "direct grayscale prepared-plan cache", + })?; + Ok(guard + .get(key) + .map(|entry| (entry.plan.clone(), entry.prepared.clone()))) +} + +pub(crate) fn store_session_direct_gray_plan( + session: &MetalBackendSession, + key: cache::PreparedPlanCacheKey<'_>, + plan: Arc, + prepared: Arc, +) -> Result<(), Error> { + prepared.disable_cpu_tier1_retention()?; + let mut guard = + session + .direct_plan_caches + .gray + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "direct grayscale prepared-plan cache", + })?; + evict_one_direct_plan_if_needed(&mut guard, key, DirectGrayPlanCacheEntry { plan, prepared }) +} + +pub(crate) fn cached_session_direct_color_plan( + session: &MetalBackendSession, + key: cache::PreparedPlanCacheKey<'_>, +) -> Result, Error> { + let mut guard = + session + .direct_plan_caches + .color + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "direct color prepared-plan cache", + })?; + Ok(guard + .get(key) + .map(|entry| (entry.plan.clone(), entry.prepared.clone()))) +} + +pub(crate) fn store_session_direct_color_plan( + session: &MetalBackendSession, + key: cache::PreparedPlanCacheKey<'_>, + plan: Arc, + prepared: Arc, +) -> Result<(), Error> { + prepared.disable_dynamic_cpu_tier1_retention()?; + let mut guard = + session + .direct_plan_caches + .color + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "direct color prepared-plan cache", + })?; + evict_one_direct_plan_if_needed( + &mut guard, + key, + DirectColorPlanCacheEntry { plan, prepared }, + ) +} + +pub(crate) fn cached_session_region_scaled_color_plan( + session: &MetalBackendSession, + key: cache::PreparedPlanCacheKey<'_>, +) -> Result>, Error> { + let mut guard = session + .direct_plan_caches + .region_scaled_color + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "session region-scaled color prepared-plan cache", + })?; + Ok(guard.get(key).cloned()) +} + +pub(crate) fn store_session_region_scaled_color_plan( + session: &MetalBackendSession, + key: cache::PreparedPlanCacheKey<'_>, + plan: Arc, +) -> Result<(), Error> { + let mut guard = session + .direct_plan_caches + .region_scaled_color + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "session region-scaled color prepared-plan cache", + })?; + guard.insert(key, plan).map(|_| ()).map_err(|error| { + prepared_plan_cache_error( + "Metal region-scaled prepared-plan cache update failed", + error, + ) + }) +} + +fn evict_one_direct_plan_if_needed( + cache: &mut cache::PreparedPlanCache, + key: cache::PreparedPlanCacheKey<'_>, + value: T, +) -> Result<(), Error> { + cache.insert(key, value).map(|_| ()).map_err(|error| { + prepared_plan_cache_error("Metal prepared-plan cache update failed", error) + }) +} + +pub(crate) fn prepared_plan_cache_error( + context: &'static str, + error: cache::PreparedPlanCacheError, +) -> Error { + match error { + cache::PreparedPlanCacheError::Allocation(source) => { + Error::PreparedPlanCacheAllocation { context, source } + } + cache::PreparedPlanCacheError::Invariant(reason) => { + Error::PreparedPlanCacheInvariant { context, reason } + } + } +} + +#[cfg(test)] +pub(crate) fn direct_cache_ids_for_test(session: &MetalBackendSession) -> (usize, usize, usize) { + session.direct_plan_caches.ids() +} + +#[cfg(test)] +mod tests; diff --git a/crates/j2k-metal/src/session/direct_plan_cache/tests.rs b/crates/j2k-metal/src/session/direct_plan_cache/tests.rs new file mode 100644 index 00000000..6b6c664b --- /dev/null +++ b/crates/j2k-metal/src/session/direct_plan_cache/tests.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_core::CodecError; + +use super::{cache, prepared_plan_cache_error}; +use crate::Error; + +#[test] +fn prepared_plan_cache_allocation_keeps_its_source_and_classification() { + let source = Vec::::new() + .try_reserve(usize::MAX) + .expect_err("capacity overflow must fail before allocation"); + let source_message = source.to_string(); + let error = prepared_plan_cache_error( + "Metal prepared-plan cache update failed", + cache::PreparedPlanCacheError::Allocation(source), + ); + + assert!(matches!( + &error, + Error::PreparedPlanCacheAllocation { + context: "Metal prepared-plan cache update failed", + .. + } + )); + let chained = std::error::Error::source(&error).expect("cache allocation source"); + assert_eq!(chained.to_string(), source_message); + assert!(!error.is_unsupported()); + assert!(!error.is_buffer_error()); +} + +#[test] +fn prepared_plan_cache_invariant_keeps_static_reason_without_source() { + let error = prepared_plan_cache_error( + "Metal region-scaled prepared-plan cache update failed", + cache::PreparedPlanCacheError::Invariant("test cache invariant"), + ); + + assert_eq!( + error.to_string(), + "Metal kernel error: Metal region-scaled prepared-plan cache update failed: cache invariant failed: test cache invariant" + ); + assert!(matches!( + &error, + Error::PreparedPlanCacheInvariant { + context: "Metal region-scaled prepared-plan cache update failed", + reason: "test cache invariant", + } + )); + assert!(std::error::Error::source(&error).is_none()); + assert!(!error.is_unsupported()); + assert!(!error.is_buffer_error()); +} diff --git a/crates/j2k-metal/src/store.metal b/crates/j2k-metal/src/store.metal index 7ba416d1..35a6e73f 100644 --- a/crates/j2k-metal/src/store.metal +++ b/crates/j2k-metal/src/store.metal @@ -56,6 +56,8 @@ struct J2kGrayStoreParams { uint copy_width; uint copy_height; uint output_width; + uint output_stride; + uint output_item_offset; uint output_x; uint output_y; float addend; @@ -64,11 +66,79 @@ struct J2kGrayStoreParams { float u16_scale; }; + +constant uint J2K_BATCH_LAYOUT_NCHW = 0; +constant uint J2K_BATCH_LAYOUT_NHWC = 1; +constant uint J2K_NATIVE_RGB_NO_MCT = 0; + +inline float j2k_unsigned_native_sample(float value, uint bit_depth) { + if (isnan(value)) { + return 0.0f; + } + const float max_value = float((1u << bit_depth) - 1u); + return floor(clamp(value, 0.0f, max_value) + 0.5f); +} + +inline float3 j2k_native_color_samples( + float value0, + float value1, + float value2, + uint mct, + uint transform, + uint is_signed, + uint bit_depth0, + uint bit_depth1, + uint bit_depth2 +) { + if (mct == J2K_NATIVE_RGB_NO_MCT) { + return float3(value0, value1, value2); + } + + const float3 addends = is_signed != 0u + ? float3(0.0f) + : float3( + float(1u << (bit_depth0 - 1u)), + float(1u << (bit_depth1 - 1u)), + float(1u << (bit_depth2 - 1u)) + ); + if (transform == J2K_MCT_TRANSFORM_REVERSIBLE53) { + const float green = value0 - floor((value2 + value1) * 0.25f); + return float3(value2 + green, green, value1 + green) + addends; + } + return float3( + value2 * 1.402f + value0, + value2 * -0.71414f + value1 * -0.34413f + value0, + value1 * 1.772f + value0 + ) + addends; +} + +inline uint j2k_native_color_output_index( + uint width, + uint height, + uint output_row_stride, + uint output_item_offset, + uint layout, + uint2 gid, + uint channels, + uint channel +) { + if (layout == J2K_BATCH_LAYOUT_NCHW) { + const uint plane_len = width * height; + return output_item_offset + channel * plane_len + gid.y * width + gid.x; + } + return output_item_offset + gid.y * output_row_stride + gid.x * channels + channel; +} + struct J2kStoreWindowIndices { uint src_idx; uint dst_idx; }; +inline short j2k_pack_native_i16(float value, float positive_max) { + const float negative_min = -positive_max - 1.0f; + return short(rint(clamp(value, negative_min, positive_max))); +} + inline J2kStoreWindowIndices j2k_store_window_indices( uint input_width, uint output_width, @@ -192,6 +262,35 @@ kernel void j2k_store_component_repeated_gray_u16( output[indices.dst_idx] = pack_to_u16(input[indices.src_idx] + params.addend, params.max_value, params.u16_scale); } +kernel void j2k_store_component_repeated_gray_i16( + device const float *input [[buffer(0)]], + device short *output [[buffer(1)]], + constant J2kRepeatedGrayStoreParams ¶ms [[buffer(2)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.copy_width || gid.y >= params.copy_height || gid.z >= params.batch_count) { + return; + } + + const uint input_plane_len = params.input_width * params.input_height; + const uint output_plane_len = params.output_width * params.output_height; + const J2kStoreWindowIndices indices = j2k_store_window_indices( + params.input_width, + params.output_width, + params.source_x, + params.source_y, + params.output_x, + params.output_y, + gid.xy, + gid.z * input_plane_len, + gid.z * output_plane_len + ); + output[indices.dst_idx] = j2k_pack_native_i16( + input[indices.src_idx] + params.addend, + params.max_value + ); +} + kernel void j2k_store_component_repeated_gray_u8_contiguous( device const float *input [[buffer(0)]], device uchar *output [[buffer(1)]], @@ -234,14 +333,14 @@ kernel void j2k_store_component_gray_u8( const J2kStoreWindowIndices indices = j2k_store_window_indices( params.input_width, - params.output_width, + params.output_stride, params.source_x, params.source_y, params.output_x, params.output_y, gid, 0u, - 0u + params.output_item_offset ); output[indices.dst_idx] = scale_to_u8(input[indices.src_idx] + params.addend, params.max_value, params.u8_scale); } @@ -258,14 +357,41 @@ kernel void j2k_store_component_gray_u16( const J2kStoreWindowIndices indices = j2k_store_window_indices( params.input_width, - params.output_width, + params.output_stride, params.source_x, params.source_y, params.output_x, params.output_y, gid, 0u, - 0u + params.output_item_offset ); output[indices.dst_idx] = pack_to_u16(input[indices.src_idx] + params.addend, params.max_value, params.u16_scale); } + +kernel void j2k_store_component_gray_i16( + device const float *input [[buffer(0)]], + device short *output [[buffer(1)]], + constant J2kGrayStoreParams ¶ms [[buffer(2)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.copy_width || gid.y >= params.copy_height) { + return; + } + + const J2kStoreWindowIndices indices = j2k_store_window_indices( + params.input_width, + params.output_stride, + params.source_x, + params.source_y, + params.output_x, + params.output_y, + gid, + 0u, + params.output_item_offset + ); + output[indices.dst_idx] = j2k_pack_native_i16( + input[indices.src_idx] + params.addend, + params.max_value + ); +} diff --git a/crates/j2k-metal/src/store_native_color_batch.metal b/crates/j2k-metal/src/store_native_color_batch.metal new file mode 100644 index 00000000..81f36ea1 --- /dev/null +++ b/crates/j2k-metal/src/store_native_color_batch.metal @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +struct J2kNativeColorBatchStoreParams { + uint width; + uint height; + uint plane_stride; + uint output_row_stride; + uint output_item_stride; + uint batch_count; + uint layout; + uint mct; + uint transform; + uint is_signed; + uint bit_depths[4]; +}; + +kernel void j2k_store_native_rgb_batch_u8( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device uchar *output [[buffer(3)]], + constant J2kNativeColorBatchStoreParams ¶ms [[buffer(4)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { + return; + } + const uint plane_idx = gid.z * params.plane_stride + gid.y * params.width + gid.x; + const float3 rgb = j2k_native_color_samples( + plane0[plane_idx], + plane1[plane_idx], + plane2[plane_idx], + params.mct, + params.transform, + params.is_signed, + params.bit_depths[0], + params.bit_depths[1], + params.bit_depths[2] + ); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 3u, 0u)] = + uchar(j2k_unsigned_native_sample(rgb[0], params.bit_depths[0])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 3u, 1u)] = + uchar(j2k_unsigned_native_sample(rgb[1], params.bit_depths[1])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 3u, 2u)] = + uchar(j2k_unsigned_native_sample(rgb[2], params.bit_depths[2])); +} + +kernel void j2k_store_native_rgb_batch_u16( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device ushort *output [[buffer(3)]], + constant J2kNativeColorBatchStoreParams ¶ms [[buffer(4)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { + return; + } + const uint plane_idx = gid.z * params.plane_stride + gid.y * params.width + gid.x; + const float3 rgb = j2k_native_color_samples( + plane0[plane_idx], + plane1[plane_idx], + plane2[plane_idx], + params.mct, + params.transform, + params.is_signed, + params.bit_depths[0], + params.bit_depths[1], + params.bit_depths[2] + ); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 3u, 0u)] = + ushort(j2k_unsigned_native_sample(rgb[0], params.bit_depths[0])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 3u, 1u)] = + ushort(j2k_unsigned_native_sample(rgb[1], params.bit_depths[1])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 3u, 2u)] = + ushort(j2k_unsigned_native_sample(rgb[2], params.bit_depths[2])); +} + +kernel void j2k_store_native_rgb_batch_i16( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device short *output [[buffer(3)]], + constant J2kNativeColorBatchStoreParams ¶ms [[buffer(4)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { + return; + } + const uint plane_idx = gid.z * params.plane_stride + gid.y * params.width + gid.x; + const float3 rgb = j2k_native_color_samples( + plane0[plane_idx], + plane1[plane_idx], + plane2[plane_idx], + params.mct, + params.transform, + params.is_signed, + params.bit_depths[0], + params.bit_depths[1], + params.bit_depths[2] + ); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 3u, 0u)] = + j2k_pack_native_i16(rgb[0], float((1u << (params.bit_depths[0] - 1u)) - 1u)); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 3u, 1u)] = + j2k_pack_native_i16(rgb[1], float((1u << (params.bit_depths[1] - 1u)) - 1u)); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 3u, 2u)] = + j2k_pack_native_i16(rgb[2], float((1u << (params.bit_depths[2] - 1u)) - 1u)); +} + +kernel void j2k_store_native_rgba_batch_u8( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device const float *plane3 [[buffer(3)]], + device uchar *output [[buffer(4)]], + constant J2kNativeColorBatchStoreParams ¶ms [[buffer(5)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { + return; + } + const uint plane_idx = gid.z * params.plane_stride + gid.y * params.width + gid.x; + const float3 rgb = j2k_native_color_samples( + plane0[plane_idx], + plane1[plane_idx], + plane2[plane_idx], + params.mct, + params.transform, + params.is_signed, + params.bit_depths[0], + params.bit_depths[1], + params.bit_depths[2] + ); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 0u)] = + uchar(j2k_unsigned_native_sample(rgb[0], params.bit_depths[0])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 1u)] = + uchar(j2k_unsigned_native_sample(rgb[1], params.bit_depths[1])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 2u)] = + uchar(j2k_unsigned_native_sample(rgb[2], params.bit_depths[2])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 3u)] = + uchar(j2k_unsigned_native_sample(plane3[plane_idx], params.bit_depths[3])); +} + +kernel void j2k_store_native_rgba_batch_u16( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device const float *plane3 [[buffer(3)]], + device ushort *output [[buffer(4)]], + constant J2kNativeColorBatchStoreParams ¶ms [[buffer(5)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { + return; + } + const uint plane_idx = gid.z * params.plane_stride + gid.y * params.width + gid.x; + const float3 rgb = j2k_native_color_samples( + plane0[plane_idx], + plane1[plane_idx], + plane2[plane_idx], + params.mct, + params.transform, + params.is_signed, + params.bit_depths[0], + params.bit_depths[1], + params.bit_depths[2] + ); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 0u)] = + ushort(j2k_unsigned_native_sample(rgb[0], params.bit_depths[0])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 1u)] = + ushort(j2k_unsigned_native_sample(rgb[1], params.bit_depths[1])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 2u)] = + ushort(j2k_unsigned_native_sample(rgb[2], params.bit_depths[2])); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 3u)] = + ushort(j2k_unsigned_native_sample(plane3[plane_idx], params.bit_depths[3])); +} + +kernel void j2k_store_native_rgba_batch_i16( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device const float *plane3 [[buffer(3)]], + device short *output [[buffer(4)]], + constant J2kNativeColorBatchStoreParams ¶ms [[buffer(5)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { + return; + } + const uint plane_idx = gid.z * params.plane_stride + gid.y * params.width + gid.x; + const float3 rgb = j2k_native_color_samples( + plane0[plane_idx], + plane1[plane_idx], + plane2[plane_idx], + params.mct, + params.transform, + params.is_signed, + params.bit_depths[0], + params.bit_depths[1], + params.bit_depths[2] + ); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 0u)] = + j2k_pack_native_i16(rgb[0], float((1u << (params.bit_depths[0] - 1u)) - 1u)); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 1u)] = + j2k_pack_native_i16(rgb[1], float((1u << (params.bit_depths[1] - 1u)) - 1u)); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 2u)] = + j2k_pack_native_i16(rgb[2], float((1u << (params.bit_depths[2] - 1u)) - 1u)); + output[j2k_native_color_output_index(params.width, params.height, params.output_row_stride, gid.z * params.output_item_stride, params.layout, uint2(gid.x, gid.y), 4u, 3u)] = + j2k_pack_native_i16( + plane3[plane_idx], + float((1u << (params.bit_depths[3] - 1u)) - 1u) + ); +} + diff --git a/crates/j2k-metal/src/surface.rs b/crates/j2k-metal/src/surface.rs index 86a72d13..6e810bd4 100644 --- a/crates/j2k-metal/src/surface.rs +++ b/crates/j2k-metal/src/surface.rs @@ -226,6 +226,20 @@ impl Surface { storage: Storage::Metal(image), }) } + + #[cfg(target_os = "macos")] + pub(crate) fn from_resident_metal_image(image: ResidentMetalImage) -> Self { + let layout = image.layout(); + Self { + backend: BackendKind::Metal, + residency: SurfaceResidency::MetalResidentDecode, + dimensions: layout.dimensions(), + fmt: layout.pixel_format(), + pitch_bytes: layout.pitch_bytes(), + byte_offset: layout.byte_offset(), + storage: Storage::Metal(image), + } + } } #[doc(hidden)] diff --git a/crates/j2k-metal/tests/batch_classic_color.rs b/crates/j2k-metal/tests/batch_classic_color.rs new file mode 100644 index 00000000..809e2b96 --- /dev/null +++ b/crates/j2k-metal/tests/batch_classic_color.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#![cfg(target_os = "macos")] + +use std::sync::Arc; + +use j2k::{ + BatchColor, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, DecodeRequest, + EncodedImage, NativeSampleType, +}; +use j2k_core::{Downscale, PixelFormat, Rect}; +use j2k_metal::{MetalBatchDecoder, MetalImageDestination, MetalImageLayout}; +use j2k_native::{encode, EncodeOptions}; + +#[derive(Debug, Clone, Copy)] +enum ClassicRgbProfile { + U8, + U8Irreversible97, + U12, + I12, +} + +fn should_run_metal_runtime() -> bool { + j2k_test_support::metal_runtime_gate(module_path!()) +} + +fn classic_rgb_fixture(profile: ClassicRgbProfile) -> Vec { + const WIDTH: u32 = 8; + const HEIGHT: u32 = 8; + let pixel_count = usize::try_from(WIDTH) + .expect("fixture width fits usize") + .checked_mul(usize::try_from(HEIGHT).expect("fixture height fits usize")) + .expect("fixture area fits usize"); + let (pixels, bit_depth, signed, reversible) = match profile { + ClassicRgbProfile::U8 | ClassicRgbProfile::U8Irreversible97 => { + let mut pixels = Vec::with_capacity(pixel_count * 3); + for y in 0..HEIGHT { + for x in 0..WIDTH { + for value in [x * 17 + y * 3, x * 5 + y * 19 + 7, x * 11 + y * 13 + 29] { + pixels.push(value.to_le_bytes()[0]); + } + } + } + (pixels, 8, false, matches!(profile, ClassicRgbProfile::U8)) + } + ClassicRgbProfile::U12 => { + let mut pixels = Vec::with_capacity(pixel_count * 3 * 2); + for y in 0..HEIGHT { + for x in 0..WIDTH { + for value in [ + x * 193 + y * 257, + x * 313 + y * 97 + 31, + x * 71 + y * 401 + 63, + ] { + let value = u16::try_from(value & 0x0fff).expect("twelve-bit RGB sample"); + pixels.extend_from_slice(&value.to_le_bytes()); + } + } + } + (pixels, 12, false, true) + } + ClassicRgbProfile::I12 => { + let mut pixels = Vec::with_capacity(pixel_count * 3 * 2); + for y in 0..HEIGHT { + for x in 0..WIDTH { + for value in [ + x * 193 + y * 257, + x * 313 + y * 97 + 31, + x * 71 + y * 401 + 63, + ] { + let value = i16::try_from( + i32::try_from(value & 0x0fff).expect("twelve-bit RGB sample fits i32") + - 2048, + ) + .expect("signed twelve-bit RGB sample"); + pixels.extend_from_slice(&value.to_le_bytes()); + } + } + } + (pixels, 12, true, true) + } + }; + encode( + &pixels, + WIDTH, + HEIGHT, + 3, + bit_depth, + signed, + &EncodeOptions { + reversible, + use_mct: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }, + ) + .unwrap_or_else(|error| panic!("encode {profile:?} classic RGB fixture: {error}")) +} + +fn rgb_format(sample_type: NativeSampleType) -> PixelFormat { + match sample_type { + NativeSampleType::U8 => PixelFormat::Rgb8, + NativeSampleType::U16 => PixelFormat::Rgb16, + NativeSampleType::I16 => PixelFormat::RgbI16, + _ => panic!("unsupported RGB sample type"), + } +} + +fn assert_native_samples(actual: &[u8], expected: &CpuBatchSamples, max_lsb_diff: u8) { + match expected { + CpuBatchSamples::U8(expected) => assert!( + actual + .iter() + .zip(expected) + .all(|(actual, expected)| actual.abs_diff(*expected) <= max_lsb_diff), + "U8 reconstruction differs by more than {max_lsb_diff} LSB" + ), + CpuBatchSamples::U16(expected) => { + let actual = actual + .chunks_exact(2) + .map(|sample| u16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + assert!( + actual + .iter() + .zip(expected) + .all(|(actual, expected)| actual.abs_diff(*expected) <= u16::from(max_lsb_diff)), + "U16 reconstruction differs by more than {max_lsb_diff} LSB" + ); + } + CpuBatchSamples::I16(expected) => { + let actual = actual + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + assert!( + actual + .iter() + .zip(expected) + .all(|(actual, expected)| actual.abs_diff(*expected) <= u16::from(max_lsb_diff)), + "I16 reconstruction differs by more than {max_lsb_diff} LSB" + ); + } + _ => panic!("unsupported CPU RGB oracle type"), + } +} + +fn assert_classic_rgb_layout( + profile: ClassicRgbProfile, + encoded: &Arc<[u8]>, + layout: BatchLayout, + requests: &[DecodeRequest], +) { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let max_lsb_diff = u8::from(matches!(profile, ClassicRgbProfile::U8Irreversible97)); + let inputs = requests + .iter() + .copied() + .map(|request| EncodedImage::new(encoded.clone(), request)) + .collect::>(); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU classic RGB oracle"); + assert!( + expected.errors().is_empty(), + "{profile:?} {layout:?} CPU errors: {:?}", + expected.errors() + ); + + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(inputs) + .expect("prepare classic Metal RGB request matrix"); + assert!( + prepared.errors().is_empty(), + "{profile:?} {layout:?} prepare errors: {:?}", + prepared.errors() + ); + + for group in prepared.groups() { + assert_eq!(group.info().color, BatchColor::Rgb); + let expected_group = expected + .groups() + .iter() + .find(|expected| expected.source_indices() == group.source_indices()) + .expect("matching CPU classic RGB group"); + let fmt = rgb_format(group.info().sample_type); + let bytes_per_sample = fmt.bytes_per_sample(); + let (width, height) = group.info().dimensions; + let row_bytes = + usize::try_from(width).expect("classic RGB width fits usize") * 3 * bytes_per_sample; + let image_bytes = + row_bytes * usize::try_from(height).expect("classic RGB height fits usize"); + let output_bytes = image_bytes * group.images().len(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + output_bytes + 8, + ) + .expect("classic RGB destination buffer"); + let destination_layout = MetalImageLayout::new_batch( + 4, + (width, height), + row_bytes, + fmt, + group.images().len(), + image_bytes, + ) + .expect("classic RGB destination layout"); + // SAFETY: this fresh output range remains exclusively retained by the + // pending codec submission until completion. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), destination_layout) + .expect("classic RGB destination") + }; + decoder + .submit_prepared_group_into(group, destination) + .unwrap_or_else(|error| { + panic!("submit {profile:?} {layout:?} classic RGB group: {error}") + }) + .wait() + .unwrap_or_else(|error| { + panic!("complete {profile:?} {layout:?} classic RGB group: {error}") + }); + + // SAFETY: codec completion released exclusive output access. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, output_bytes) + .expect("classic RGB output samples") + }; + assert_native_samples(&actual, expected_group.samples(), max_lsb_diff); + } +} + +#[test] +fn prepared_classic_rgb_matches_cpu_for_native_types_requests_and_layouts() { + if !should_run_metal_runtime() { + return; + } + + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + for profile in [ + ClassicRgbProfile::U8, + ClassicRgbProfile::U8Irreversible97, + ClassicRgbProfile::U12, + ClassicRgbProfile::I12, + ] { + let encoded = Arc::<[u8]>::from(classic_rgb_fixture(profile)); + for layout in [BatchLayout::Nchw, BatchLayout::Nhwc] { + assert_classic_rgb_layout(profile, &encoded, layout, &requests); + } + } +} diff --git a/crates/j2k-metal/tests/batch_decoder_contract.rs b/crates/j2k-metal/tests/batch_decoder_contract.rs new file mode 100644 index 00000000..de15ae88 --- /dev/null +++ b/crates/j2k-metal/tests/batch_decoder_contract.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::{BatchDecodeOptions, BatchDecoder, EncodedImage, PreparedBatch}; +#[cfg(target_os = "macos")] +use j2k_metal::SubmittedMetalPreparedBatch; +use j2k_metal::{Error, MetalBatchDecodeResult, MetalBatchDecoder}; + +const _: fn(&MetalBatchDecoder) -> BatchDecodeOptions = + ::options; +const _: fn(&MetalBatchDecoder, Vec) -> Result = + MetalBatchDecoder::prepare; +const _: fn(&mut MetalBatchDecoder, Vec) -> Result = + MetalBatchDecoder::decode_batch; + +#[cfg(target_os = "macos")] +const _: fn( + &mut MetalBatchDecoder, + Vec, +) -> Result = MetalBatchDecoder::submit_batch; + +#[test] +fn metal_batch_decoder_implements_shared_codec_contract() { + fn assert_contract() + where + D: BatchDecoder, + { + } + + assert_contract::(); +} diff --git a/crates/j2k-metal/tests/batch_rgba.rs b/crates/j2k-metal/tests/batch_rgba.rs new file mode 100644 index 00000000..a3d61245 --- /dev/null +++ b/crates/j2k-metal/tests/batch_rgba.rs @@ -0,0 +1,383 @@ +#![cfg(target_os = "macos")] + +use std::sync::Arc; + +use j2k::{ + wrap_j2k_codestream, BatchAlpha, BatchColor, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, + CpuBatchSamples, DecodeRequest, EncodedImage, J2kChannelAssociation, J2kChannelDefinition, + J2kChannelType, J2kFileBoxMetadata, J2kFileColorSpec, J2kFileWrapOptions, NativeSampleType, +}; +use j2k_core::{Colorspace, DeviceSurface, Downscale, PixelFormat, Rect}; +use j2k_metal::{MetalBatchDecoder, MetalImageDestination, MetalImageLayout}; +use j2k_test_support::{ + generated_htj2k_rgba_fixture, Htj2kRgbaAlpha, Htj2kRgbaFixture, Htj2kRgbaSampleProfile, + Htj2kRgbaSamples, +}; + +fn should_run_metal_runtime() -> bool { + j2k_test_support::metal_runtime_gate(module_path!()) +} + +fn rgba_format(sample_type: NativeSampleType) -> PixelFormat { + match sample_type { + NativeSampleType::U8 => PixelFormat::Rgba8, + NativeSampleType::U16 => PixelFormat::Rgba16, + NativeSampleType::I16 => PixelFormat::RgbaI16, + _ => panic!("unsupported RGBA sample type"), + } +} + +fn assert_native_samples(actual: &[u8], expected: &CpuBatchSamples) { + match expected { + CpuBatchSamples::U8(expected) => assert_eq!(actual, expected), + CpuBatchSamples::U16(expected) => { + let actual = actual + .chunks_exact(2) + .map(|sample| u16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + assert_eq!(&actual, expected); + } + CpuBatchSamples::I16(expected) => { + let actual = actual + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + assert_eq!(&actual, expected); + } + _ => panic!("unsupported CPU RGBA oracle type"), + } +} + +fn completed_resident_batch_bytes(group: &j2k_metal::MetalBatchGroup) -> Vec { + let resident = group + .resident_batch() + .expect("codec-owned decode must expose its dense resident allocation"); + // SAFETY: the group is visible only after codec completion and this test + // performs one readback without submitting a writer or retaining the handle. + unsafe { + j2k_metal_support::checked_buffer_read_vec::( + resident.metal_buffer(), + resident.byte_offset(), + resident.byte_len(), + ) + .expect("read completed dense resident RGBA batch") + } +} + +fn wrap_rgba_jph(codestream: &[u8], alpha: Htj2kRgbaAlpha) -> Vec { + wrap_rgba_container( + codestream, + alpha, + J2kFileWrapOptions::jph(), + "wrap explicit HTJ2K RGBA image", + ) +} + +fn wrap_rgba_container( + codestream: &[u8], + alpha: Htj2kRgbaAlpha, + options: J2kFileWrapOptions<'_>, + context: &str, +) -> Vec { + let alpha_type = match alpha { + Htj2kRgbaAlpha::Straight => J2kChannelType::Opacity, + Htj2kRgbaAlpha::Premultiplied => J2kChannelType::PremultipliedOpacity, + }; + let channel_definitions = [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 3, + channel_type: alpha_type, + association: J2kChannelAssociation::WholeImage, + }, + ]; + wrap_j2k_codestream( + codestream, + options + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &channel_definitions, + }), + ) + .unwrap_or_else(|error| panic!("{context}: {error}")) +} + +fn wrap_classic_rgba_jp2(fixture: &Htj2kRgbaFixture) -> Vec { + let pixels = match &fixture.samples { + Htj2kRgbaSamples::U8(samples) => samples.clone(), + Htj2kRgbaSamples::U16(samples) => samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect(), + Htj2kRgbaSamples::I16(samples) => samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect(), + }; + let codestream = j2k_native::encode( + &pixels, + fixture.width, + fixture.height, + 4, + fixture.bit_depth, + fixture.signed, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + use_mct: fixture.use_mct, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode classic RGBA fixture"); + wrap_rgba_container( + &codestream, + fixture.alpha, + J2kFileWrapOptions::jp2(), + "wrap classic RGBA image", + ) +} + +fn assert_rgba_encoding(codec: &str, encoded: &Arc<[u8]>, requests: &[DecodeRequest]) { + for layout in [BatchLayout::Nchw, BatchLayout::Nhwc] { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let inputs = requests + .iter() + .copied() + .map(|request| EncodedImage::new(encoded.clone(), request)) + .collect::>(); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU RGBA oracle"); + assert!( + expected.errors().is_empty(), + "{codec} CPU RGBA oracle errors: {:?}", + expected.errors() + ); + + let mut decoder = MetalBatchDecoder::system_default_with_options(options) + .expect("persistent Metal decoder"); + let prepared = decoder + .prepare(inputs) + .expect("prepare Metal RGBA request matrix"); + assert!( + prepared.errors().is_empty(), + "{codec} prepare errors: {:?}", + prepared.errors() + ); + + for group in prepared.groups() { + assert_eq!(group.info().color, BatchColor::Rgba); + assert_eq!(group.info().alpha, BatchAlpha::Straight); + let expected_group = expected + .groups() + .iter() + .find(|expected| expected.source_indices() == group.source_indices()) + .expect("matching CPU RGBA group"); + let fmt = rgba_format(group.info().sample_type); + let bytes_per_sample = fmt.bytes_per_sample(); + let (width, height) = group.info().dimensions; + let row_bytes = width as usize * 4 * bytes_per_sample; + let image_bytes = row_bytes * height as usize; + let output_bytes = image_bytes * group.images().len(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + output_bytes + 8, + ) + .expect("RGBA destination buffer"); + let destination_layout = MetalImageLayout::new_batch( + 4, + (width, height), + row_bytes, + fmt, + group.images().len(), + image_bytes, + ) + .expect("RGBA destination layout"); + // SAFETY: this fresh output range remains exclusively retained by + // the pending codec submission until completion. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), destination_layout) + .expect("RGBA destination") + }; + decoder + .submit_prepared_group_into(group, destination) + .expect("submit prepared RGBA group") + .wait() + .expect("complete prepared RGBA group"); + + // SAFETY: codec completion released exclusive output access. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, output_bytes) + .expect("RGBA output samples") + }; + assert_native_samples(&actual, expected_group.samples()); + } + } +} + +#[test] +fn prepared_rgba_matches_cpu_for_codecs_native_types_requests_and_layouts() { + if !should_run_metal_runtime() { + return; + } + + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + + for profile in [ + Htj2kRgbaSampleProfile::U8Rct, + Htj2kRgbaSampleProfile::U12, + Htj2kRgbaSampleProfile::I16, + ] { + let fixture = generated_htj2k_rgba_fixture(profile, Htj2kRgbaAlpha::Straight); + let htj2k = Arc::<[u8]>::from(wrap_rgba_jph(&fixture.encoded, fixture.alpha)); + let classic = Arc::<[u8]>::from(wrap_classic_rgba_jp2(&fixture)); + assert_rgba_encoding("HTJ2K", &htj2k, &requests); + assert_rgba_encoding("classic JPEG 2000", &classic, &requests); + } +} + +#[test] +fn prepared_htj2k_rgba_nhwc_resident_group_is_exact_and_uses_one_allocation() { + if !should_run_metal_runtime() { + return; + } + + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let fixture = + generated_htj2k_rgba_fixture(Htj2kRgbaSampleProfile::U8Rct, Htj2kRgbaAlpha::Straight); + let encoded = Arc::<[u8]>::from(wrap_rgba_jph(&fixture.encoded, fixture.alpha)); + let inputs = vec![ + EncodedImage::full(encoded.clone()), + EncodedImage::full(encoded), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(inputs.clone()) + .expect("CPU resident RGBA oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("U8 RGBA fixture must use U8 batch storage") + }; + + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(inputs) + .expect("prepare resident RGBA group"); + let result = decoder + .decode_prepared(&prepared) + .expect("decode resident RGBA group"); + assert!(result.errors().is_empty()); + assert!(result.group_errors().is_empty()); + assert_eq!(result.groups().len(), 1); + let group = &result.groups()[0]; + assert_eq!(group.info().color, BatchColor::Rgba); + assert_eq!(group.info().layout, BatchLayout::Nhwc); + assert_eq!(group.surfaces().len(), 2); + let image_bytes = expected.len() / group.surfaces().len(); + for (index, surface) in group.surfaces().iter().enumerate() { + assert_eq!(surface.pixel_format(), PixelFormat::Rgba8); + assert_eq!( + surface.as_bytes().expect("resident RGBA bytes").as_ref(), + &expected[index * image_bytes..(index + 1) * image_bytes] + ); + } + let first_range = group.surfaces()[0] + .memory_range() + .expect("first resident RGBA range"); + let second_range = group.surfaces()[1] + .memory_range() + .expect("second resident RGBA range"); + assert_eq!(first_range.allocation, second_range.allocation); + assert_eq!(first_range.offset, 0); + assert_eq!(second_range.offset, image_bytes); + assert_eq!(first_range.len, image_bytes); + assert_eq!(second_range.len, image_bytes); +} + +#[test] +fn prepared_htj2k_rgba_nchw_resident_group_is_exact_without_surface_mislabeling() { + if !should_run_metal_runtime() { + return; + } + + let options = BatchDecodeOptions { + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let fixture = + generated_htj2k_rgba_fixture(Htj2kRgbaSampleProfile::U8Rct, Htj2kRgbaAlpha::Straight); + let encoded = Arc::<[u8]>::from(wrap_rgba_jph(&fixture.encoded, fixture.alpha)); + let inputs = vec![ + EncodedImage::full(encoded.clone()), + EncodedImage::full(encoded), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(inputs.clone()) + .expect("CPU NCHW resident RGBA oracle"); + + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(inputs) + .expect("prepare NCHW resident RGBA group"); + let result = decoder + .decode_prepared(&prepared) + .expect("decode NCHW resident RGBA group"); + assert!(result.errors().is_empty()); + assert!(result.group_errors().is_empty()); + assert_eq!(result.groups().len(), 1); + let group = &result.groups()[0]; + assert_eq!(group.info().color, BatchColor::Rgba); + assert_eq!(group.info().layout, BatchLayout::Nchw); + assert!( + group.surfaces().is_empty(), + "planar RGBA bytes must not be mislabeled as interleaved Surface values" + ); + let resident = group + .resident_batch() + .expect("NCHW RGBA group must retain its dense allocation"); + assert_eq!(resident.image_count(), 2); + assert_eq!(resident.image_stride_bytes(), resident.byte_len() / 2); + assert_native_samples( + &completed_resident_batch_bytes(group), + expected.groups()[0].samples(), + ); +} diff --git a/crates/j2k-metal/tests/device.rs b/crates/j2k-metal/tests/device.rs index dafe6edb..1314b148 100644 --- a/crates/j2k-metal/tests/device.rs +++ b/crates/j2k-metal/tests/device.rs @@ -2,7 +2,10 @@ use std::sync::Arc; -use j2k::J2kContext; +use j2k::{ + BatchCodecRoute, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, + DecodeRequest, EncodedImage, J2kContext, NativeSampleType, PreparationDepth, +}; use j2k_core::{ BackendKind, BackendRequest, CodecError, DeviceSubmission, DeviceSurface, Downscale, ImageDecode, ImageDecodeDevice, PixelFormat, Rect, TileBatchDecodeDevice, @@ -10,7 +13,8 @@ use j2k_core::{ }; use j2k_metal::{ Codec, DecodeOperation, Error, J2kDecoder, J2kScratchPool, MetalBackendSession, - MetalDecodeRequest, MetalSession, MetalTileBatch, Surface, SurfaceResidency, + MetalBatchDecoder, MetalDecodeRequest, MetalImageDestination, MetalImageLayout, MetalSession, + MetalTileBatch, Surface, SurfaceResidency, }; use j2k_native::{encode, encode_htj2k, EncodeOptions}; @@ -39,12 +43,44 @@ fn should_run_metal_runtime() -> bool { j2k_test_support::metal_runtime_gate(module_path!()) } +fn unsupported_classic_roi_rgb() -> Arc<[u8]> { + let pixels = (0..4_u8) + .flat_map(|index| [index * 17, index * 29 + 3, index * 41 + 5]) + .collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + roi_component_shifts: vec![3, 0, 0], + ..EncodeOptions::default() + }; + Arc::from( + encode(&pixels, 2, 2, 3, 8, false, &options) + .expect("encode classic RGB8 with unsupported RGN maxshift"), + ) +} + fn completed_surface_metal_buffer(surface: &Surface) -> Option<(&metal::Buffer, usize)> { // SAFETY: Every surface passed by these tests has completed its decode, and // the tests never submit a writer or mutate a returned handle. unsafe { surface.metal_buffer() } } +fn completed_resident_batch_bytes(group: &j2k_metal::MetalBatchGroup) -> Vec { + let resident = group + .resident_batch() + .expect("codec-owned decode must expose its dense resident allocation"); + // SAFETY: the group is returned only after codec completion and this test + // performs a readback without submitting a writer or retaining the handle. + unsafe { + j2k_metal_support::checked_buffer_read_vec::( + resident.metal_buffer(), + resident.byte_offset(), + resident.byte_len(), + ) + .expect("read completed dense resident batch") + } +} + fn assert_unsupported_rgba16_report(result: Result) { match result { Err(Error::UnsupportedMetalRequest { reason }) => { @@ -128,6 +164,28 @@ fn fixture_rgb8_sized(width: u32, height: u32) -> Vec { encode(&pixels, width, height, 3, 8, false, &options).expect("encode sized rgb8") } +fn fixture_classic_multitile_rgb8() -> (Vec, Vec) { + let width = 19_u32; + let height = 13_u32; + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 17 + y * 13 + 3) & 0xFF) as u8); + pixels.push(((x * 7 + y * 29 + 41) & 0xFF) as u8); + pixels.push(((x * 31 + y * 5 + 97) & 0xFF) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + tile_size: Some((11, 7)), + ..EncodeOptions::default() + }; + let encoded = encode(&pixels, width, height, 3, 8, false, &options) + .expect("encode odd-edge multi-tile classic RGB8"); + (encoded, pixels) +} + fn fixture_ht_gray8_unsupported_direct_width() -> Vec { let width = 512u32; let height = 8u32; @@ -187,6 +245,47 @@ fn fixture_ht_gray12_offset(offset: u16) -> Vec { encode_htj2k(&pixels, 4, 4, 1, 12, false, &options).expect("encode ht gray12") } +fn fixture_ht_signed_gray12(offset: i16) -> (Vec, Vec) { + let samples = (0..16_i16) + .map(|index| -1_700 + offset + index * 197) + .collect::>(); + assert!(samples + .iter() + .all(|sample| (-2_048..=2_047).contains(sample))); + let pixels = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + ( + encode_htj2k(&pixels, 4, 4, 1, 12, true, &options).expect("encode signed ht gray12"), + samples, + ) +} + +fn fixture_classic_signed_gray12() -> (Vec, Vec) { + let samples = (0..16_i16) + .map(|index| -1_700 + index * 197) + .collect::>(); + let pixels = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + ( + encode(&pixels, 4, 4, 1, 12, true, &options).expect("encode signed classic gray12"), + samples, + ) +} + fn fixture_gray8_irreversible() -> Vec { let pixels: Vec = (0..16).collect(); let options = EncodeOptions { @@ -230,6 +329,64 @@ fn fixture_ht_gray8_reversed() -> Vec { encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode reversed ht gray8") } +fn fixture_ht_gray8_offset_sized(width: u32, height: u32, offset: u8) -> Vec { + let pixels = (0..width * height) + .map(|index| offset.wrapping_add(index.to_le_bytes()[0].wrapping_mul(17))) + .collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 1, 8, false, &options).expect("encode offset HT gray8") +} + +fn fixture_ht_rgb_u8_sized(width: u32, height: u32, offset: u8) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + for pattern in [x * 3 + y * 5, x * 7 + y * 11 + 13, x * 17 + y * 19 + 29] { + let pattern = u8::try_from(pattern & 0x3f).expect("six-bit RGB fixture pattern"); + pixels.push(offset.wrapping_add(pattern) & 0x3f); + } + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + guard_bits: 2, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 3, 6, false, &options) + .expect("encode sub-native HT RGB U8") +} + +fn fixture_ht_rgb_u16_sized(width: u32, height: u32, offset: u16) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3 * 2); + for y in 0..height { + for x in 0..width { + for pattern in [ + x * 193 + y * 257, + x * 313 + y * 97 + 31, + x * 71 + y * 401 + 63, + ] { + let pattern = + u16::try_from(pattern & 0x07ff).expect("twelve-bit RGB fixture pattern"); + let sample = offset.wrapping_add(pattern); + pixels.extend_from_slice(&(sample & 0x0fff).to_le_bytes()); + } + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + guard_bits: 2, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 3, 12, false, &options) + .expect("encode sub-native HT RGB U16") +} + fn fixture_direct_rgb8() -> Vec { fixture_direct_rgb8_offset(0) } @@ -266,2308 +423,29 @@ fn fixture_direct_rgb8_variant(seed: u8) -> Vec { encode(&pixels, 8, 8, 3, 8, false, &options).expect("encode direct rgb8 variant") } -#[test] -fn full_classic_grayscale_decode_to_metal_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 16]; - host_decoder - .decode_into(&mut host, 4, PixelFormat::Gray8) - .expect("host decode"); - - let surface = decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) - .expect("device decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (4, 4)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn full_htj2k_decode_to_metal_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_ht_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 16]; - host_decoder - .decode_into(&mut host, 4, PixelFormat::Gray8) - .expect("host decode"); - - let surface = decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) - .expect("device decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (4, 4)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn htj2k_direct_decode_clears_reused_classic_scratch_buffers() { - if !should_run_metal_runtime() { - return; - } - - let classic_bytes = fixture_gray8(); - let mut classic_decoder = J2kDecoder::new(&classic_bytes).expect("classic decoder"); - let classic_surface = classic_decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) - .expect("classic device decode"); - assert_eq!(classic_surface.backend_kind(), BackendKind::Metal); - - let bytes = fixture_ht_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 16]; - host_decoder - .decode_into(&mut host, 4, PixelFormat::Gray8) - .expect("host decode"); - - let surface = decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) - .expect("device decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (4, 4)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn full_irreversible_j2k_decode_to_metal_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8_irreversible(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 16]; - host_decoder - .decode_into(&mut host, 4, PixelFormat::Gray8) - .expect("host decode"); - - let surface = decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) - .expect("device decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (4, 4)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn auto_full_grayscale_prefers_cpu_for_small_classic_fixture() { - let bytes = fixture_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Auto) - .expect("auto decode"); - assert_eq!(surface.backend_kind(), BackendKind::Cpu); -} - -#[test] -fn auto_full_htj2k_prefers_cpu_for_small_fixture() { - let bytes = fixture_ht_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Auto) - .expect("auto decode"); - assert_eq!(surface.backend_kind(), BackendKind::Cpu); -} - -#[test] -fn auto_repeated_grayscale_keeps_short_512_batch_on_cpu() { - let bytes = fixture_gray8_sized(512, 512); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surfaces = decoder - .decode_repeated_grayscale_auto_to_device(PixelFormat::Gray8, 8) - .expect("auto repeated decode"); - assert_eq!(surfaces.len(), 8); - assert!(surfaces - .iter() - .all(|surface| surface.backend_kind() == BackendKind::Cpu)); -} - -#[test] -fn auto_repeated_grayscale_uses_metal_for_512_batch() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8_sized(512, 512); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surfaces = decoder - .decode_repeated_grayscale_auto_to_device(PixelFormat::Gray8, 16) - .expect("auto repeated decode"); - assert_eq!(surfaces.len(), 16); - assert!(surfaces - .iter() - .all(|surface| surface.backend_kind() == BackendKind::Metal)); -} - -#[test] -fn tile_full_grayscale_device_path_uses_metal_direct() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let mut ctx = J2kContext::default(); - let mut pool = J2kScratchPool::new(); - let surface = Codec::decode_tile_to_device( - &mut ctx, - &mut pool, - &bytes, - PixelFormat::Gray8, - BackendRequest::Metal, - ) - .expect("tile surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (4, 4)); -} - -#[test] -fn metal_surface_exposes_buffer_for_on_device_consumers() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let mut metal_decoder = J2kDecoder::new(&bytes).expect("metal decoder"); - let metal_surface = metal_decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) - .expect("metal surface"); - let (buffer, byte_offset) = - completed_surface_metal_buffer(&metal_surface).expect("metal buffer"); - assert_eq!(byte_offset, 0); - let buffer_len = usize::try_from(buffer.length()).expect("metal buffer length fits usize"); - assert!(buffer_len >= metal_surface.byte_len()); - - let mut cpu_decoder = J2kDecoder::new(&bytes).expect("cpu decoder"); - let cpu_surface = cpu_decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Cpu) - .expect("cpu surface"); - assert!(completed_surface_metal_buffer(&cpu_surface).is_none()); -} - -#[test] -fn metal_encoded_raw_parts_validate_ranges_and_support_consuming_handoff() { - use metal::foreign_types::ForeignType; - - if !should_run_metal_runtime() { - return; - } - - let Some(device) = metal::Device::system_default() else { - j2k_test_support::metal_device_unavailable_is_skip(module_path!()); - return; - }; - let invalid_buffer = - j2k_metal_support::checked_shared_buffer(&device, 64).expect("test buffer allocation"); - // SAFETY: This fresh allocation has no prior or concurrent writers and is - // retained only for this constructor call. - let invalid = unsafe { - j2k_metal::MetalEncodedJ2k::from_raw_parts(invalid_buffer, 16..32, 64, (4, 4), 1, 8, false) - }; - assert!(matches!( - invalid, - Err(Error::MetalKernel { message }) if message.contains("exceeds allocation length") - )); - - let buffer = - j2k_metal_support::checked_shared_buffer(&device, 64).expect("test buffer allocation"); - let expected_ptr = buffer.as_ptr(); - // SAFETY: This fresh allocation has no writers and stays immutable until - // the encoded object is consumed below. - let encoded = unsafe { - j2k_metal::MetalEncodedJ2k::from_raw_parts(buffer, 8..24, 32, (4, 4), 1, 8, false) - } - .expect("valid raw Metal codestream parts"); - assert_eq!(encoded.byte_offset(), 8); - assert_eq!(encoded.byte_len(), 16); - assert_eq!(encoded.capacity(), 32); - assert_eq!(encoded.dimensions(), (4, 4)); - assert_eq!(encoded.components(), 1); - assert_eq!(encoded.bit_depth(), 8); - assert!(!encoded.is_signed()); - // SAFETY: This encoded descriptor is the allocation's only owner and no - // sibling descriptor or cloned handle exists. - let handed_off = unsafe { encoded.into_codestream_buffer() }; - assert_eq!(handed_off.as_ptr(), expected_ptr); -} - -#[cfg(target_os = "macos")] -#[test] -fn decode_to_device_with_session_uses_session_device() { - use metal::foreign_types::ForeignTypeRef; - - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let session = MetalBackendSession::system_default().expect("Metal backend session"); - let mut decoder = J2kDecoder::new(&bytes).expect("metal decoder"); - - let surface = decoder - .decode_request_to_device_with_session( - MetalDecodeRequest::full(PixelFormat::Gray8, BackendRequest::Metal), - &session, - ) - .expect("session decode"); - - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - let (buffer, _) = completed_surface_metal_buffer(&surface).expect("metal buffer"); - assert_eq!(buffer.device().as_ptr(), session.device().as_ptr()); -} - -#[cfg(target_os = "macos")] -#[test] -fn decode_scaled_to_device_with_session_supports_rgb8_resident_surface() { - use metal::foreign_types::ForeignTypeRef; - - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_rgb8_sized(8, 8); - let scale = Downscale::Half; - let scaled = Rect { - x: 0, - y: 0, - w: 8, - h: 8, - } - .scaled_covering(scale); - let session = MetalBackendSession::system_default().expect("Metal backend session"); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut pool = J2kScratchPool::new(); - let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let mut host = vec![0u8; stride * scaled.h as usize]; - host_decoder - .decode_scaled_into(&mut pool, &mut host, stride, PixelFormat::Rgb8, scale) - .expect("host scaled RGB8 decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("metal decoder"); - let surface = decoder - .decode_request_to_device_with_session( - MetalDecodeRequest::scaled(PixelFormat::Rgb8, scale, BackendRequest::Metal), - &session, - ) - .expect("session scaled RGB8 decode"); - - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - let (buffer, _) = completed_surface_metal_buffer(&surface).expect("metal buffer"); - assert_eq!(buffer.device().as_ptr(), session.device().as_ptr()); -} - -#[cfg(target_os = "macos")] -#[test] -fn explicit_cpu_staged_metal_api_uses_session_device_and_marks_residency() { - use metal::foreign_types::ForeignTypeRef; - - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_rgb8(); - let session = MetalBackendSession::system_default().expect("Metal backend session"); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 12]; - host_decoder - .decode_into(&mut host, 6, PixelFormat::Rgb8) - .expect("host decode"); - - let surface = decoder - .decode_request_to_cpu_staged_metal_surface_with_session( - MetalDecodeRequest::full(PixelFormat::Rgb8, BackendRequest::Metal), - &session, - ) - .expect("CPU-staged Metal surface"); - - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::CpuStagedMetalUpload); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - let (buffer, byte_offset) = completed_surface_metal_buffer(&surface).expect("Metal buffer"); - assert_eq!(byte_offset, 0); - assert_eq!(buffer.device().as_ptr(), session.device().as_ptr()); -} - -#[cfg(target_os = "macos")] -#[test] -fn decode_to_device_with_session_unsupported_rgba16_is_rejected() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_rgb12(); - let session = MetalBackendSession::system_default().expect("Metal backend session"); - let mut decoder = J2kDecoder::new(&bytes).expect("metal decoder"); - - let result = decoder.decode_request_to_device_with_session( - MetalDecodeRequest::full(PixelFormat::Rgba16, BackendRequest::Metal), - &session, - ); - - match result { - Err(Error::UnsupportedMetalRequest { reason }) => { - assert_eq!(reason, UNSUPPORTED_RGBA16_REASON); - } - Err(other) => panic!("unexpected explicit Metal session error: {other:?}"), - Ok(surface) => panic!( - "explicit Metal session request must not fall back; got {:?}", - surface.backend_kind() - ), - } -} - -#[test] -fn submitted_full_grayscale_tiles_flush_as_one_device_batch() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - - let submissions = (0..3) - .map(|_| { - Codec::submit_tile_to_device( - &mut ctx, - &mut session, - &mut pool, - &bytes, - PixelFormat::Gray8, - BackendRequest::Metal, - ) - .expect("submit tile") - }) - .collect::>(); - - assert_eq!( - session.submissions().expect("session submissions"), - 0, - "submitted tile surfaces should stay queued until a wait flushes the session" - ); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 16]; - host_decoder - .decode_into(&mut host, 4, PixelFormat::Gray8) - .expect("host decode"); - - for submission in submissions { - let surface = submission.wait().expect("surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "compatible queued grayscale tiles should flush through one repeated Metal batch" - ); -} - -#[test] -fn submitted_auto_512_grayscale_tiles_flush_as_one_metal_batch() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8_sized(512, 512); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - - let submissions = (0..16) - .map(|_| { - Codec::submit_tile_to_device( - &mut ctx, - &mut session, - &mut pool, - &bytes, - PixelFormat::Gray8, - BackendRequest::Auto, - ) - .expect("submit auto tile") - }) - .collect::>(); - - assert_eq!( - session.submissions().expect("session submissions"), - 0, - "auto submitted tile surfaces should stay queued until a wait flushes the session" - ); - - for submission in submissions { - let surface = submission.wait().expect("surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (512, 512)); - } - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "compatible auto grayscale tiles should flush through one repeated Metal batch" - ); -} - -#[test] -fn submitted_distinct_full_grayscale_tiles_flush_as_one_device_batch() { - if !should_run_metal_runtime() { - return; - } - - let classic_bytes = fixture_gray8(); - let reversed_bytes = fixture_gray8_reversed(); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - - let classic_submission = Codec::submit_tile_to_device( - &mut ctx, - &mut session, - &mut pool, - &classic_bytes, - PixelFormat::Gray8, - BackendRequest::Metal, - ) - .expect("submit classic tile"); - let reversed_submission = Codec::submit_tile_to_device( - &mut ctx, - &mut session, - &mut pool, - &reversed_bytes, - PixelFormat::Gray8, - BackendRequest::Metal, - ) - .expect("submit reversed tile"); - - assert_eq!( - session.submissions().expect("session submissions"), - 0, - "distinct submitted tile surfaces should stay queued until wait" - ); - - let mut classic_host_decoder = J2kDecoder::new(&classic_bytes).expect("classic host decoder"); - let mut classic_host = [0u8; 16]; - classic_host_decoder - .decode_into(&mut classic_host, 4, PixelFormat::Gray8) - .expect("classic host decode"); - - let mut reversed_host_decoder = - J2kDecoder::new(&reversed_bytes).expect("reversed host decoder"); - let mut reversed_host = [0u8; 16]; - reversed_host_decoder - .decode_into(&mut reversed_host, 4, PixelFormat::Gray8) - .expect("reversed host decode"); - - let classic_surface = classic_submission.wait().expect("classic surface"); - let reversed_surface = reversed_submission.wait().expect("reversed surface"); - assert_eq!(classic_surface.backend_kind(), BackendKind::Metal); - assert_eq!(reversed_surface.backend_kind(), BackendKind::Metal); - assert_eq!( - classic_surface.as_bytes().expect("surface byte access"), - classic_host.as_slice() - ); - assert_eq!( - reversed_surface.as_bytes().expect("surface byte access"), - reversed_host.as_slice() - ); - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "distinct queued grayscale tiles should flush through one Metal command buffer" - ); -} - -#[test] -fn submitted_full_rgb_tiles_flush_as_one_device_batch() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_direct_rgb8(); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - - let submissions = (0..3) - .map(|_| { - Codec::submit_tile_to_device( - &mut ctx, - &mut session, - &mut pool, - &bytes, - PixelFormat::Rgb8, - BackendRequest::Metal, - ) - .expect("submit rgb tile") - }) - .collect::>(); - - assert_eq!( - session.submissions().expect("session submissions"), - 0, - "submitted RGB tile surfaces should stay queued until a wait flushes the session" - ); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 12]; - host_decoder - .decode_into(&mut host, 6, PixelFormat::Rgb8) - .expect("host decode"); - - for submission in submissions { - let surface = submission.wait().expect("surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "compatible queued RGB tiles should flush through one Metal batch" - ); -} - -#[test] -fn submitted_distinct_full_rgb_tiles_stay_resident_when_batch_route_falls_back() { - if !should_run_metal_runtime() { - return; - } - - let rgb_tiles = [ - fixture_direct_rgb8_variant(0), - fixture_direct_rgb8_variant(5), - fixture_direct_rgb8_variant(11), - ]; - assert_ne!(rgb_tiles[0], rgb_tiles[1], "RGB batch fixtures must differ"); - assert_ne!(rgb_tiles[1], rgb_tiles[2], "RGB batch fixtures must differ"); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - - let submissions = rgb_tiles - .iter() - .map(|bytes| { - Codec::submit_tile_to_device( - &mut ctx, - &mut session, - &mut pool, - bytes, - PixelFormat::Rgb8, - BackendRequest::Metal, - ) - .expect("submit distinct rgb tile") - }) - .collect::>(); - - assert_eq!( - session.submissions().expect("session submissions"), - 0, - "distinct RGB tile surfaces should stay queued until a wait flushes the session" - ); - - let expected = rgb_tiles - .iter() - .map(|bytes| { - let mut host_decoder = J2kDecoder::new(bytes).expect("host decoder"); - let stride = 8 * 3; - let mut host = vec![0u8; stride * 8]; - host_decoder - .decode_into(&mut host, stride, PixelFormat::Rgb8) - .expect("host decode"); - host - }) - .collect::>(); - - let mut surfaces = Vec::with_capacity(submissions.len()); - for (submission, host) in submissions.into_iter().zip(expected) { - let surface = submission.wait().expect("surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - surfaces.push(surface); - } - assert!( - session.submissions().expect("session submissions") >= 1, - "queued RGB tiles should submit at least one resident Metal decode" - ); - for surface in surfaces { - assert!(completed_surface_metal_buffer(&surface).is_some()); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - } -} - -#[test] -fn metal_tile_batch_decodes_submitted_tiles_in_order() { - if !should_run_metal_runtime() { - return; - } - - let classic_bytes = fixture_gray8(); - let reversed_bytes = fixture_gray8_reversed(); - let mut batch = MetalTileBatch::new(); - - assert!(batch.is_empty()); - assert_eq!( - batch - .push_tile_request( - &classic_bytes, - MetalDecodeRequest::full(PixelFormat::Gray8, BackendRequest::Metal) - ) - .expect("push classic tile"), - 0 - ); - assert_eq!( - batch - .push_shared_tile_request( - Arc::<[u8]>::from(reversed_bytes.as_slice()), - MetalDecodeRequest::full(PixelFormat::Gray8, BackendRequest::Metal) - ) - .expect("push reversed tile"), - 1 - ); - assert_eq!(batch.len(), 2); - assert_eq!(batch.submissions().expect("batch submissions"), 0); - - let surfaces = batch.decode_all().expect("batch decode"); - assert_eq!(surfaces.len(), 2); - assert_eq!(surfaces[0].backend_kind(), BackendKind::Metal); - assert_eq!(surfaces[1].backend_kind(), BackendKind::Metal); - - let mut classic_host_decoder = J2kDecoder::new(&classic_bytes).expect("classic host decoder"); - let mut classic_host = [0u8; 16]; - classic_host_decoder - .decode_into(&mut classic_host, 4, PixelFormat::Gray8) - .expect("classic host decode"); - - let mut reversed_host_decoder = - J2kDecoder::new(&reversed_bytes).expect("reversed host decoder"); - let mut reversed_host = [0u8; 16]; - reversed_host_decoder - .decode_into(&mut reversed_host, 4, PixelFormat::Gray8) - .expect("reversed host decode"); - - assert_eq!( - surfaces[0].as_bytes().expect("surface byte access"), - classic_host.as_slice() - ); - assert_eq!( - surfaces[1].as_bytes().expect("surface byte access"), - reversed_host.as_slice() - ); -} - -#[test] -fn tile_batch_decode_many_device_preserves_full_tile_order() { - if !should_run_metal_runtime() { - return; - } - - let classic_bytes = fixture_gray8(); - let reversed_bytes = fixture_gray8_reversed(); - let mut ctx = J2kContext::default(); - let mut pool = J2kScratchPool::new(); - let inputs = [classic_bytes.as_slice(), reversed_bytes.as_slice()]; - - let surfaces = Codec::decode_tiles_to_device( - &mut ctx, - &mut pool, - &inputs, - PixelFormat::Gray8, - BackendRequest::Metal, - ) - .expect("decode full-tile batch"); - - let mut classic_host_decoder = J2kDecoder::new(&classic_bytes).expect("classic host decoder"); - let mut classic_host = [0u8; 16]; - classic_host_decoder - .decode_into(&mut classic_host, 4, PixelFormat::Gray8) - .expect("classic host decode"); - - let mut reversed_host_decoder = - J2kDecoder::new(&reversed_bytes).expect("reversed host decoder"); - let mut reversed_host = [0u8; 16]; - reversed_host_decoder - .decode_into(&mut reversed_host, 4, PixelFormat::Gray8) - .expect("reversed host decode"); - - assert_eq!(surfaces.len(), 2); - assert_eq!(surfaces[0].backend_kind(), BackendKind::Metal); - assert_eq!(surfaces[1].backend_kind(), BackendKind::Metal); - assert_eq!( - surfaces[0].as_bytes().expect("surface byte access"), - classic_host.as_slice() - ); - assert_eq!( - surfaces[1].as_bytes().expect("surface byte access"), - reversed_host.as_slice() - ); -} - -#[test] -fn metal_tile_batch_supports_region_and_scaled_requests() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let roi = Rect { - x: 0, - y: 0, - w: 2, - h: 2, - }; - let mut batch = MetalTileBatch::with_capacity(2); - - assert_eq!( - batch - .push_tile_request( - &bytes, - MetalDecodeRequest::region(PixelFormat::Gray8, roi, BackendRequest::Metal) - ) - .expect("push region tile"), - 0 - ); - assert_eq!( - batch - .push_tile_request( - &bytes, - MetalDecodeRequest::scaled( - PixelFormat::Gray8, - Downscale::Half, - BackendRequest::Metal - ) - ) - .expect("push scaled tile"), - 1 - ); - - let surfaces = batch.decode_all().expect("batch decode"); - assert_eq!(surfaces.len(), 2); - assert_eq!(surfaces[0].dimensions(), (2, 2)); - assert_eq!(surfaces[1].dimensions(), (2, 2)); - assert_eq!(surfaces[0].backend_kind(), BackendKind::Metal); - assert_eq!(surfaces[1].backend_kind(), BackendKind::Metal); -} - -#[test] -fn metal_tile_batch_supports_region_scaled_requests() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let roi = Rect { - x: 1, - y: 0, - w: 2, - h: 3, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - let mut batch = MetalTileBatch::with_capacity(1); - - assert_eq!( - batch - .push_tile_request( - &bytes, - MetalDecodeRequest::region_scaled( - PixelFormat::Gray8, - roi, - scale, - BackendRequest::Metal - ) - ) - .expect("push region scaled tile"), - 0 - ); - - let surfaces = batch.decode_all().expect("batch decode"); - assert_eq!(surfaces.len(), 1); - assert_eq!(surfaces[0].dimensions(), (scaled.w, scaled.h)); - assert_eq!(surfaces[0].backend_kind(), BackendKind::Metal); -} - -#[test] -fn submitted_distinct_region_scaled_htj2k_grayscale_tiles_flush_as_one_device_batch() { - if !should_run_metal_runtime() { - return; - } - - let ht_bytes = fixture_ht_gray8(); - let reversed_bytes = fixture_ht_gray8_reversed(); - assert_ne!(ht_bytes, reversed_bytes, "HTJ2K batch fixtures must differ"); - let roi = Rect { - x: 1, - y: 0, - w: 2, - h: 3, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - - let ht_submission = submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &ht_bytes, - PixelFormat::Gray8, - roi, - scale, - BackendRequest::Metal, - ) - .expect("submit ht region-scaled tile"); - let reversed_submission = submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &reversed_bytes, - PixelFormat::Gray8, - roi, - scale, - BackendRequest::Metal, - ) - .expect("submit reversed ht region-scaled tile"); - - assert_eq!( - session.submissions().expect("session submissions"), - 0, - "region-scaled submitted tile surfaces should stay queued until wait" - ); - - let expected = [&ht_bytes, &reversed_bytes] - .into_iter() - .map(|bytes| { - let mut decoder = J2kDecoder::new(bytes).expect("host decoder"); - let stride = scaled.w as usize; - let mut host = vec![0u8; stride * scaled.h as usize]; - decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host region-scaled decode"); - host - }) - .collect::>(); - - let ht_surface = ht_submission.wait().expect("ht region-scaled surface"); - let reversed_surface = reversed_submission - .wait() - .expect("reversed ht region-scaled surface"); - assert_eq!(ht_surface.backend_kind(), BackendKind::Metal); - assert_eq!(reversed_surface.backend_kind(), BackendKind::Metal); - assert_eq!(ht_surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!(reversed_surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - ht_surface.as_bytes().expect("surface byte access"), - expected[0].as_slice() - ); - assert_eq!( - reversed_surface.as_bytes().expect("surface byte access"), - expected[1].as_slice() - ); - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "distinct queued HTJ2K region-scaled grayscale tiles should flush through one Metal command buffer" - ); -} - -#[test] -fn submitted_distinct_region_scaled_htj2k_gray16_tiles_flush_as_one_device_batch() { - if !should_run_metal_runtime() { - return; - } - - let first_bytes = fixture_ht_gray12_offset(0); - let second_bytes = fixture_ht_gray12_offset(37); - assert_ne!( - first_bytes, second_bytes, - "HTJ2K Gray16 batch fixtures must differ" - ); - let roi = Rect { - x: 1, - y: 0, - w: 2, - h: 3, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - - let first_submission = submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &first_bytes, - PixelFormat::Gray16, - roi, - scale, - BackendRequest::Metal, - ) - .expect("submit first ht gray16 region-scaled tile"); - let second_submission = submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &second_bytes, - PixelFormat::Gray16, - roi, - scale, - BackendRequest::Metal, - ) - .expect("submit second ht gray16 region-scaled tile"); - - let expected = [&first_bytes, &second_bytes] - .into_iter() - .map(|bytes| { - let mut decoder = J2kDecoder::new(bytes).expect("host decoder"); - let stride = scaled.w as usize * PixelFormat::Gray16.bytes_per_pixel(); - let mut host = vec![0u8; stride * scaled.h as usize]; - decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Gray16, - roi, - scale, - ) - .expect("host region-scaled gray16 decode"); - host - }) - .collect::>(); - - let first_surface = first_submission.wait().expect("first gray16 surface"); - let second_surface = second_submission.wait().expect("second gray16 surface"); - assert_eq!(first_surface.backend_kind(), BackendKind::Metal); - assert_eq!(second_surface.backend_kind(), BackendKind::Metal); - assert_eq!(first_surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!(second_surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - first_surface.as_bytes().expect("surface byte access"), - expected[0].as_slice() - ); - assert_eq!( - second_surface.as_bytes().expect("surface byte access"), - expected[1].as_slice() - ); - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "distinct queued HTJ2K region-scaled Gray16 tiles should flush through one Metal command buffer" - ); -} - -#[test] -fn submitted_auto_region_scaled_grayscale_keeps_short_batch_on_cpu() { - let bytes = fixture_gray8_sized(512, 512); - let roi = Rect { - x: 128, - y: 128, - w: 256, - h: 256, - }; - let scale = Downscale::Quarter; - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - let submissions = (0..16) - .map(|_| { - submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &bytes, - PixelFormat::Gray8, - roi, - scale, - BackendRequest::Auto, - ) - .expect("submit auto region-scaled tile") - }) - .collect::>(); - - for submission in submissions { - let surface = submission.wait().expect("auto region-scaled surface"); - assert_eq!(surface.backend_kind(), BackendKind::Cpu); - } - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "short auto ROI+scaled grayscale tile batches should use one CPU batch fallback" - ); -} - -#[test] -fn submitted_auto_region_scaled_rgb_tiles_flush_as_one_cpu_batch() { - let bytes = fixture_rgb8(); - let roi = Rect { - x: 0, - y: 0, - w: 1, - h: 1, - }; - let scale = Downscale::None; - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - let submissions = (0..3) - .map(|_| { - submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &bytes, - PixelFormat::Rgb8, - roi, - scale, - BackendRequest::Auto, - ) - .expect("submit auto RGB region-scaled tile") - }) - .collect::>(); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 3]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - 3, - PixelFormat::Rgb8, - roi, - scale, - ) - .expect("host region-scaled decode"); - - for submission in submissions { - let surface = submission.wait().expect("auto RGB region-scaled surface"); - assert_eq!(surface.backend_kind(), BackendKind::Cpu); - assert_eq!(surface.residency(), SurfaceResidency::Host); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "auto RGB ROI+scaled tile batches should flush through one CPU batch fallback" - ); -} - -#[test] -fn submitted_auto_region_scaled_grayscale_batch64_uses_one_metal_batch() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8_sized(512, 512); - let roi = Rect { - x: 128, - y: 128, - w: 256, - h: 256, - }; - let scale = Downscale::Quarter; - let scaled = roi.scaled_covering(scale); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - let submissions = (0..64) - .map(|_| { - submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &bytes, - PixelFormat::Gray8, - roi, - scale, - BackendRequest::Auto, - ) - .expect("submit auto region-scaled tile") - }) - .collect::>(); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let stride = scaled.w as usize; - let mut host = vec![0u8; stride * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host region-scaled decode"); - - for submission in submissions { - let surface = submission.wait().expect("auto region-scaled surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "large auto ROI+scaled grayscale tile batches should use one Metal batch" - ); -} - -#[test] -fn submitted_auto_region_scaled_ht_grayscale_1024_batch16_uses_one_metal_batch() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_ht_gray8_sized(1024, 1024); - let roi = Rect { - x: 128, - y: 128, - w: 512, - h: 256, - }; - let scale = Downscale::Quarter; - let scaled = roi.scaled_covering(scale); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - let submissions = (0..16) - .map(|_| { - submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &bytes, - PixelFormat::Gray8, - roi, - scale, - BackendRequest::Auto, - ) - .expect("submit auto region-scaled HT tile") - }) - .collect::>(); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let stride = scaled.w as usize; - let mut host = vec![0u8; stride * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host region-scaled decode"); - - for submission in submissions { - let surface = submission.wait().expect("auto region-scaled surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "1024-class auto HT ROI+scaled grayscale tile batches should use one Metal batch" - ); -} - -#[test] -fn submitted_auto_region_scaled_rgb_1024_batch16_uses_hybrid_metal() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_rgb8_sized(1024, 1024); - let roi = Rect { - x: 128, - y: 128, - w: 512, - h: 256, - }; - let scale = Downscale::Quarter; - let scaled = roi.scaled_covering(scale); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - let submissions = (0..16) - .map(|_| { - submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &bytes, - PixelFormat::Rgb8, - roi, - scale, - BackendRequest::Auto, - ) - .expect("submit auto region-scaled RGB tile") - }) - .collect::>(); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let mut host = vec![0u8; stride * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Rgb8, - roi, - scale, - ) - .expect("host region-scaled RGB decode"); - - for submission in submissions { - let surface = submission.wait().expect("auto region-scaled RGB surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } - assert_eq!( - session.submissions().expect("session submissions"), - 1, - "1024-class auto ROI+scaled RGB tile batches should use one resident hybrid Metal batch" - ); -} - -#[test] -fn submitted_auto_region_scaled_ht_grayscale_batch16_is_not_order_dependent() { - if !should_run_metal_runtime() { - return; - } - - let small_bytes = fixture_ht_gray8_sized(64, 64); - let large_bytes = fixture_ht_gray8_sized(1024, 1024); - let small_roi = Rect { - x: 8, - y: 8, - w: 32, - h: 32, - }; - let large_roi = Rect { - x: 128, - y: 128, - w: 512, - h: 256, - }; - let scale = Downscale::Quarter; - let large_scaled = large_roi.scaled_covering(scale); - let mut ctx = J2kContext::default(); - let mut session = MetalSession::default(); - let mut pool = J2kScratchPool::new(); - - let mut submissions = Vec::with_capacity(17); - submissions.push( - submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &small_bytes, - PixelFormat::Gray8, - small_roi, - scale, - BackendRequest::Auto, - ) - .expect("submit small leading auto region-scaled tile"), - ); - for _ in 0..16 { - submissions.push( - submit_tile_region_scaled_to_device!( - &mut ctx, - &mut session, - &mut pool, - &large_bytes, - PixelFormat::Gray8, - large_roi, - scale, - BackendRequest::Auto, - ) - .expect("submit large auto region-scaled tile"), - ); - } - - let mut host_decoder = J2kDecoder::new(&large_bytes).expect("host decoder"); - let stride = large_scaled.w as usize; - let mut host = vec![0u8; stride * large_scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Gray8, - large_roi, - scale, - ) - .expect("host region-scaled decode"); - - let mut surfaces = Vec::with_capacity(submissions.len()); - for submission in submissions { - surfaces.push(submission.wait().expect("auto region-scaled surface")); - } - assert_eq!( - surfaces[1].backend_kind(), - BackendKind::Metal, - "large 1024-class tiles should not be routed to CPU just because a small tile was submitted first" - ); - assert_eq!(surfaces[1].dimensions(), (large_scaled.w, large_scaled.h)); - assert_eq!( - surfaces[1].as_bytes().expect("surface byte access"), - host.as_slice() - ); - assert_eq!( - session.submissions().expect("session submissions"), - 2, - "auto ROI+scaled should use one Metal batch for the sixteen qualifying 1024-class tiles and leave the leading small tile on CPU" - ); -} - -#[test] -fn repeated_classic_grayscale_direct_decode_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surfaces = decoder - .decode_repeated_grayscale_direct_to_device(PixelFormat::Gray8, 3) - .expect("repeated direct decode"); - assert_eq!(surfaces.len(), 3); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 16]; - host_decoder - .decode_into(&mut host, 4, PixelFormat::Gray8) - .expect("host decode"); - - for surface in surfaces { - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } -} - -#[test] -fn repeated_ht_grayscale_direct_decode_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_ht_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surfaces = decoder - .decode_repeated_grayscale_direct_to_device(PixelFormat::Gray8, 3) - .expect("repeated direct decode"); - assert_eq!(surfaces.len(), 3); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 16]; - host_decoder - .decode_into(&mut host, 4, PixelFormat::Gray8) - .expect("host decode"); - - for surface in surfaces { - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } -} - -#[test] -fn metal_gray16_matches_host_decode_for_12bit_source() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray12(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 8]; - host_decoder - .decode_into(&mut host, 4, PixelFormat::Gray16) - .expect("host decode"); - - let surface = decoder - .decode_to_device(PixelFormat::Gray16, BackendRequest::Metal) - .expect("device decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn explicit_metal_rgb_full_tile_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let rgb8 = fixture_rgb8(); - { - let mut decoder = J2kDecoder::new(&rgb8).expect("rgb8 decoder"); - let mut host_decoder = J2kDecoder::new(&rgb8).expect("rgb8 host decoder"); - let mut host = [0u8; 12]; - host_decoder - .decode_into(&mut host, 6, PixelFormat::Rgb8) - .expect("host rgb8 decode"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Metal) - .expect("explicit Metal rgb8 decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (2, 2)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } - - { - let mut decoder = J2kDecoder::new(&rgb8).expect("rgba8 decoder"); - let mut host_decoder = J2kDecoder::new(&rgb8).expect("rgba8 host decoder"); - let mut host = [0u8; 16]; - host_decoder - .decode_into(&mut host, 8, PixelFormat::Rgba8) - .expect("host rgba8 decode"); - let surface = decoder - .decode_to_device(PixelFormat::Rgba8, BackendRequest::Metal) - .expect("explicit Metal rgba8 decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (2, 2)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } - - let rgb12 = fixture_rgb12(); - { - let mut decoder = J2kDecoder::new(&rgb12).expect("rgb12 decoder"); - let mut host_decoder = J2kDecoder::new(&rgb12).expect("rgb12 host decoder"); - let mut host = [0u8; 12]; - host_decoder - .decode_into(&mut host, 12, PixelFormat::Rgb16) - .expect("host rgb16 decode"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb16, BackendRequest::Metal) - .expect("explicit Metal rgb16 decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (2, 1)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - } -} - -#[test] -fn explicit_metal_unsupported_rgba16_full_decode_is_rejected() { - let bytes = fixture_rgb12(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - - let result = decoder.decode_to_device(PixelFormat::Rgba16, BackendRequest::Metal); - - match result { - Err(Error::UnsupportedMetalRequest { reason }) => { - assert_eq!(reason, UNSUPPORTED_RGBA16_REASON); - } - Err(other) => panic!("unexpected explicit Metal error: {other:?}"), - Ok(surface) => panic!( - "explicit Metal must not silently fall back; got {:?}", - surface.backend_kind() - ), - } -} - -#[test] -fn explicit_metal_unsupported_rgba16_error_is_codec_unsupported() { - let bytes = fixture_rgb12(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let err = match decoder.decode_to_device(PixelFormat::Rgba16, BackendRequest::Metal) { - Err(err) => err, - Ok(surface) => panic!( - "explicit Metal must not silently fall back; got {:?}", - surface.backend_kind() - ), - }; - - assert!(err.is_unsupported()); -} - -#[test] -fn auto_decode_report_explains_cpu_fallback_and_residency() { - let bytes = fixture_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - - let reported = decoder - .decode_request_to_device_with_report(MetalDecodeRequest::full( - PixelFormat::Gray8, - BackendRequest::Auto, - )) - .expect("reported Auto decode"); - - assert_eq!(reported.surface.backend_kind(), BackendKind::Cpu); - assert_eq!(reported.surface.residency(), SurfaceResidency::Host); - assert!(completed_surface_metal_buffer(&reported.surface).is_none()); - assert_eq!(reported.report.operation, DecodeOperation::Full); - assert_eq!(reported.report.requested_backend, BackendRequest::Auto); - assert_eq!(reported.report.selected_backend, BackendKind::Cpu); - assert_eq!(reported.report.pixel_format, PixelFormat::Gray8); - assert_eq!(reported.report.surface_residency, SurfaceResidency::Host); - assert_eq!( - reported.report.fallback_reason, - Some(AUTO_DECODE_CPU_FALLBACK_REASON) - ); -} - -#[test] -fn explicit_metal_decode_report_records_resident_surface() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - - let reported = decoder - .decode_request_to_device_with_report(MetalDecodeRequest::full( - PixelFormat::Gray8, - BackendRequest::Metal, - )) - .expect("reported explicit Metal decode"); - - assert_eq!(reported.surface.backend_kind(), BackendKind::Metal); - assert_eq!( - reported.surface.residency(), - SurfaceResidency::MetalResidentDecode - ); - assert_eq!(reported.report.operation, DecodeOperation::Full); - assert_eq!(reported.report.requested_backend, BackendRequest::Metal); - assert_eq!(reported.report.selected_backend, BackendKind::Metal); - assert_eq!( - reported.report.surface_residency, - SurfaceResidency::MetalResidentDecode - ); - assert_eq!(reported.report.fallback_reason, None); -} - -#[test] -fn explicit_metal_unsupported_rgba16_report_variants_are_rejected() { - let bytes = fixture_rgb12(); - let roi = Rect { - x: 0, - y: 0, - w: 2, - h: 1, - }; - let scale = Downscale::Half; - - let mut decoder = J2kDecoder::new(&bytes).expect("full decoder"); - assert_unsupported_rgba16_report(decoder.decode_request_to_device_with_report( - MetalDecodeRequest::full(PixelFormat::Rgba16, BackendRequest::Metal), - )); - - let mut decoder = J2kDecoder::new(&bytes).expect("region decoder"); - assert_unsupported_rgba16_report(decoder.decode_request_to_device_with_report( - MetalDecodeRequest::region(PixelFormat::Rgba16, roi, BackendRequest::Metal), - )); - - let mut decoder = J2kDecoder::new(&bytes).expect("scaled decoder"); - assert_unsupported_rgba16_report(decoder.decode_request_to_device_with_report( - MetalDecodeRequest::scaled(PixelFormat::Rgba16, scale, BackendRequest::Metal), - )); - - let mut decoder = J2kDecoder::new(&bytes).expect("region scaled decoder"); - assert_unsupported_rgba16_report(decoder.decode_request_to_device_with_report( - MetalDecodeRequest::region_scaled(PixelFormat::Rgba16, roi, scale, BackendRequest::Metal), - )); -} - -#[test] -fn explicit_metal_region_and_scaled_grayscale_match_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let roi = Rect { - x: 0, - y: 0, - w: 2, - h: 2, - }; - - let mut host_region_decoder = J2kDecoder::new(&bytes).expect("host region decoder"); - let mut host_region = [0u8; 4]; - host_region_decoder - .decode_region_into( - &mut J2kScratchPool::new(), - &mut host_region, - 2, - PixelFormat::Gray8, - roi, - ) - .expect("host region decode"); - - let mut region_decoder = J2kDecoder::new(&bytes).expect("decoder"); - let region_surface = region_decoder - .decode_region_to_device(PixelFormat::Gray8, roi, BackendRequest::Metal) - .expect("explicit Metal region decode"); - assert_eq!(region_surface.backend_kind(), BackendKind::Metal); - assert_eq!(region_surface.dimensions(), (2, 2)); - assert_eq!( - region_surface.as_bytes().expect("surface byte access"), - host_region.as_slice() - ); - - let mut host_scaled_decoder = J2kDecoder::new(&bytes).expect("host scaled decoder"); - let mut host_scaled = [0u8; 4]; - host_scaled_decoder - .decode_scaled_into( - &mut J2kScratchPool::new(), - &mut host_scaled, - 2, - PixelFormat::Gray8, - Downscale::Half, - ) - .expect("host scaled decode"); - - let mut scaled_decoder = J2kDecoder::new(&bytes).expect("decoder"); - let scaled_surface = scaled_decoder - .decode_scaled_to_device(PixelFormat::Gray8, Downscale::Half, BackendRequest::Metal) - .expect("explicit Metal scaled decode"); - assert_eq!(scaled_surface.backend_kind(), BackendKind::Metal); - assert_eq!(scaled_surface.dimensions(), (2, 2)); - assert_eq!( - scaled_surface.as_bytes().expect("surface byte access"), - host_scaled.as_slice() - ); -} - -#[test] -fn explicit_metal_scaled_rgb8_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_rgb8_sized(8, 8); - let scale = Downscale::Half; - let scaled = Rect { - x: 0, - y: 0, - w: 8, - h: 8, - } - .scaled_covering(scale); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host scaled decoder"); - let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let mut host = vec![0u8; stride * scaled.h as usize]; - host_decoder - .decode_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Rgb8, - scale, - ) - .expect("host scaled RGB8 decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_scaled_to_device(PixelFormat::Rgb8, scale, BackendRequest::Metal) - .expect("explicit Metal scaled RGB8 decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn explicit_metal_region_and_scaled_htj2k_grayscale_match_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_ht_gray8(); - let roi = Rect { - x: 0, - y: 0, - w: 2, - h: 2, - }; - - let mut host_region_decoder = J2kDecoder::new(&bytes).expect("host region decoder"); - let mut host_region = [0u8; 4]; - host_region_decoder - .decode_region_into( - &mut J2kScratchPool::new(), - &mut host_region, - 2, - PixelFormat::Gray8, - roi, - ) - .expect("host region decode"); - - let mut region_decoder = J2kDecoder::new(&bytes).expect("decoder"); - let region_surface = region_decoder - .decode_region_to_device(PixelFormat::Gray8, roi, BackendRequest::Metal) - .expect("explicit Metal region decode"); - assert_eq!(region_surface.backend_kind(), BackendKind::Metal); - assert_eq!(region_surface.dimensions(), (2, 2)); - assert_eq!( - region_surface.as_bytes().expect("surface byte access"), - host_region.as_slice() - ); - - let mut host_scaled_decoder = J2kDecoder::new(&bytes).expect("host scaled decoder"); - let mut host_scaled = [0u8; 4]; - host_scaled_decoder - .decode_scaled_into( - &mut J2kScratchPool::new(), - &mut host_scaled, - 2, - PixelFormat::Gray8, - Downscale::Half, - ) - .expect("host scaled decode"); - - let mut scaled_decoder = J2kDecoder::new(&bytes).expect("decoder"); - let scaled_surface = scaled_decoder - .decode_scaled_to_device(PixelFormat::Gray8, Downscale::Half, BackendRequest::Metal) - .expect("explicit Metal scaled decode"); - assert_eq!(scaled_surface.backend_kind(), BackendKind::Metal); - assert_eq!(scaled_surface.dimensions(), (2, 2)); - assert_eq!( - scaled_surface.as_bytes().expect("surface byte access"), - host_scaled.as_slice() - ); -} - -#[test] -fn explicit_metal_region_scaled_grayscale_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8(); - let roi = Rect { - x: 1, - y: 0, - w: 2, - h: 3, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - scaled.w as usize, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host region scaled decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) - .expect("explicit Metal region scaled decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn explicit_metal_region_scaled_grayscale_large_cropped_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_gray8_sized(1024, 1024); - let roi = Rect { - x: 128, - y: 128, - w: 512, - h: 512, - }; - - for scale in [Downscale::Half, Downscale::None] { - let scaled = roi.scaled_covering(scale); - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - scaled.w as usize, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host region scaled decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) - .expect("explicit Metal region scaled decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - let surface_bytes = surface.as_bytes().expect("surface byte access"); - if surface_bytes.as_ref() != host.as_slice() { - let mismatch = surface_bytes - .iter() - .zip(&host) - .position(|(actual, expected)| actual != expected) - .expect("mismatched buffers should have a differing byte"); - panic!( - "scale={scale:?} first mismatch at byte {mismatch}: metal={} host={}", - surface_bytes[mismatch], host[mismatch] - ); - } - } -} - -#[test] -fn explicit_metal_region_scaled_htj2k_grayscale_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_ht_gray8(); - let roi = Rect { - x: 1, - y: 0, - w: 2, - h: 3, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - scaled.w as usize, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host region scaled decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) - .expect("explicit Metal region scaled decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn explicit_metal_region_scaled_htj2k_falls_back_when_direct_width_is_unsupported() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_ht_gray8_unsupported_direct_width(); - let roi = Rect { - x: 48, - y: 2, - w: 96, - h: 4, - }; - let scale = Downscale::None; - let scaled = roi.scaled_covering(scale); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - scaled.w as usize, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host region scaled decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) - .expect("explicit Metal should fall back after unsupported direct HT geometry"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn explicit_metal_region_scaled_rgb_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_direct_rgb8_variant(3); - let roi = Rect { - x: 1, - y: 2, - w: 5, - h: 4, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let mut host = vec![0u8; stride * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Rgb8, - roi, - scale, - ) - .expect("host region scaled RGB decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Rgb8, roi, scale, BackendRequest::Metal) - .expect("explicit Metal region scaled RGB decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("rgba8 host decoder"); - let stride = scaled.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); - let mut host = vec![0u8; stride * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Rgba8, - roi, - scale, - ) - .expect("host region scaled RGBA decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("rgba8 decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Rgba8, roi, scale, BackendRequest::Metal) - .expect("explicit Metal region scaled RGBA decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); - - let bytes = fixture_rgb12(); - let roi = Rect { - x: 0, - y: 0, - w: 2, - h: 1, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - let mut host_decoder = J2kDecoder::new(&bytes).expect("rgb16 host decoder"); - let stride = scaled.w as usize * PixelFormat::Rgb16.bytes_per_pixel(); - let mut host = vec![0u8; stride * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - PixelFormat::Rgb16, - roi, - scale, - ) - .expect("host region scaled RGB16 decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("rgb16 decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Rgb16, roi, scale, BackendRequest::Metal) - .expect("explicit Metal region scaled RGB16 decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - assert_eq!( - surface.as_bytes().expect("surface byte access"), - host.as_slice() - ); -} - -#[test] -fn explicit_metal_region_scaled_rgb_large_cropped_matches_host_decode() { - if !should_run_metal_runtime() { - return; - } - - let bytes = fixture_rgb8_sized(1024, 1024); - let roi = Rect { - x: 128, - y: 128, - w: 512, - h: 512, - }; - - for scale in [Downscale::Half, Downscale::None] { - let scaled = roi.scaled_covering(scale); - for fmt in [PixelFormat::Rgb8, PixelFormat::Rgba8] { - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let stride = scaled.w as usize * fmt.bytes_per_pixel(); - let mut host = vec![0u8; stride * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut host, - stride, - fmt, - roi, - scale, - ) - .expect("host region scaled RGB decode"); - - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(fmt, roi, scale, BackendRequest::Metal) - .expect("explicit Metal region scaled RGB decode"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - let surface_bytes = surface.as_bytes().expect("surface byte access"); - if surface_bytes.as_ref() != host.as_slice() { - let mismatch = surface_bytes - .iter() - .zip(&host) - .position(|(actual, expected)| actual != expected) - .expect("mismatched buffers should have a differing byte"); - panic!( - "fmt={fmt:?} scale={scale:?} first mismatch at byte {mismatch}: metal={} host={}", - surface_bytes[mismatch], - host[mismatch] - ); - } - } - } -} - -#[test] -fn auto_region_and_scaled_fallback_to_cpu_surface_and_match_host_decode() { - let bytes = fixture_rgb8(); - let roi = Rect { - x: 0, - y: 0, - w: 1, - h: 1, - }; - - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let region_surface = decoder - .decode_region_to_device(PixelFormat::Rgb8, roi, BackendRequest::Auto) - .expect("region surface"); - assert_eq!(region_surface.backend_kind(), BackendKind::Cpu); - assert!(completed_surface_metal_buffer(®ion_surface).is_none()); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut region_host = [0u8; 3]; - host_decoder - .decode_region_into( - &mut J2kScratchPool::new(), - &mut region_host, - 3, - PixelFormat::Rgb8, - roi, - ) - .expect("host region"); - assert_eq!( - region_surface.as_bytes().expect("surface byte access"), - region_host.as_slice() - ); - - let scaled_surface = decoder - .decode_scaled_to_device(PixelFormat::Rgb8, Downscale::Half, BackendRequest::Auto) - .expect("scaled surface"); - assert_eq!(scaled_surface.backend_kind(), BackendKind::Cpu); - assert!(completed_surface_metal_buffer(&scaled_surface).is_none()); - - let mut scaled_host = [0u8; 3]; - host_decoder - .decode_scaled_into( - &mut J2kScratchPool::new(), - &mut scaled_host, - 3, - PixelFormat::Rgb8, - Downscale::Half, - ) - .expect("host scaled"); - assert_eq!( - scaled_surface.as_bytes().expect("surface byte access"), - scaled_host.as_slice() - ); -} - -#[test] -fn invalid_region_reports_error_instead_of_panicking() { - let bytes = fixture_rgb8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let roi = Rect { - x: 1, - y: 1, - w: 2, - h: 2, - }; - match decoder.decode_region_to_device(PixelFormat::Rgb8, roi, BackendRequest::Auto) { - Err(Error::Decode(j2k::J2kError::InvalidRegion { .. })) => {} - Err(other) => panic!("unexpected error for invalid ROI: {other:?}"), - Ok(_) => panic!("invalid ROI should fail"), - } -} - -#[test] -fn explicit_metal_tile_unsupported_rgba16_is_rejected() { - let bytes = fixture_rgb12(); - let mut ctx = J2kContext::default(); - let mut pool = J2kScratchPool::new(); - - let result = Codec::decode_tile_to_device( - &mut ctx, - &mut pool, - &bytes, - PixelFormat::Rgba16, - BackendRequest::Metal, - ); - - match result { - Err(Error::UnsupportedMetalRequest { reason }) => { - assert_eq!(reason, UNSUPPORTED_RGBA16_REASON); - } - Err(other) => panic!("unexpected explicit Metal tile error: {other:?}"), - Ok(surface) => panic!( - "explicit Metal tile request must not fall back; got {:?}", - surface.backend_kind() - ), - } -} - -#[test] -fn hybrid_ht_cpuupload_uses_worker_local_decode_workspace() { - let source = include_str!("../src/compute/direct_cpu.rs"); - - assert!( - source.contains("decode_prepared_ht_jobs_on_cpu_with_workspace"), - "HT CPUUpload decode must expose a workspace-aware helper" - ); - assert!( - source.contains("HtCodeBlockDecodeWorkspace::default()"), - "parallel HT CPUUpload decode must initialize worker-local HT decode workspaces" - ); - assert!( - source.contains("decode_ht_code_block_scalar_with_workspace"), - "HT CPUUpload decode must call the scratch-reusing scalar helper" - ); -} +#[path = "device/auto_tile_batch.rs"] +mod auto_tile_batch; +#[path = "device/batch_sessions.rs"] +mod batch_sessions; +#[path = "device/color_batch.rs"] +mod color_batch; +#[path = "device/decode.rs"] +mod decode; +#[path = "device/direct_gray_requests.rs"] +mod direct_gray_requests; +#[path = "device/direct_repeated.rs"] +mod direct_repeated; +#[path = "device/direct_rgb_requests.rs"] +mod direct_rgb_requests; +#[path = "device/external_batch.rs"] +mod external_batch; +#[path = "device/grayscale_external.rs"] +mod grayscale_external; +#[path = "device/legacy_batch.rs"] +mod legacy_batch; +#[path = "device/multitile_color.rs"] +mod multitile_color; +#[path = "device/resident_batch.rs"] +mod resident_batch; +#[path = "device/tile_batch.rs"] +mod tile_batch; diff --git a/crates/j2k-metal/tests/device/auto_tile_batch.rs b/crates/j2k-metal/tests/device/auto_tile_batch.rs new file mode 100644 index 00000000..a4dd8727 --- /dev/null +++ b/crates/j2k-metal/tests/device/auto_tile_batch.rs @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn submitted_auto_region_scaled_grayscale_keeps_short_batch_on_cpu() { + let bytes = fixture_gray8_sized(512, 512); + let roi = Rect { + x: 128, + y: 128, + w: 256, + h: 256, + }; + let scale = Downscale::Quarter; + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + let submissions = (0..16) + .map(|_| { + submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Gray8, + roi, + scale, + BackendRequest::Auto, + ) + .expect("submit auto region-scaled tile") + }) + .collect::>(); + + for submission in submissions { + let surface = submission.wait().expect("auto region-scaled surface"); + assert_eq!(surface.backend_kind(), BackendKind::Cpu); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "short auto ROI+scaled grayscale tile batches should use one CPU batch fallback" + ); +} + +#[test] +fn submitted_auto_region_scaled_rgb_tiles_flush_as_one_cpu_batch() { + let bytes = fixture_rgb8(); + let roi = Rect { + x: 0, + y: 0, + w: 1, + h: 1, + }; + let scale = Downscale::None; + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + let submissions = (0..3) + .map(|_| { + submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Rgb8, + roi, + scale, + BackendRequest::Auto, + ) + .expect("submit auto RGB region-scaled tile") + }) + .collect::>(); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 3]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + 3, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("host region-scaled decode"); + + for submission in submissions { + let surface = submission.wait().expect("auto RGB region-scaled surface"); + assert_eq!(surface.backend_kind(), BackendKind::Cpu); + assert_eq!(surface.residency(), SurfaceResidency::Host); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "auto RGB ROI+scaled tile batches should flush through one CPU batch fallback" + ); +} + +#[test] +fn submitted_auto_region_scaled_grayscale_batch64_uses_one_metal_batch() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8_sized(512, 512); + let roi = Rect { + x: 128, + y: 128, + w: 256, + h: 256, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + let submissions = (0..64) + .map(|_| { + submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Gray8, + roi, + scale, + BackendRequest::Auto, + ) + .expect("submit auto region-scaled tile") + }) + .collect::>(); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize; + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region-scaled decode"); + + for submission in submissions { + let surface = submission.wait().expect("auto region-scaled surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "large auto ROI+scaled grayscale tile batches should use one Metal batch" + ); +} + +#[test] +fn submitted_auto_region_scaled_ht_grayscale_1024_batch16_uses_one_metal_batch() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_ht_gray8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 256, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + let submissions = (0..16) + .map(|_| { + submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Gray8, + roi, + scale, + BackendRequest::Auto, + ) + .expect("submit auto region-scaled HT tile") + }) + .collect::>(); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize; + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region-scaled decode"); + + for submission in submissions { + let surface = submission.wait().expect("auto region-scaled surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "1024-class auto HT ROI+scaled grayscale tile batches should use one Metal batch" + ); +} + +#[test] +fn submitted_auto_region_scaled_rgb_1024_batch16_uses_hybrid_metal() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_rgb8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 256, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + let submissions = (0..16) + .map(|_| { + submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Rgb8, + roi, + scale, + BackendRequest::Auto, + ) + .expect("submit auto region-scaled RGB tile") + }) + .collect::>(); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("host region-scaled RGB decode"); + + for submission in submissions { + let surface = submission.wait().expect("auto region-scaled RGB surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "1024-class auto ROI+scaled RGB tile batches should use one resident hybrid Metal batch" + ); +} + +#[test] +fn submitted_auto_region_scaled_ht_grayscale_batch16_is_not_order_dependent() { + if !should_run_metal_runtime() { + return; + } + + let small_bytes = fixture_ht_gray8_sized(64, 64); + let large_bytes = fixture_ht_gray8_sized(1024, 1024); + let small_roi = Rect { + x: 8, + y: 8, + w: 32, + h: 32, + }; + let large_roi = Rect { + x: 128, + y: 128, + w: 512, + h: 256, + }; + let scale = Downscale::Quarter; + let large_scaled = large_roi.scaled_covering(scale); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + + let mut submissions = Vec::with_capacity(17); + submissions.push( + submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &small_bytes, + PixelFormat::Gray8, + small_roi, + scale, + BackendRequest::Auto, + ) + .expect("submit small leading auto region-scaled tile"), + ); + for _ in 0..16 { + submissions.push( + submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &large_bytes, + PixelFormat::Gray8, + large_roi, + scale, + BackendRequest::Auto, + ) + .expect("submit large auto region-scaled tile"), + ); + } + + let mut host_decoder = J2kDecoder::new(&large_bytes).expect("host decoder"); + let stride = large_scaled.w as usize; + let mut host = vec![0u8; stride * large_scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Gray8, + large_roi, + scale, + ) + .expect("host region-scaled decode"); + + let mut surfaces = Vec::with_capacity(submissions.len()); + for submission in submissions { + surfaces.push(submission.wait().expect("auto region-scaled surface")); + } + assert_eq!( + surfaces[1].backend_kind(), + BackendKind::Metal, + "large 1024-class tiles should not be routed to CPU just because a small tile was submitted first" + ); + assert_eq!(surfaces[1].dimensions(), (large_scaled.w, large_scaled.h)); + assert_eq!( + surfaces[1].as_bytes().expect("surface byte access"), + host.as_slice() + ); + assert_eq!( + session.submissions().expect("session submissions"), + 2, + "auto ROI+scaled should use one Metal batch for the sixteen qualifying 1024-class tiles and leave the leading small tile on CPU" + ); +} diff --git a/crates/j2k-metal/tests/device/batch_sessions.rs b/crates/j2k-metal/tests/device/batch_sessions.rs new file mode 100644 index 00000000..83f6f5f5 --- /dev/null +++ b/crates/j2k-metal/tests/device/batch_sessions.rs @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn persistent_metal_batch_decoder_reuses_one_session_for_distinct_and_repeated_ht_batches() { + if !should_run_metal_runtime() { + return; + } + + let first = Arc::<[u8]>::from(fixture_ht_gray8()); + let second = Arc::<[u8]>::from(fixture_ht_gray8_reversed()); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let registry_id = decoder.backend_session().device().registry_id(); + + let before = decoder.submissions().expect("initial submissions"); + let distinct = decoder + .decode_batch(vec![ + EncodedImage::full(first.clone()), + EncodedImage::full(second), + ]) + .expect("distinct HT batch"); + let after_distinct = decoder.submissions().expect("distinct submissions"); + + assert!(distinct.errors().is_empty()); + assert!(distinct.group_errors().is_empty()); + assert_eq!(distinct.groups().len(), 1); + assert_eq!(distinct.groups()[0].surfaces().len(), 2); + assert!(distinct.groups()[0].surfaces().iter().all(|surface| { + surface.backend_kind() == BackendKind::Metal + && surface.residency() == SurfaceResidency::MetalResidentDecode + })); + assert_eq!( + after_distinct - before, + 1, + "distinct HT inputs must coalesce" + ); + + let repeated = decoder + .decode_batch(vec![ + EncodedImage::full(first.clone()), + EncodedImage::full(first), + ]) + .expect("repeated HT batch"); + let after_repeated = decoder.submissions().expect("repeated submissions"); + + assert!(repeated.errors().is_empty()); + assert!(repeated.group_errors().is_empty()); + assert_eq!(repeated.groups().len(), 1); + assert_eq!(repeated.groups()[0].surfaces().len(), 2); + assert_eq!( + after_repeated - after_distinct, + 1, + "repeated HT inputs must use one batch submission" + ); + assert_eq!( + decoder.backend_session().device().registry_id(), + registry_id + ); +} + +#[test] +fn persistent_metal_batch_decoder_accepts_and_reuses_shared_prepared_groups() { + if !should_run_metal_runtime() { + return; + } + + let first = Arc::<[u8]>::from(fixture_ht_gray8()); + let second = Arc::<[u8]>::from(fixture_ht_gray8_reversed()); + let options = BatchDecodeOptions { + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::new(first.clone(), DecodeRequest::Full), + EncodedImage::new(second, DecodeRequest::Full), + ]) + .expect("shared batch preparation"); + + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + assert_eq!(prepared.groups()[0].source_indices(), &[0, 1]); + assert!(Arc::ptr_eq( + prepared.groups()[0].images()[0].bytes(), + &first + )); + + let before = decoder.submissions().expect("initial submissions"); + for expected_after in [before + 1, before + 2] { + let result = decoder + .decode_prepared(&prepared) + .expect("prepared Metal batch decode"); + assert!(result.errors().is_empty()); + assert_eq!(result.groups().len(), 1); + let group = &result.groups()[0]; + assert_eq!(group.source_indices(), &[0, 1]); + assert_eq!(group.decoded_rects().len(), 2); + assert_eq!(group.surfaces().len(), 2); + let resident = group + .resident_batch() + .expect("NCHW Gray group must retain its dense allocation"); + assert_eq!(resident.image_count(), 2); + let dense = completed_resident_batch_bytes(group); + assert_eq!(resident.image_stride_bytes(), dense.len() / 2); + for (index, surface) in group.surfaces().iter().enumerate() { + let start = index * resident.image_stride_bytes(); + let end = start + resident.image_stride_bytes(); + assert_eq!( + surface.as_bytes().expect("resident Gray view").as_ref(), + &dense[start..end] + ); + } + assert_eq!(group.info().route, BatchCodecRoute::Htj2k); + assert_eq!(group.info().sample_type, NativeSampleType::U8); + assert!(group + .surfaces() + .iter() + .all(|surface| surface.residency() == SurfaceResidency::MetalResidentDecode)); + assert_eq!( + decoder.submissions().expect("completed submissions"), + expected_after + ); + } +} + +#[test] +fn submitted_shared_prepared_batch_preserves_indexed_errors_and_resident_output() { + if !should_run_metal_runtime() { + return; + } + + let malformed = Arc::<[u8]>::from(&b"not a codestream"[..]); + let valid = Arc::<[u8]>::from(fixture_ht_gray8()); + let options = BatchDecodeOptions::default(); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(vec![EncodedImage::full(valid.clone())]) + .expect("CPU submitted batch oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("gray8 fixture must use U8 batch storage") + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let submitted = decoder + .submit_batch(vec![ + EncodedImage::full(malformed), + EncodedImage::full(valid), + ]) + .expect("submit shared encoded Metal batch"); + + assert_eq!(submitted.len(), 1); + let result = submitted + .wait() + .expect("complete shared encoded Metal batch"); + assert_eq!(result.errors().len(), 1); + assert_eq!(result.errors()[0].index, 0); + assert!(result.group_errors().is_empty()); + assert_eq!(result.groups().len(), 1); + assert_eq!(result.groups()[0].source_indices(), &[1]); + assert_eq!(result.groups()[0].surfaces().len(), 1); + assert_eq!( + result.groups()[0].surfaces()[0] + .as_bytes() + .expect("submitted resident bytes"), + expected.as_slice() + ); +} + +#[test] +fn submitted_shared_batch_continues_after_nonfatal_group_submit_failure() { + if !should_run_metal_runtime() { + return; + } + + let valid_gray = Arc::<[u8]>::from(fixture_ht_gray8()); + let unsupported_roi_rgb = unsupported_classic_roi_rgb(); + let options = BatchDecodeOptions::default(); + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(valid_gray), + EncodedImage::full(unsupported_roi_rgb), + ]) + .expect("prepare mixed shared Metal batch"); + let submitted = decoder + .submit_prepared(&prepared) + .expect("submit supported shared group"); + + assert_eq!(submitted.len(), 1); + let result = submitted.wait().expect("complete supported shared group"); + assert!(result.errors().is_empty()); + assert_eq!(result.groups().len(), 1); + assert_eq!(result.groups()[0].source_indices(), &[0]); + assert_eq!(result.group_errors().len(), 1); + assert_eq!(result.group_errors()[0].source_indices(), &[1]); +} + +#[test] +fn dropped_shared_prepared_batch_retires_work_and_reuses_decoder() { + if !should_run_metal_runtime() { + return; + } + + let first = Arc::<[u8]>::from(fixture_ht_gray8()); + let second = Arc::<[u8]>::from(fixture_ht_gray8_reversed()); + let options = BatchDecodeOptions::default(); + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::full(first), EncodedImage::full(second)]) + .expect("prepare reusable shared Metal batch"); + + let dropped = decoder + .submit_prepared(&prepared) + .expect("submit dropped shared Metal batch"); + assert_eq!(dropped.len(), 1); + drop(dropped); + + let completed = decoder + .submit_prepared(&prepared) + .expect("resubmit shared Metal batch") + .wait() + .expect("complete resubmitted shared Metal batch"); + assert_eq!(completed.groups().len(), 1); + assert_eq!(completed.groups()[0].surfaces().len(), 2); +} + +#[test] +fn shared_metal_prepared_batch_decodes_classic_resident_color_without_legacy_staging() { + if !should_run_metal_runtime() { + return; + } + + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let inputs = vec![EncodedImage::full(Arc::from(fixture_rgb8()))]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU classic RGB oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("classic RGB8 must use U8 batch storage") + }; + let prepared = decoder.prepare(inputs).expect("prepare RGB batch"); + assert!(prepared.errors().is_empty()); + assert_eq!( + prepared.groups()[0].images()[0].preparation_depth(), + PreparationDepth::ClassicOffsetPlan + ); + + let result = decoder + .decode_prepared(&prepared) + .expect("classic resident RGB decode"); + assert!(result.group_errors().is_empty()); + assert_eq!(result.groups().len(), 1); + assert_eq!(result.groups()[0].surfaces().len(), 1); + assert_eq!( + result.groups()[0].surfaces()[0] + .as_bytes() + .expect("classic resident RGB bytes") + .as_ref(), + expected.as_slice() + ); +} + +#[test] +fn prepared_ht_rgb_nchw_resident_group_is_exact_without_interleaved_surface_views() { + if !should_run_metal_runtime() { + return; + } + + let options = BatchDecodeOptions { + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::full(Arc::from(fixture_ht_rgb_u8_sized(8, 8, 0))), + EncodedImage::full(Arc::from(fixture_ht_rgb_u8_sized(8, 8, 17))), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU NCHW RGB oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("six-bit RGB must use U8 batch storage") + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder.prepare(inputs).expect("prepare NCHW HT RGB batch"); + assert!(prepared.errors().is_empty()); + + let result = decoder + .decode_prepared(&prepared) + .expect("decode NCHW resident RGB group"); + assert!(result.group_errors().is_empty()); + assert_eq!(result.groups().len(), 1); + let group = &result.groups()[0]; + assert_eq!(group.info().layout, BatchLayout::Nchw); + assert_eq!(group.source_indices(), &[0, 1]); + assert!( + group.surfaces().is_empty(), + "planar RGB bytes must not be mislabeled as interleaved Surface values" + ); + let resident = group + .resident_batch() + .expect("NCHW RGB group must retain its dense allocation"); + assert_eq!(resident.image_count(), 2); + assert_eq!(resident.image_stride_bytes(), expected.len() / 2); + assert_eq!(completed_resident_batch_bytes(group), expected.as_slice()); +} diff --git a/crates/j2k-metal/tests/device/color_batch.rs b/crates/j2k-metal/tests/device/color_batch.rs new file mode 100644 index 00000000..a9cfba38 --- /dev/null +++ b/crates/j2k-metal/tests/device/color_batch.rs @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn submitted_prepared_ht_rgb_u8_stores_exact_native_nhwc_and_nchw_for_all_requests() { + if !should_run_metal_runtime() { + return; + } + + let first = Arc::<[u8]>::from(fixture_ht_rgb_u8_sized(8, 8, 0)); + let second = Arc::<[u8]>::from(fixture_ht_rgb_u8_sized(8, 8, 17)); + let roi = Rect { + x: 2, + y: 2, + w: 4, + h: 4, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + for request in requests { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::new(first.clone(), request), + EncodedImage::new(second.clone(), request), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU RGB U8 oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("six-bit RGB must use U8 batch storage") + }; + + let mut decoder = MetalBatchDecoder::system_default_with_options(options) + .expect("persistent Metal decoder"); + let prepared = decoder + .prepare(inputs) + .expect("prepare exact-native HT RGB U8 group"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + let group = &prepared.groups()[0]; + assert_eq!(group.info().sample_type, NativeSampleType::U8); + let (width, height) = group.info().dimensions; + let samples_per_image = + usize::try_from(width).unwrap() * usize::try_from(height).unwrap() * 3; + let output_len = samples_per_image * group.images().len(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + output_len + 8, + ) + .expect("RGB U8 destination buffer"); + let row_bytes = usize::try_from(width).unwrap() * 3; + let destination_layout = MetalImageLayout::new_batch( + 4, + (width, height), + row_bytes, + PixelFormat::Rgb8, + group.images().len(), + samples_per_image, + ) + .expect("dense RGB U8 destination layout"); + // SAFETY: this fresh allocation is retained exclusively by the + // pending submission until its explicit completion wait. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), destination_layout) + .expect("RGB U8 destination") + }; + decoder + .submit_prepared_group_into(group, destination) + .expect("submit exact-native HT RGB U8 group") + .wait() + .expect("complete exact-native HT RGB U8 group"); + + // SAFETY: codec completion released exclusive destination access. + let actual = + unsafe { j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, output_len) } + .expect("RGB U8 destination samples"); + assert_eq!( + actual.as_slice(), + expected.as_slice(), + "{layout:?} {request:?}" + ); + assert!( + actual.iter().all(|sample| *sample <= 0x3f), + "six-bit samples must not be scaled to the full U8 range" + ); + } + } +} + +#[test] +fn submitted_prepared_ht_rgb_u16_stores_exact_native_nhwc_and_nchw() { + if !should_run_metal_runtime() { + return; + } + + let first = Arc::<[u8]>::from(fixture_ht_rgb_u16_sized(8, 8, 0)); + let second = Arc::<[u8]>::from(fixture_ht_rgb_u16_sized(8, 8, 257)); + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::full(first.clone()), + EncodedImage::full(second.clone()), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU RGB U16 oracle"); + let CpuBatchSamples::U16(expected) = expected.groups()[0].samples() else { + panic!("twelve-bit RGB must use U16 batch storage") + }; + + let mut decoder = MetalBatchDecoder::system_default_with_options(options) + .expect("persistent Metal decoder"); + let prepared = decoder + .prepare(inputs) + .expect("prepare exact-native HT RGB U16 group"); + assert!(prepared.errors().is_empty()); + let group = &prepared.groups()[0]; + let (width, height) = group.info().dimensions; + let samples_per_image = + usize::try_from(width).unwrap() * usize::try_from(height).unwrap() * 3; + let output_len = samples_per_image * group.images().len(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + output_len * 2 + 8, + ) + .expect("RGB U16 destination buffer"); + let row_bytes = usize::try_from(width).unwrap() * 3 * 2; + let destination_layout = MetalImageLayout::new_batch( + 4, + (width, height), + row_bytes, + PixelFormat::Rgb16, + group.images().len(), + samples_per_image * 2, + ) + .expect("dense RGB U16 destination layout"); + // SAFETY: this fresh allocation remains exclusive until completion. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), destination_layout) + .expect("RGB U16 destination") + }; + decoder + .submit_prepared_group_into(group, destination) + .expect("submit exact-native HT RGB U16 group") + .wait() + .expect("complete exact-native HT RGB U16 group"); + + // SAFETY: codec completion released exclusive destination access. + let actual = + unsafe { j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, output_len) } + .expect("RGB U16 destination samples"); + assert_eq!(actual.as_slice(), expected.as_slice(), "{layout:?}"); + assert!( + actual.iter().all(|sample| *sample <= 0x0fff), + "twelve-bit samples must not be scaled to the full U16 range" + ); + } +} + +fn assert_signed_rgb_request( + name: &str, + encoded: &Arc<[u8]>, + layout: BatchLayout, + request: DecodeRequest, +) { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::new(encoded.clone(), request), + EncodedImage::new(encoded.clone(), request), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(inputs.clone()) + .unwrap_or_else(|error| panic!("{name} CPU oracle: {error}")); + let CpuBatchSamples::I16(expected) = expected.groups()[0].samples() else { + panic!("{name} must use I16 batch storage") + }; + + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(inputs) + .unwrap_or_else(|error| panic!("{name} prepare: {error}")); + assert!(prepared.errors().is_empty(), "{name}"); + assert_eq!(prepared.groups().len(), 1, "{name}"); + let group = &prepared.groups()[0]; + assert_eq!(group.info().sample_type, NativeSampleType::I16); + assert!(group + .images() + .iter() + .all(|image| image.preparation_depth() == PreparationDepth::Htj2kOffsetPlan)); + let (width, height) = group.info().dimensions; + let samples_per_image = usize::try_from(width).unwrap() * usize::try_from(height).unwrap() * 3; + let output_len = samples_per_image * group.images().len(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + output_len * 2 + 8, + ) + .expect("signed RGB destination buffer"); + let destination_layout = MetalImageLayout::new_batch( + 4, + (width, height), + usize::try_from(width).unwrap() * 3 * 2, + PixelFormat::RgbI16, + group.images().len(), + samples_per_image * 2, + ) + .expect("dense signed RGB destination layout"); + // SAFETY: this fresh allocation remains exclusively owned by the pending + // codec submission until its completion wait. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), destination_layout) + .expect("signed RGB destination") + }; + decoder + .submit_prepared_group_into(group, destination) + .unwrap_or_else(|error| panic!("{name} submit: {error}")) + .wait() + .unwrap_or_else(|error| panic!("{name} completion: {error}")); + + // SAFETY: completion released exclusive destination access. + let actual = + unsafe { j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, output_len) } + .expect("signed RGB destination samples"); + assert_eq!( + actual.as_slice(), + expected.as_slice(), + "{name} {layout:?} {request:?}" + ); +} + +#[test] +fn independent_openjph_signed_rgb_stores_exact_i16_for_all_requests_and_layouts() { + if !should_run_metal_runtime() { + return; + } + + let roi = Rect { + x: 3, + y: 2, + w: 9, + h: 7, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + let fixtures = j2k_test_support::openjph_batch_fixtures() + .iter() + .filter(|fixture| { + matches!( + fixture.name, + "openjph-rgb-s8-53-single-raw" + | "openjph-rgb-s12-53-single-raw" + | "openjph-rgb-s16-53-single-raw" + ) + }) + .collect::>(); + assert_eq!(fixtures.len(), 3); + + for fixture in fixtures { + let encoded = Arc::<[u8]>::from(fixture.encoded); + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + for request in requests { + assert_signed_rgb_request(fixture.name, &encoded, layout, request); + } + } + } +} + +#[test] +fn dropped_pending_prepared_ht_rgb_group_reuses_session_and_prepared_plan() { + if !should_run_metal_runtime() { + return; + } + + let first = Arc::<[u8]>::from(fixture_ht_rgb_u8_sized(8, 8, 0)); + let second = Arc::<[u8]>::from(fixture_ht_rgb_u8_sized(8, 8, 17)); + let options = BatchDecodeOptions { + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::full(first.clone()), + EncodedImage::full(second.clone()), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU RGB reuse oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("six-bit RGB must use U8 batch storage") + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder.prepare(inputs).expect("prepare reusable RGB group"); + let group = &prepared.groups()[0]; + let (width, height) = group.info().dimensions; + let image_len = width as usize * height as usize * 3; + let layout = MetalImageLayout::new_batch( + 4, + (width, height), + width as usize * 3, + PixelFormat::Rgb8, + 2, + image_len, + ) + .expect("reusable RGB group layout"); + + let dropped_buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + image_len * 2 + 8, + ) + .expect("dropped RGB destination"); + // SAFETY: the fresh range is retained exclusively by the pending owner. + let dropped_destination = unsafe { + MetalImageDestination::from_exclusive_buffer(dropped_buffer, layout) + .expect("dropped RGB destination guard") + }; + drop( + decoder + .submit_prepared_group_into(group, dropped_destination) + .expect("submit disposable RGB group"), + ); + + let completed_buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + image_len * 2 + 8, + ) + .expect("completed RGB destination"); + // SAFETY: the fresh range stays exclusive through explicit completion. + let completed_destination = unsafe { + MetalImageDestination::from_exclusive_buffer(completed_buffer.clone(), layout) + .expect("completed RGB destination guard") + }; + decoder + .submit_prepared_group_into(group, completed_destination) + .expect("reuse RGB session after pending drop") + .wait() + .expect("complete reused RGB group"); + + // SAFETY: completion released the exclusive destination owner. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&completed_buffer, 4, image_len * 2) + } + .expect("completed RGB samples"); + assert_eq!(actual.as_slice(), expected.as_slice()); + assert_eq!( + decoder.submissions().expect("RGB submission count"), + 2, + "a dropped pending group must retire exactly once and leave the session reusable" + ); +} diff --git a/crates/j2k-metal/tests/device/decode.rs b/crates/j2k-metal/tests/device/decode.rs new file mode 100644 index 00000000..c27462ef --- /dev/null +++ b/crates/j2k-metal/tests/device/decode.rs @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn full_classic_grayscale_decode_to_metal_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 16]; + host_decoder + .decode_into(&mut host, 4, PixelFormat::Gray8) + .expect("host decode"); + + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) + .expect("device decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (4, 4)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} + +#[test] +fn full_htj2k_decode_to_metal_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_ht_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 16]; + host_decoder + .decode_into(&mut host, 4, PixelFormat::Gray8) + .expect("host decode"); + + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) + .expect("device decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (4, 4)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} + +#[test] +fn htj2k_direct_decode_clears_reused_classic_scratch_buffers() { + if !should_run_metal_runtime() { + return; + } + + let classic_bytes = fixture_gray8(); + let mut classic_decoder = J2kDecoder::new(&classic_bytes).expect("classic decoder"); + let classic_surface = classic_decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) + .expect("classic device decode"); + assert_eq!(classic_surface.backend_kind(), BackendKind::Metal); + + let bytes = fixture_ht_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 16]; + host_decoder + .decode_into(&mut host, 4, PixelFormat::Gray8) + .expect("host decode"); + + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) + .expect("device decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (4, 4)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} + +#[test] +fn full_irreversible_j2k_decode_to_metal_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8_irreversible(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 16]; + host_decoder + .decode_into(&mut host, 4, PixelFormat::Gray8) + .expect("host decode"); + + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) + .expect("device decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (4, 4)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} + +#[test] +fn auto_full_grayscale_prefers_cpu_for_small_classic_fixture() { + let bytes = fixture_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Auto) + .expect("auto decode"); + assert_eq!(surface.backend_kind(), BackendKind::Cpu); +} + +#[test] +fn auto_full_htj2k_prefers_cpu_for_small_fixture() { + let bytes = fixture_ht_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Auto) + .expect("auto decode"); + assert_eq!(surface.backend_kind(), BackendKind::Cpu); +} + +#[test] +fn auto_repeated_grayscale_keeps_short_512_batch_on_cpu() { + let bytes = fixture_gray8_sized(512, 512); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surfaces = decoder + .decode_repeated_grayscale_auto_to_device(PixelFormat::Gray8, 8) + .expect("auto repeated decode"); + assert_eq!(surfaces.len(), 8); + assert!(surfaces + .iter() + .all(|surface| surface.backend_kind() == BackendKind::Cpu)); +} + +#[test] +fn auto_repeated_grayscale_uses_metal_for_512_batch() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8_sized(512, 512); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surfaces = decoder + .decode_repeated_grayscale_auto_to_device(PixelFormat::Gray8, 16) + .expect("auto repeated decode"); + assert_eq!(surfaces.len(), 16); + assert!(surfaces + .iter() + .all(|surface| surface.backend_kind() == BackendKind::Metal)); +} + +#[test] +fn tile_full_grayscale_device_path_uses_metal_direct() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let mut ctx = J2kContext::default(); + let mut pool = J2kScratchPool::new(); + let surface = Codec::decode_tile_to_device( + &mut ctx, + &mut pool, + &bytes, + PixelFormat::Gray8, + BackendRequest::Metal, + ) + .expect("tile surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (4, 4)); +} + +#[test] +fn metal_surface_exposes_buffer_for_on_device_consumers() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let mut metal_decoder = J2kDecoder::new(&bytes).expect("metal decoder"); + let metal_surface = metal_decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Metal) + .expect("metal surface"); + let (buffer, byte_offset) = + completed_surface_metal_buffer(&metal_surface).expect("metal buffer"); + assert_eq!(byte_offset, 0); + let buffer_len = usize::try_from(buffer.length()).expect("metal buffer length fits usize"); + assert!(buffer_len >= metal_surface.byte_len()); + + let mut cpu_decoder = J2kDecoder::new(&bytes).expect("cpu decoder"); + let cpu_surface = cpu_decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Cpu) + .expect("cpu surface"); + assert!(completed_surface_metal_buffer(&cpu_surface).is_none()); +} + +#[test] +fn metal_encoded_raw_parts_validate_ranges_and_support_consuming_handoff() { + use metal::foreign_types::ForeignType; + + if !should_run_metal_runtime() { + return; + } + + let Some(device) = metal::Device::system_default() else { + j2k_test_support::metal_device_unavailable_is_skip(module_path!()); + return; + }; + let invalid_buffer = + j2k_metal_support::checked_shared_buffer(&device, 64).expect("test buffer allocation"); + // SAFETY: This fresh allocation has no prior or concurrent writers and is + // retained only for this constructor call. + let invalid = unsafe { + j2k_metal::MetalEncodedJ2k::from_raw_parts(invalid_buffer, 16..32, 64, (4, 4), 1, 8, false) + }; + assert!(matches!( + invalid, + Err(Error::MetalKernel { message }) if message.contains("exceeds allocation length") + )); + + let buffer = + j2k_metal_support::checked_shared_buffer(&device, 64).expect("test buffer allocation"); + let expected_ptr = buffer.as_ptr(); + // SAFETY: This fresh allocation has no writers and stays immutable until + // the encoded object is consumed below. + let encoded = unsafe { + j2k_metal::MetalEncodedJ2k::from_raw_parts(buffer, 8..24, 32, (4, 4), 1, 8, false) + } + .expect("valid raw Metal codestream parts"); + assert_eq!(encoded.byte_offset(), 8); + assert_eq!(encoded.byte_len(), 16); + assert_eq!(encoded.capacity(), 32); + assert_eq!(encoded.dimensions(), (4, 4)); + assert_eq!(encoded.components(), 1); + assert_eq!(encoded.bit_depth(), 8); + assert!(!encoded.is_signed()); + // SAFETY: This encoded descriptor is the allocation's only owner and no + // sibling descriptor or cloned handle exists. + let handed_off = unsafe { encoded.into_codestream_buffer() }; + assert_eq!(handed_off.as_ptr(), expected_ptr); +} + +#[cfg(target_os = "macos")] +#[test] +fn decode_to_device_with_session_uses_session_device() { + use metal::foreign_types::ForeignTypeRef; + + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut decoder = J2kDecoder::new(&bytes).expect("metal decoder"); + + let surface = decoder + .decode_request_to_device_with_session( + MetalDecodeRequest::full(PixelFormat::Gray8, BackendRequest::Metal), + &session, + ) + .expect("session decode"); + + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + let (buffer, _) = completed_surface_metal_buffer(&surface).expect("metal buffer"); + assert_eq!(buffer.device().as_ptr(), session.device().as_ptr()); +} + +#[cfg(target_os = "macos")] +#[test] +fn decode_scaled_to_device_with_session_supports_rgb8_resident_surface() { + use metal::foreign_types::ForeignTypeRef; + + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_rgb8_sized(8, 8); + let scale = Downscale::Half; + let scaled = Rect { + x: 0, + y: 0, + w: 8, + h: 8, + } + .scaled_covering(scale); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut pool = J2kScratchPool::new(); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_scaled_into(&mut pool, &mut host, stride, PixelFormat::Rgb8, scale) + .expect("host scaled RGB8 decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("metal decoder"); + let surface = decoder + .decode_request_to_device_with_session( + MetalDecodeRequest::scaled(PixelFormat::Rgb8, scale, BackendRequest::Metal), + &session, + ) + .expect("session scaled RGB8 decode"); + + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + let (buffer, _) = completed_surface_metal_buffer(&surface).expect("metal buffer"); + assert_eq!(buffer.device().as_ptr(), session.device().as_ptr()); +} + +#[cfg(target_os = "macos")] +#[test] +fn explicit_cpu_staged_metal_api_uses_session_device_and_marks_residency() { + use metal::foreign_types::ForeignTypeRef; + + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_rgb8(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 12]; + host_decoder + .decode_into(&mut host, 6, PixelFormat::Rgb8) + .expect("host decode"); + + let surface = decoder + .decode_request_to_cpu_staged_metal_surface_with_session( + MetalDecodeRequest::full(PixelFormat::Rgb8, BackendRequest::Metal), + &session, + ) + .expect("CPU-staged Metal surface"); + + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::CpuStagedMetalUpload); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + let (buffer, byte_offset) = completed_surface_metal_buffer(&surface).expect("Metal buffer"); + assert_eq!(byte_offset, 0); + assert_eq!(buffer.device().as_ptr(), session.device().as_ptr()); +} + +#[cfg(target_os = "macos")] +#[test] +fn decode_to_device_with_session_unsupported_rgba16_is_rejected() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_rgb12(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut decoder = J2kDecoder::new(&bytes).expect("metal decoder"); + + let result = decoder.decode_request_to_device_with_session( + MetalDecodeRequest::full(PixelFormat::Rgba16, BackendRequest::Metal), + &session, + ); + + match result { + Err(Error::UnsupportedMetalRequest { reason }) => { + assert_eq!(reason, UNSUPPORTED_RGBA16_REASON); + } + Err(other) => panic!("unexpected explicit Metal session error: {other:?}"), + Ok(surface) => panic!( + "explicit Metal session request must not fall back; got {:?}", + surface.backend_kind() + ), + } +} diff --git a/crates/j2k-metal/tests/device/direct_gray_requests.rs b/crates/j2k-metal/tests/device/direct_gray_requests.rs new file mode 100644 index 00000000..fdf9602d --- /dev/null +++ b/crates/j2k-metal/tests/device/direct_gray_requests.rs @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn explicit_metal_region_and_scaled_grayscale_match_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let roi = Rect { + x: 0, + y: 0, + w: 2, + h: 2, + }; + + let mut host_region_decoder = J2kDecoder::new(&bytes).expect("host region decoder"); + let mut host_region = [0u8; 4]; + host_region_decoder + .decode_region_into( + &mut J2kScratchPool::new(), + &mut host_region, + 2, + PixelFormat::Gray8, + roi, + ) + .expect("host region decode"); + + let mut region_decoder = J2kDecoder::new(&bytes).expect("decoder"); + let region_surface = region_decoder + .decode_region_to_device(PixelFormat::Gray8, roi, BackendRequest::Metal) + .expect("explicit Metal region decode"); + assert_eq!(region_surface.backend_kind(), BackendKind::Metal); + assert_eq!(region_surface.dimensions(), (2, 2)); + assert_eq!( + region_surface.as_bytes().expect("surface byte access"), + host_region.as_slice() + ); + + let mut host_scaled_decoder = J2kDecoder::new(&bytes).expect("host scaled decoder"); + let mut host_scaled = [0u8; 4]; + host_scaled_decoder + .decode_scaled_into( + &mut J2kScratchPool::new(), + &mut host_scaled, + 2, + PixelFormat::Gray8, + Downscale::Half, + ) + .expect("host scaled decode"); + + let mut scaled_decoder = J2kDecoder::new(&bytes).expect("decoder"); + let scaled_surface = scaled_decoder + .decode_scaled_to_device(PixelFormat::Gray8, Downscale::Half, BackendRequest::Metal) + .expect("explicit Metal scaled decode"); + assert_eq!(scaled_surface.backend_kind(), BackendKind::Metal); + assert_eq!(scaled_surface.dimensions(), (2, 2)); + assert_eq!( + scaled_surface.as_bytes().expect("surface byte access"), + host_scaled.as_slice() + ); +} + +#[test] +fn explicit_metal_scaled_rgb8_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_rgb8_sized(8, 8); + let scale = Downscale::Half; + let scaled = Rect { + x: 0, + y: 0, + w: 8, + h: 8, + } + .scaled_covering(scale); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host scaled decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb8, + scale, + ) + .expect("host scaled RGB8 decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_scaled_to_device(PixelFormat::Rgb8, scale, BackendRequest::Metal) + .expect("explicit Metal scaled RGB8 decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} + +#[test] +fn explicit_metal_region_and_scaled_htj2k_grayscale_match_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_ht_gray8(); + let roi = Rect { + x: 0, + y: 0, + w: 2, + h: 2, + }; + + let mut host_region_decoder = J2kDecoder::new(&bytes).expect("host region decoder"); + let mut host_region = [0u8; 4]; + host_region_decoder + .decode_region_into( + &mut J2kScratchPool::new(), + &mut host_region, + 2, + PixelFormat::Gray8, + roi, + ) + .expect("host region decode"); + + let mut region_decoder = J2kDecoder::new(&bytes).expect("decoder"); + let region_surface = region_decoder + .decode_region_to_device(PixelFormat::Gray8, roi, BackendRequest::Metal) + .expect("explicit Metal region decode"); + assert_eq!(region_surface.backend_kind(), BackendKind::Metal); + assert_eq!(region_surface.dimensions(), (2, 2)); + assert_eq!( + region_surface.as_bytes().expect("surface byte access"), + host_region.as_slice() + ); + + let mut host_scaled_decoder = J2kDecoder::new(&bytes).expect("host scaled decoder"); + let mut host_scaled = [0u8; 4]; + host_scaled_decoder + .decode_scaled_into( + &mut J2kScratchPool::new(), + &mut host_scaled, + 2, + PixelFormat::Gray8, + Downscale::Half, + ) + .expect("host scaled decode"); + + let mut scaled_decoder = J2kDecoder::new(&bytes).expect("decoder"); + let scaled_surface = scaled_decoder + .decode_scaled_to_device(PixelFormat::Gray8, Downscale::Half, BackendRequest::Metal) + .expect("explicit Metal scaled decode"); + assert_eq!(scaled_surface.backend_kind(), BackendKind::Metal); + assert_eq!(scaled_surface.dimensions(), (2, 2)); + assert_eq!( + scaled_surface.as_bytes().expect("surface byte access"), + host_scaled.as_slice() + ); +} + +#[test] +fn explicit_metal_region_scaled_grayscale_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + scaled.w as usize, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region scaled decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} + +#[test] +fn explicit_metal_region_scaled_grayscale_large_cropped_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 512, + }; + + for scale in [Downscale::Half, Downscale::None] { + let scaled = roi.scaled_covering(scale); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + scaled.w as usize, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region scaled decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + let surface_bytes = surface.as_bytes().expect("surface byte access"); + if surface_bytes.as_ref() != host.as_slice() { + let mismatch = surface_bytes + .iter() + .zip(&host) + .position(|(actual, expected)| actual != expected) + .expect("mismatched buffers should have a differing byte"); + panic!( + "scale={scale:?} first mismatch at byte {mismatch}: metal={} host={}", + surface_bytes[mismatch], host[mismatch] + ); + } + } +} + +#[test] +fn explicit_metal_region_scaled_htj2k_grayscale_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_ht_gray8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + scaled.w as usize, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region scaled decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} + +#[test] +fn explicit_metal_region_scaled_htj2k_falls_back_when_direct_width_is_unsupported() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_ht_gray8_unsupported_direct_width(); + let roi = Rect { + x: 48, + y: 2, + w: 96, + h: 4, + }; + let scale = Downscale::None; + let scaled = roi.scaled_covering(scale); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + scaled.w as usize, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region scaled decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal should fall back after unsupported direct HT geometry"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} diff --git a/crates/j2k-metal/tests/device/direct_repeated.rs b/crates/j2k-metal/tests/device/direct_repeated.rs new file mode 100644 index 00000000..41f29a3d --- /dev/null +++ b/crates/j2k-metal/tests/device/direct_repeated.rs @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn repeated_classic_grayscale_direct_decode_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surfaces = decoder + .decode_repeated_grayscale_direct_to_device(PixelFormat::Gray8, 3) + .expect("repeated direct decode"); + assert_eq!(surfaces.len(), 3); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 16]; + host_decoder + .decode_into(&mut host, 4, PixelFormat::Gray8) + .expect("host decode"); + + for surface in surfaces { + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } +} + +#[test] +fn repeated_ht_grayscale_direct_decode_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_ht_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surfaces = decoder + .decode_repeated_grayscale_direct_to_device(PixelFormat::Gray8, 3) + .expect("repeated direct decode"); + assert_eq!(surfaces.len(), 3); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 16]; + host_decoder + .decode_into(&mut host, 4, PixelFormat::Gray8) + .expect("host decode"); + + for surface in surfaces { + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } +} + +#[test] +fn metal_gray16_matches_host_decode_for_12bit_source() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray12(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 8]; + host_decoder + .decode_into(&mut host, 4, PixelFormat::Gray16) + .expect("host decode"); + + let surface = decoder + .decode_to_device(PixelFormat::Gray16, BackendRequest::Metal) + .expect("device decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} + +#[test] +fn explicit_metal_rgb_full_tile_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let rgb8 = fixture_rgb8(); + { + let mut decoder = J2kDecoder::new(&rgb8).expect("rgb8 decoder"); + let mut host_decoder = J2kDecoder::new(&rgb8).expect("rgb8 host decoder"); + let mut host = [0u8; 12]; + host_decoder + .decode_into(&mut host, 6, PixelFormat::Rgb8) + .expect("host rgb8 decode"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Metal) + .expect("explicit Metal rgb8 decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (2, 2)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } + + { + let mut decoder = J2kDecoder::new(&rgb8).expect("rgba8 decoder"); + let mut host_decoder = J2kDecoder::new(&rgb8).expect("rgba8 host decoder"); + let mut host = [0u8; 16]; + host_decoder + .decode_into(&mut host, 8, PixelFormat::Rgba8) + .expect("host rgba8 decode"); + let surface = decoder + .decode_to_device(PixelFormat::Rgba8, BackendRequest::Metal) + .expect("explicit Metal rgba8 decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (2, 2)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } + + let rgb12 = fixture_rgb12(); + { + let mut decoder = J2kDecoder::new(&rgb12).expect("rgb12 decoder"); + let mut host_decoder = J2kDecoder::new(&rgb12).expect("rgb12 host decoder"); + let mut host = [0u8; 12]; + host_decoder + .decode_into(&mut host, 12, PixelFormat::Rgb16) + .expect("host rgb16 decode"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb16, BackendRequest::Metal) + .expect("explicit Metal rgb16 decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (2, 1)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } +} + +#[test] +fn explicit_metal_unsupported_rgba16_full_decode_is_rejected() { + let bytes = fixture_rgb12(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + + let result = decoder.decode_to_device(PixelFormat::Rgba16, BackendRequest::Metal); + + match result { + Err(Error::UnsupportedMetalRequest { reason }) => { + assert_eq!(reason, UNSUPPORTED_RGBA16_REASON); + } + Err(other) => panic!("unexpected explicit Metal error: {other:?}"), + Ok(surface) => panic!( + "explicit Metal must not silently fall back; got {:?}", + surface.backend_kind() + ), + } +} + +#[test] +fn explicit_metal_unsupported_rgba16_error_is_codec_unsupported() { + let bytes = fixture_rgb12(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let err = match decoder.decode_to_device(PixelFormat::Rgba16, BackendRequest::Metal) { + Err(err) => err, + Ok(surface) => panic!( + "explicit Metal must not silently fall back; got {:?}", + surface.backend_kind() + ), + }; + + assert!(err.is_unsupported()); +} + +#[test] +fn auto_decode_report_explains_cpu_fallback_and_residency() { + let bytes = fixture_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + + let reported = decoder + .decode_request_to_device_with_report(MetalDecodeRequest::full( + PixelFormat::Gray8, + BackendRequest::Auto, + )) + .expect("reported Auto decode"); + + assert_eq!(reported.surface.backend_kind(), BackendKind::Cpu); + assert_eq!(reported.surface.residency(), SurfaceResidency::Host); + assert!(completed_surface_metal_buffer(&reported.surface).is_none()); + assert_eq!(reported.report.operation, DecodeOperation::Full); + assert_eq!(reported.report.requested_backend, BackendRequest::Auto); + assert_eq!(reported.report.selected_backend, BackendKind::Cpu); + assert_eq!(reported.report.pixel_format, PixelFormat::Gray8); + assert_eq!(reported.report.surface_residency, SurfaceResidency::Host); + assert_eq!( + reported.report.fallback_reason, + Some(AUTO_DECODE_CPU_FALLBACK_REASON) + ); +} + +#[test] +fn explicit_metal_decode_report_records_resident_surface() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + + let reported = decoder + .decode_request_to_device_with_report(MetalDecodeRequest::full( + PixelFormat::Gray8, + BackendRequest::Metal, + )) + .expect("reported explicit Metal decode"); + + assert_eq!(reported.surface.backend_kind(), BackendKind::Metal); + assert_eq!( + reported.surface.residency(), + SurfaceResidency::MetalResidentDecode + ); + assert_eq!(reported.report.operation, DecodeOperation::Full); + assert_eq!(reported.report.requested_backend, BackendRequest::Metal); + assert_eq!(reported.report.selected_backend, BackendKind::Metal); + assert_eq!( + reported.report.surface_residency, + SurfaceResidency::MetalResidentDecode + ); + assert_eq!(reported.report.fallback_reason, None); +} + +#[test] +fn explicit_metal_unsupported_rgba16_report_variants_are_rejected() { + let bytes = fixture_rgb12(); + let roi = Rect { + x: 0, + y: 0, + w: 2, + h: 1, + }; + let scale = Downscale::Half; + + let mut decoder = J2kDecoder::new(&bytes).expect("full decoder"); + assert_unsupported_rgba16_report(decoder.decode_request_to_device_with_report( + MetalDecodeRequest::full(PixelFormat::Rgba16, BackendRequest::Metal), + )); + + let mut decoder = J2kDecoder::new(&bytes).expect("region decoder"); + assert_unsupported_rgba16_report(decoder.decode_request_to_device_with_report( + MetalDecodeRequest::region(PixelFormat::Rgba16, roi, BackendRequest::Metal), + )); + + let mut decoder = J2kDecoder::new(&bytes).expect("scaled decoder"); + assert_unsupported_rgba16_report(decoder.decode_request_to_device_with_report( + MetalDecodeRequest::scaled(PixelFormat::Rgba16, scale, BackendRequest::Metal), + )); + + let mut decoder = J2kDecoder::new(&bytes).expect("region scaled decoder"); + assert_unsupported_rgba16_report(decoder.decode_request_to_device_with_report( + MetalDecodeRequest::region_scaled(PixelFormat::Rgba16, roi, scale, BackendRequest::Metal), + )); +} diff --git a/crates/j2k-metal/tests/device/direct_rgb_requests.rs b/crates/j2k-metal/tests/device/direct_rgb_requests.rs new file mode 100644 index 00000000..6198df6c --- /dev/null +++ b/crates/j2k-metal/tests/device/direct_rgb_requests.rs @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn explicit_metal_region_scaled_rgb_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_direct_rgb8_variant(3); + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("host region scaled RGB decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgb8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGB decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("rgba8 host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgba8, + roi, + scale, + ) + .expect("host region scaled RGBA decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("rgba8 decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgba8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGBA decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + + let bytes = fixture_rgb12(); + let roi = Rect { + x: 0, + y: 0, + w: 2, + h: 1, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut host_decoder = J2kDecoder::new(&bytes).expect("rgb16 host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb16, + roi, + scale, + ) + .expect("host region scaled RGB16 decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("rgb16 decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgb16, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGB16 decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); +} + +#[test] +fn explicit_metal_region_scaled_rgb_large_cropped_matches_host_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_rgb8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 512, + }; + + for scale in [Downscale::Half, Downscale::None] { + let scaled = roi.scaled_covering(scale); + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgba8] { + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize * fmt.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + fmt, + roi, + scale, + ) + .expect("host region scaled RGB decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(fmt, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGB decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + let surface_bytes = surface.as_bytes().expect("surface byte access"); + if surface_bytes.as_ref() != host.as_slice() { + let mismatch = surface_bytes + .iter() + .zip(&host) + .position(|(actual, expected)| actual != expected) + .expect("mismatched buffers should have a differing byte"); + panic!( + "fmt={fmt:?} scale={scale:?} first mismatch at byte {mismatch}: metal={} host={}", + surface_bytes[mismatch], + host[mismatch] + ); + } + } + } +} + +#[test] +fn auto_region_and_scaled_fallback_to_cpu_surface_and_match_host_decode() { + let bytes = fixture_rgb8(); + let roi = Rect { + x: 0, + y: 0, + w: 1, + h: 1, + }; + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let region_surface = decoder + .decode_region_to_device(PixelFormat::Rgb8, roi, BackendRequest::Auto) + .expect("region surface"); + assert_eq!(region_surface.backend_kind(), BackendKind::Cpu); + assert!(completed_surface_metal_buffer(®ion_surface).is_none()); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut region_host = [0u8; 3]; + host_decoder + .decode_region_into( + &mut J2kScratchPool::new(), + &mut region_host, + 3, + PixelFormat::Rgb8, + roi, + ) + .expect("host region"); + assert_eq!( + region_surface.as_bytes().expect("surface byte access"), + region_host.as_slice() + ); + + let scaled_surface = decoder + .decode_scaled_to_device(PixelFormat::Rgb8, Downscale::Half, BackendRequest::Auto) + .expect("scaled surface"); + assert_eq!(scaled_surface.backend_kind(), BackendKind::Cpu); + assert!(completed_surface_metal_buffer(&scaled_surface).is_none()); + + let mut scaled_host = [0u8; 3]; + host_decoder + .decode_scaled_into( + &mut J2kScratchPool::new(), + &mut scaled_host, + 3, + PixelFormat::Rgb8, + Downscale::Half, + ) + .expect("host scaled"); + assert_eq!( + scaled_surface.as_bytes().expect("surface byte access"), + scaled_host.as_slice() + ); +} + +#[test] +fn invalid_region_reports_error_instead_of_panicking() { + let bytes = fixture_rgb8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + match decoder.decode_region_to_device(PixelFormat::Rgb8, roi, BackendRequest::Auto) { + Err(Error::Decode(j2k::J2kError::InvalidRegion { .. })) => {} + Err(other) => panic!("unexpected error for invalid ROI: {other:?}"), + Ok(_) => panic!("invalid ROI should fail"), + } +} + +#[test] +fn explicit_metal_tile_unsupported_rgba16_is_rejected() { + let bytes = fixture_rgb12(); + let mut ctx = J2kContext::default(); + let mut pool = J2kScratchPool::new(); + + let result = Codec::decode_tile_to_device( + &mut ctx, + &mut pool, + &bytes, + PixelFormat::Rgba16, + BackendRequest::Metal, + ); + + match result { + Err(Error::UnsupportedMetalRequest { reason }) => { + assert_eq!(reason, UNSUPPORTED_RGBA16_REASON); + } + Err(other) => panic!("unexpected explicit Metal tile error: {other:?}"), + Ok(surface) => panic!( + "explicit Metal tile request must not fall back; got {:?}", + surface.backend_kind() + ), + } +} + +#[test] +fn hybrid_ht_cpuupload_uses_worker_local_decode_workspace() { + let source = include_str!("../../src/compute/direct_cpu.rs"); + + assert!( + source.contains("decode_prepared_ht_jobs_on_cpu_with_workspace"), + "HT CPUUpload decode must expose a workspace-aware helper" + ); + assert!( + source.contains("HtCodeBlockDecodeWorkspace::default()"), + "parallel HT CPUUpload decode must initialize worker-local HT decode workspaces" + ); + assert!( + source.contains("decode_ht_code_block_scalar_with_workspace"), + "HT CPUUpload decode must call the scratch-reusing scalar helper" + ); +} diff --git a/crates/j2k-metal/tests/device/external_batch.rs b/crates/j2k-metal/tests/device/external_batch.rs new file mode 100644 index 00000000..13db8ffe --- /dev/null +++ b/crates/j2k-metal/tests/device/external_batch.rs @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn shared_metal_batch_keeps_indexed_prepare_failures_and_decodes_other_groups() { + if !should_run_metal_runtime() { + return; + } + + let valid = Arc::<[u8]>::from(fixture_ht_gray8()); + let malformed = Arc::<[u8]>::from(&b"not a codestream"[..]); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let result = decoder + .decode_batch(vec![ + EncodedImage::full(malformed), + EncodedImage::full(valid), + ]) + .expect("one-shot shared Metal batch"); + + assert_eq!(result.errors().len(), 1); + assert_eq!(result.errors()[0].index, 0); + assert_eq!(result.groups().len(), 1); + assert_eq!(result.groups()[0].source_indices(), &[1]); +} + +#[test] +fn persistent_metal_batch_decoder_validates_external_destination_subranges() { + if !should_run_metal_runtime() { + return; + } + + let decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let device = decoder.backend_session().device(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::(device, 32) + .expect("destination buffer"); + let layout = j2k_metal_support::MetalImageLayout::new(8, (4, 2), 4, PixelFormat::Gray8) + .expect("destination layout"); + // SAFETY: The test excludes every other CPU/GPU access to this fresh range. + let destination = unsafe { MetalImageDestination::from_exclusive_buffer(buffer, layout) } + .expect("validated external destination"); + + decoder + .validate_destination(&destination, (4, 2), PixelFormat::Gray8) + .expect("matching decode destination"); + assert!(matches!( + decoder.validate_destination(&destination, (8, 1), PixelFormat::Gray8), + Err(Error::MetalSupport { + source: j2k_metal_support::MetalSupportError::MetalImageLayout { .. }, + .. + }) + )); + assert!(matches!( + decoder.validate_destination(&destination, (4, 2), PixelFormat::Gray16), + Err(Error::MetalSupport { + source: j2k_metal_support::MetalSupportError::MetalImageLayout { .. }, + .. + }) + )); +} + +#[test] +fn persistent_metal_batch_decoder_writes_distinct_ht_gray_group_into_external_subranges() { + if !should_run_metal_runtime() { + return; + } + + let first_bytes = fixture_ht_gray8(); + let second_bytes = fixture_ht_gray8_reversed(); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let device = decoder.backend_session().device(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::(device, 40) + .expect("external batch destination"); + let layout = + j2k_metal_support::MetalImageLayout::new_batch(4, (4, 4), 4, PixelFormat::Gray8, 2, 16) + .expect("group layout"); + // SAFETY: No other CPU/GPU access occurs until the synchronous group + // decode completes and the destination owner drops. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("group destination") + }; + let prepared = decoder + .prepare(vec![ + EncodedImage::full(Arc::from(first_bytes.as_slice())), + EncodedImage::full(Arc::from(second_bytes.as_slice())), + ]) + .expect("prepare external HT grayscale group"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + + decoder + .decode_prepared_group_into(&prepared.groups()[0], &destination) + .expect("direct external HT grayscale batch"); + drop(destination); + + // SAFETY: Decode completion is synchronous and all exclusive destination + // owners have been dropped before this readback assertion. + let bytes = unsafe { j2k_metal_support::checked_buffer_read_vec::(&buffer, 0, 40) } + .expect("completed external batch bytes"); + let expected = [&first_bytes, &second_bytes] + .into_iter() + .map(|encoded| { + let mut host = [0_u8; 16]; + J2kDecoder::new(encoded) + .expect("host decoder") + .decode_into(&mut host, 4, PixelFormat::Gray8) + .expect("host decode"); + host + }) + .collect::>(); + + assert_eq!(&bytes[4..20], expected[0]); + assert_eq!(&bytes[20..36], expected[1]); +} + +#[test] +fn shared_prepared_gray8_group_writes_one_destination_with_unaligned_item_offsets() { + if !should_run_metal_runtime() { + return; + } + + let first_bytes = Arc::<[u8]>::from(fixture_ht_gray8_offset_sized(3, 3, 1)); + let second_bytes = Arc::<[u8]>::from(fixture_ht_gray8_offset_sized(3, 3, 101)); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(first_bytes.clone()), + EncodedImage::full(second_bytes.clone()), + ]) + .expect("prepare odd Gray8 group"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + + let device = decoder.backend_session().device(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::(device, 26) + .expect("group destination buffer"); + let layout = + j2k_metal_support::MetalImageLayout::new_batch(4, (3, 3), 3, PixelFormat::Gray8, 2, 9) + .expect("odd Gray8 group layout"); + // SAFETY: This fresh group allocation remains exclusively owned until the + // synchronous direct final-store command has completed. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("group destination") + }; + + decoder + .decode_prepared_group_into(&prepared.groups()[0], &destination) + .expect("direct shared prepared group store"); + drop(destination); + + // SAFETY: The direct store completed and its exclusive destination owner + // was dropped before host verification. + let bytes = unsafe { j2k_metal_support::checked_buffer_read_vec::(&buffer, 0, 26) } + .expect("completed group buffer"); + for (encoded, output) in [first_bytes, second_bytes] + .iter() + .zip([&bytes[4..13], &bytes[13..22]]) + { + let mut expected = [0_u8; 9]; + J2kDecoder::new(encoded) + .expect("CPU oracle decoder") + .decode_into(&mut expected, 3, PixelFormat::Gray8) + .expect("CPU oracle decode"); + assert_eq!(output, expected); + } +} + +#[test] +fn submitted_prepared_group_retains_destination_until_wait_and_decoder_reuses_after_drop() { + if !should_run_metal_runtime() { + return; + } + + let first_bytes = Arc::<[u8]>::from(fixture_ht_gray8_offset_sized(3, 3, 1)); + let second_bytes = Arc::<[u8]>::from(fixture_ht_gray8_offset_sized(3, 3, 101)); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(first_bytes.clone()), + EncodedImage::full(second_bytes.clone()), + ]) + .expect("prepare submitted Gray8 group"); + let layout = MetalImageLayout::new_batch(4, (3, 3), 3, PixelFormat::Gray8, 2, 9) + .expect("submitted group layout"); + + let dropped_buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + 26, + ) + .expect("dropped pending destination buffer"); + // SAFETY: The fresh range is inaccessible until the pending owner is + // dropped; its drop path must retire the committed command before release. + let dropped_destination = unsafe { + MetalImageDestination::from_exclusive_buffer(dropped_buffer, layout) + .expect("dropped pending destination") + }; + let pending = decoder + .submit_prepared_group_into(&prepared.groups()[0], dropped_destination) + .expect("submit group before drop"); + drop(pending); + + let completed_buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + 26, + ) + .expect("completed pending destination buffer"); + // SAFETY: The fresh range remains exclusively retained by the pending + // submission through its explicit completion wait. + let completed_destination = unsafe { + MetalImageDestination::from_exclusive_buffer(completed_buffer.clone(), layout) + .expect("completed pending destination") + }; + decoder + .submit_prepared_group_into(&prepared.groups()[0], completed_destination) + .expect("submit group after dropped pending") + .wait() + .expect("wait for submitted group"); + + // SAFETY: `wait` completed GPU execution and released the exclusive + // destination guard before this host verification. + let bytes = + unsafe { j2k_metal_support::checked_buffer_read_vec::(&completed_buffer, 0, 26) } + .expect("completed submitted group bytes"); + for (encoded, output) in [first_bytes, second_bytes] + .iter() + .zip([&bytes[4..13], &bytes[13..22]]) + { + let mut expected = [0_u8; 9]; + J2kDecoder::new(encoded) + .expect("CPU oracle decoder") + .decode_into(&mut expected, 3, PixelFormat::Gray8) + .expect("CPU oracle decode"); + assert_eq!(output, expected); + } +} + +#[test] +fn submitted_prepared_group_orders_a_same_device_consumer_queue() { + if !should_run_metal_runtime() { + return; + } + + let first_bytes = Arc::<[u8]>::from(fixture_ht_gray8_offset_sized(3, 3, 7)); + let second_bytes = Arc::<[u8]>::from(fixture_ht_gray8_offset_sized(3, 3, 71)); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(first_bytes.clone()), + EncodedImage::full(second_bytes.clone()), + ]) + .expect("prepare consumer-ordered Gray8 group"); + let device = decoder.backend_session().device().to_owned(); + let destination_buffer = j2k_metal_support::checked_shared_buffer_for_len::(&device, 26) + .expect("consumer-ordered destination"); + let consumer_buffer = j2k_metal_support::checked_shared_buffer_for_len::(&device, 18) + .expect("consumer copy destination"); + let layout = MetalImageLayout::new_batch(4, (3, 3), 3, PixelFormat::Gray8, 2, 9) + .expect("consumer-ordered group layout"); + // SAFETY: The consumer access below is submitted only after the codec + // dependency is registered on that queue. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(destination_buffer.clone(), layout) + .expect("consumer-ordered destination guard") + }; + let mut pending = decoder + .submit_prepared_group_into(&prepared.groups()[0], destination) + .expect("submit producer group"); + let consumer_queue = + j2k_metal_support::checked_command_queue(&device).expect("consumer command queue"); + pending + .enqueue_consumer_wait(&consumer_queue) + .expect("register producer dependency on consumer queue"); + let consumer_command = j2k_metal_support::checked_command_buffer(&consumer_queue) + .expect("consumer command buffer"); + let blit = j2k_metal_support::checked_blit_command_encoder(&consumer_command) + .expect("consumer blit encoder"); + blit.copy_from_buffer(&destination_buffer, 4, &consumer_buffer, 0, 18); + blit.end_encoding(); + consumer_command.commit(); + + pending.wait().expect("producer group completion"); + j2k_metal_support::wait_for_completion(&consumer_command) + .expect("consumer queue completion after producer signal"); + // SAFETY: The producer and the event-ordered consumer copy have completed. + let copied = + unsafe { j2k_metal_support::checked_buffer_read_vec::(&consumer_buffer, 0, 18) } + .expect("consumer-ordered copied bytes"); + for (encoded, output) in [first_bytes, second_bytes] + .iter() + .zip([&copied[..9], &copied[9..]]) + { + let mut expected = [0_u8; 9]; + J2kDecoder::new(encoded) + .expect("CPU oracle decoder") + .decode_into(&mut expected, 3, PixelFormat::Gray8) + .expect("CPU oracle decode"); + assert_eq!(output, expected); + } +} + +#[test] +fn submitted_prepared_signed_gray12_group_stores_native_i16_samples() { + if !should_run_metal_runtime() { + return; + } + + let (first_encoded, first_expected) = fixture_ht_signed_gray12(0); + let (second_encoded, second_expected) = fixture_ht_signed_gray12(31); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(Arc::<[u8]>::from(first_encoded)), + EncodedImage::full(Arc::<[u8]>::from(second_encoded)), + ]) + .expect("prepare signed Gray12 group"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + assert_eq!( + prepared.groups()[0].info().sample_type, + NativeSampleType::I16 + ); + + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + 72, + ) + .expect("signed group destination buffer"); + let layout = MetalImageLayout::new_batch(4, (4, 4), 8, PixelFormat::GrayI16, 2, 32) + .expect("signed group layout"); + // SAFETY: The pending submission exclusively retains this fresh range + // until its explicit completion wait. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("signed group destination") + }; + decoder + .submit_prepared_group_into(&prepared.groups()[0], destination) + .expect("submit signed Gray12 group") + .wait() + .expect("complete signed Gray12 group"); + + // SAFETY: Codec completion released exclusive destination access. + let bytes = unsafe { j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, 64) } + .expect("signed group bytes"); + let actual = bytes + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + assert_eq!(actual, [first_expected, second_expected].concat()); +} diff --git a/crates/j2k-metal/tests/device/grayscale_external.rs b/crates/j2k-metal/tests/device/grayscale_external.rs new file mode 100644 index 00000000..2777653c --- /dev/null +++ b/crates/j2k-metal/tests/device/grayscale_external.rs @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn submitted_prepared_ht_grayscale_roi_and_reduction_match_cpu_oracle() { + if !should_run_metal_runtime() { + return; + } + + let encoded = Arc::<[u8]>::from(j2k_test_support::openhtj2k_refinement_odd_fixture()); + let roi = Rect { + x: 3, + y: 5, + w: 9, + h: 21, + }; + let requests = [ + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + + for request in requests { + let options = BatchDecodeOptions::default(); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(vec![EncodedImage::new(encoded.clone(), request)]) + .expect("CPU request oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("odd OpenHT fixture must decode to U8") + }; + + let mut decoder = MetalBatchDecoder::system_default_with_options(options) + .expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::new(encoded.clone(), request)]) + .expect("prepare Metal request"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + let group = &prepared.groups()[0]; + let (width, height) = group.info().dimensions; + let image_len = usize::try_from(width) + .unwrap() + .checked_mul(usize::try_from(height).unwrap()) + .unwrap(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + image_len + 8, + ) + .expect("request destination buffer"); + let layout = MetalImageLayout::new_batch( + 4, + (width, height), + width as usize, + PixelFormat::Gray8, + 1, + image_len, + ) + .expect("request destination layout"); + // SAFETY: the pending submission exclusively retains this fresh range. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("request destination") + }; + decoder + .submit_prepared_group_into(group, destination) + .expect("submit prepared request") + .wait() + .expect("complete prepared request"); + + // SAFETY: codec completion released exclusive destination access. + let actual = + unsafe { j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, image_len) } + .expect("request pixels"); + assert_eq!( + actual.as_slice(), + expected.as_slice(), + "request {request:?}" + ); + } +} + +#[test] +fn independent_openht_sigprop_overlap_matches_openht_oracle_within_one_lsb() { + const WIDTH: u32 = 512; + const HEIGHT: u32 = 64; + + if !should_run_metal_runtime() { + return; + } + let encoded = Arc::<[u8]>::from(j2k_test_support::openhtj2k_sigprop_overlap_fixture()); + let expected = j2k_test_support::openhtj2k_sigprop_overlap_pixels(); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::full(encoded.clone())]) + .expect("prepare independent SigProp-overlap fixture"); + assert!( + prepared.errors().is_empty(), + "prepare errors: {:?}", + prepared.errors() + ); + assert_eq!(prepared.groups().len(), 1); + let group = &prepared.groups()[0]; + assert_eq!(group.info().route, BatchCodecRoute::Htj2k); + assert_eq!(group.info().dimensions, (WIDTH, HEIGHT)); + + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + expected.len() + 8, + ) + .expect("SigProp-overlap destination buffer"); + let layout = MetalImageLayout::new_batch( + 4, + (WIDTH, HEIGHT), + WIDTH as usize * 3, + PixelFormat::Rgb8, + 1, + expected.len(), + ) + .expect("SigProp-overlap destination layout"); + // SAFETY: this fresh range remains exclusively retained by the submitted + // codec work until its explicit completion wait. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("SigProp-overlap destination") + }; + decoder + .submit_prepared_group_into(group, destination) + .expect("submit independent SigProp-overlap fixture") + .wait() + .expect("complete independent SigProp-overlap fixture"); + + // SAFETY: codec completion released exclusive destination access. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, expected.len()) + .expect("SigProp-overlap pixels") + }; + let mut cpu = vec![0_u8; expected.len()]; + J2kDecoder::new(&encoded) + .expect("SigProp-overlap CPU decoder") + .decode_into(&mut cpu, WIDTH as usize * 3, PixelFormat::Rgb8) + .expect("SigProp-overlap CPU decode"); + assert_eq!(actual, cpu, "Metal must match the corrected scalar reader"); + let max_difference = actual + .iter() + .zip(expected.iter()) + .map(|(&actual, &expected)| actual.abs_diff(expected)) + .max() + .unwrap_or(0); + assert!( + max_difference <= 1, + "Metal differs from the independent OpenHTJ2K oracle by {max_difference} LSB" + ); +} + +#[test] +fn submitted_prepared_classic_grayscale_writes_external_group_without_staging() { + if !should_run_metal_runtime() { + return; + } + + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::full(Arc::<[u8]>::from(fixture_gray8()))]) + .expect("prepare classic grayscale group"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups()[0].info().route, BatchCodecRoute::Classic); + + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + 24, + ) + .expect("classic destination buffer"); + let layout = MetalImageLayout::new_batch(4, (4, 4), 4, PixelFormat::Gray8, 1, 16) + .expect("classic destination layout"); + // SAFETY: the pending submission exclusively retains this fresh range. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("classic destination") + }; + decoder + .submit_prepared_group_into(&prepared.groups()[0], destination) + .expect("submit classic prepared group") + .wait() + .expect("complete classic prepared group"); + + // SAFETY: codec completion released exclusive destination access. + let actual = unsafe { j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, 16) } + .expect("classic pixels"); + assert_eq!(actual, (0..16).collect::>()); +} + +#[test] +fn submitted_prepared_classic_signed_gray12_preserves_native_i16_samples() { + if !should_run_metal_runtime() { + return; + } + + let (encoded, expected) = fixture_classic_signed_gray12(); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::full(Arc::<[u8]>::from(encoded))]) + .expect("prepare classic signed group"); + assert_eq!( + prepared.groups()[0].info().sample_type, + NativeSampleType::I16 + ); + + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + 40, + ) + .expect("classic signed destination buffer"); + let layout = MetalImageLayout::new_batch(4, (4, 4), 8, PixelFormat::GrayI16, 1, 32) + .expect("classic signed destination layout"); + // SAFETY: the pending submission exclusively retains this fresh range. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("classic signed destination") + }; + decoder + .submit_prepared_group_into(&prepared.groups()[0], destination) + .expect("submit classic signed group") + .wait() + .expect("complete classic signed group"); + + // SAFETY: codec completion released exclusive destination access. + let bytes = unsafe { j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, 32) } + .expect("classic signed pixels"); + let actual = bytes + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + assert_eq!(actual, expected); +} + +#[test] +fn dropped_pending_metal_batch_can_be_followed_by_a_successful_decode() { + if !should_run_metal_runtime() { + return; + } + + let bytes = Arc::<[u8]>::from(fixture_ht_gray8()); + let mut decoder = MetalBatchDecoder::system_default().expect("persistent Metal decoder"); + let pending = decoder + .submit_batch(vec![ + EncodedImage::full(bytes.clone()), + EncodedImage::full(bytes.clone()), + ]) + .expect("pending batch"); + assert_eq!(pending.len(), 1); + drop(pending); + + let result = decoder + .decode_batch(vec![ + EncodedImage::full(bytes.clone()), + EncodedImage::full(bytes), + ]) + .expect("decoder reuse after pending drop"); + assert!(result.errors().is_empty()); + assert!(result.group_errors().is_empty()); + assert_eq!(result.groups().len(), 1); + assert_eq!(result.groups()[0].surfaces().len(), 2); + assert!(result.groups()[0] + .surfaces() + .iter() + .all(|surface| surface.residency() == SurfaceResidency::MetalResidentDecode)); +} diff --git a/crates/j2k-metal/tests/device/legacy_batch.rs b/crates/j2k-metal/tests/device/legacy_batch.rs new file mode 100644 index 00000000..604eaf66 --- /dev/null +++ b/crates/j2k-metal/tests/device/legacy_batch.rs @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn submitted_full_grayscale_tiles_flush_as_one_device_batch() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + + let submissions = (0..3) + .map(|_| { + Codec::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Gray8, + BackendRequest::Metal, + ) + .expect("submit tile") + }) + .collect::>(); + + assert_eq!( + session.submissions().expect("session submissions"), + 0, + "submitted tile surfaces should stay queued until a wait flushes the session" + ); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 16]; + host_decoder + .decode_into(&mut host, 4, PixelFormat::Gray8) + .expect("host decode"); + + for submission in submissions { + let surface = submission.wait().expect("surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "compatible queued grayscale tiles should flush through one repeated Metal batch" + ); +} + +#[test] +fn submitted_auto_512_grayscale_tiles_flush_as_one_metal_batch() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8_sized(512, 512); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + + let submissions = (0..16) + .map(|_| { + Codec::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Gray8, + BackendRequest::Auto, + ) + .expect("submit auto tile") + }) + .collect::>(); + + assert_eq!( + session.submissions().expect("session submissions"), + 0, + "auto submitted tile surfaces should stay queued until a wait flushes the session" + ); + + for submission in submissions { + let surface = submission.wait().expect("surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (512, 512)); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "compatible auto grayscale tiles should flush through one repeated Metal batch" + ); +} + +#[test] +fn submitted_distinct_full_grayscale_tiles_flush_as_one_device_batch() { + if !should_run_metal_runtime() { + return; + } + + let classic_bytes = fixture_gray8(); + let reversed_bytes = fixture_gray8_reversed(); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + + let classic_submission = Codec::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + &classic_bytes, + PixelFormat::Gray8, + BackendRequest::Metal, + ) + .expect("submit classic tile"); + let reversed_submission = Codec::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + &reversed_bytes, + PixelFormat::Gray8, + BackendRequest::Metal, + ) + .expect("submit reversed tile"); + + assert_eq!( + session.submissions().expect("session submissions"), + 0, + "distinct submitted tile surfaces should stay queued until wait" + ); + + let mut classic_host_decoder = J2kDecoder::new(&classic_bytes).expect("classic host decoder"); + let mut classic_host = [0u8; 16]; + classic_host_decoder + .decode_into(&mut classic_host, 4, PixelFormat::Gray8) + .expect("classic host decode"); + + let mut reversed_host_decoder = + J2kDecoder::new(&reversed_bytes).expect("reversed host decoder"); + let mut reversed_host = [0u8; 16]; + reversed_host_decoder + .decode_into(&mut reversed_host, 4, PixelFormat::Gray8) + .expect("reversed host decode"); + + let classic_surface = classic_submission.wait().expect("classic surface"); + let reversed_surface = reversed_submission.wait().expect("reversed surface"); + assert_eq!(classic_surface.backend_kind(), BackendKind::Metal); + assert_eq!(reversed_surface.backend_kind(), BackendKind::Metal); + assert_eq!( + classic_surface.as_bytes().expect("surface byte access"), + classic_host.as_slice() + ); + assert_eq!( + reversed_surface.as_bytes().expect("surface byte access"), + reversed_host.as_slice() + ); + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "distinct queued grayscale tiles should flush through one Metal command buffer" + ); +} + +#[test] +fn submitted_non_stackable_grayscale_tiles_decode_every_input() { + if !should_run_metal_runtime() { + return; + } + + let small = fixture_gray8(); + let large = fixture_gray8_sized(8, 8); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + + let submissions = [&small, &large] + .into_iter() + .map(|bytes| { + Codec::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + bytes, + PixelFormat::Gray8, + BackendRequest::Metal, + ) + .expect("submit non-stackable grayscale tile") + }) + .collect::>(); + let expected = [&small, &large] + .into_iter() + .map(|bytes| { + let mut decoder = J2kDecoder::new(bytes).expect("host decoder"); + let (width, height) = decoder.inner().info().dimensions; + let mut pixels = vec![0_u8; width as usize * height as usize]; + decoder + .decode_into(&mut pixels, width as usize, PixelFormat::Gray8) + .expect("host grayscale decode"); + (width, height, pixels) + }) + .collect::>(); + + for (submission, (width, height, pixels)) in submissions.into_iter().zip(expected) { + let surface = submission.wait().expect("non-stackable Metal surface"); + assert_eq!(surface.dimensions(), (width, height)); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + pixels.as_slice() + ); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "non-stackable grayscale inputs should still share one producer command buffer" + ); +} + +#[test] +fn submitted_full_rgb_tiles_flush_as_one_device_batch() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_direct_rgb8(); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + + let submissions = (0..3) + .map(|_| { + Codec::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Rgb8, + BackendRequest::Metal, + ) + .expect("submit rgb tile") + }) + .collect::>(); + + assert_eq!( + session.submissions().expect("session submissions"), + 0, + "submitted RGB tile surfaces should stay queued until a wait flushes the session" + ); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 12]; + host_decoder + .decode_into(&mut host, 6, PixelFormat::Rgb8) + .expect("host decode"); + + for submission in submissions { + let surface = submission.wait().expect("surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "compatible queued RGB tiles should flush through one Metal batch" + ); +} + +#[test] +fn submitted_distinct_full_rgb_tiles_stay_resident_when_batch_route_falls_back() { + if !should_run_metal_runtime() { + return; + } + + let rgb_tiles = [ + fixture_direct_rgb8_variant(0), + fixture_direct_rgb8_variant(5), + fixture_direct_rgb8_variant(11), + ]; + assert_ne!(rgb_tiles[0], rgb_tiles[1], "RGB batch fixtures must differ"); + assert_ne!(rgb_tiles[1], rgb_tiles[2], "RGB batch fixtures must differ"); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + + let submissions = rgb_tiles + .iter() + .map(|bytes| { + Codec::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + bytes, + PixelFormat::Rgb8, + BackendRequest::Metal, + ) + .expect("submit distinct rgb tile") + }) + .collect::>(); + + assert_eq!( + session.submissions().expect("session submissions"), + 0, + "distinct RGB tile surfaces should stay queued until a wait flushes the session" + ); + + let expected = rgb_tiles + .iter() + .map(|bytes| { + let mut host_decoder = J2kDecoder::new(bytes).expect("host decoder"); + let stride = 8 * 3; + let mut host = vec![0u8; stride * 8]; + host_decoder + .decode_into(&mut host, stride, PixelFormat::Rgb8) + .expect("host decode"); + host + }) + .collect::>(); + + let mut surfaces = Vec::with_capacity(submissions.len()); + for (submission, host) in submissions.into_iter().zip(expected) { + let surface = submission.wait().expect("surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + host.as_slice() + ); + surfaces.push(surface); + } + assert!( + session.submissions().expect("session submissions") >= 1, + "queued RGB tiles should submit at least one resident Metal decode" + ); + for surface in surfaces { + assert!(completed_surface_metal_buffer(&surface).is_some()); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + } +} + +#[test] +fn metal_tile_batch_decodes_submitted_tiles_in_order() { + if !should_run_metal_runtime() { + return; + } + + let classic_bytes = fixture_gray8(); + let reversed_bytes = fixture_gray8_reversed(); + let mut batch = MetalTileBatch::new(); + + assert!(batch.is_empty()); + assert_eq!( + batch + .push_tile_request( + &classic_bytes, + MetalDecodeRequest::full(PixelFormat::Gray8, BackendRequest::Metal) + ) + .expect("push classic tile"), + 0 + ); + assert_eq!( + batch + .push_shared_tile_request( + Arc::<[u8]>::from(reversed_bytes.as_slice()), + MetalDecodeRequest::full(PixelFormat::Gray8, BackendRequest::Metal) + ) + .expect("push reversed tile"), + 1 + ); + assert_eq!(batch.len(), 2); + assert_eq!(batch.submissions().expect("batch submissions"), 0); + + let surfaces = batch.decode_all().expect("batch decode"); + assert_eq!(surfaces.len(), 2); + assert_eq!(surfaces[0].backend_kind(), BackendKind::Metal); + assert_eq!(surfaces[1].backend_kind(), BackendKind::Metal); + + let mut classic_host_decoder = J2kDecoder::new(&classic_bytes).expect("classic host decoder"); + let mut classic_host = [0u8; 16]; + classic_host_decoder + .decode_into(&mut classic_host, 4, PixelFormat::Gray8) + .expect("classic host decode"); + + let mut reversed_host_decoder = + J2kDecoder::new(&reversed_bytes).expect("reversed host decoder"); + let mut reversed_host = [0u8; 16]; + reversed_host_decoder + .decode_into(&mut reversed_host, 4, PixelFormat::Gray8) + .expect("reversed host decode"); + + assert_eq!( + surfaces[0].as_bytes().expect("surface byte access"), + classic_host.as_slice() + ); + assert_eq!( + surfaces[1].as_bytes().expect("surface byte access"), + reversed_host.as_slice() + ); +} diff --git a/crates/j2k-metal/tests/device/multitile_color.rs b/crates/j2k-metal/tests/device/multitile_color.rs new file mode 100644 index 00000000..b8838641 --- /dev/null +++ b/crates/j2k-metal/tests/device/multitile_color.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[path = "multitile_color/batch_inputs.rs"] +mod batch_inputs; +#[path = "multitile_color/classic.rs"] +mod classic; +#[path = "multitile_color/gray12.rs"] +mod gray12; +#[path = "multitile_color/rgb.rs"] +mod rgb; +#[path = "multitile_color/signed.rs"] +mod signed; diff --git a/crates/j2k-metal/tests/device/multitile_color/batch_inputs.rs b/crates/j2k-metal/tests/device/multitile_color/batch_inputs.rs new file mode 100644 index 00000000..b04f6e47 --- /dev/null +++ b/crates/j2k-metal/tests/device/multitile_color/batch_inputs.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +pub(super) fn independent_multitile_inputs( + encoded: &[u8], + request: DecodeRequest, +) -> Vec { + let first = Arc::<[u8]>::from(encoded); + let second = Arc::<[u8]>::from(encoded); + assert!( + !Arc::ptr_eq(&first, &second), + "batch regression requires independent encoded-byte owners" + ); + vec![ + EncodedImage::new(first, request), + EncodedImage::new(second, request), + ] +} diff --git a/crates/j2k-metal/tests/device/multitile_color/classic.rs b/crates/j2k-metal/tests/device/multitile_color/classic.rs new file mode 100644 index 00000000..97e6a000 --- /dev/null +++ b/crates/j2k-metal/tests/device/multitile_color/classic.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn classic_multitile_rgb8_decodes_exactly_on_metal() { + if !should_run_metal_runtime() { + return; + } + let (encoded, source_pixels) = fixture_classic_multitile_rgb8(); + let dimensions = (19_u32, 13_u32); + let encoded = Arc::<[u8]>::from(encoded); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut host_decoder = J2kDecoder::new(encoded.as_ref()).expect("classic CPU decoder"); + let mut expected = vec![0_u8; source_pixels.len()]; + host_decoder + .decode_into(&mut expected, dimensions.0 as usize * 3, PixelFormat::Rgb8) + .expect("CPU multi-tile classic RGB8 oracle"); + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::full(encoded)]) + .expect("prepare odd-edge multi-tile classic RGB8 fixture"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + assert_eq!( + prepared.groups()[0].images()[0].preparation_depth(), + PreparationDepth::ClassicOffsetPlan + ); + + let image_len = expected.len(); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + image_len + 4, + ) + .expect("multi-tile classic RGB8 destination"); + let layout = MetalImageLayout::new_batch( + 4, + dimensions, + dimensions.0 as usize * 3, + PixelFormat::Rgb8, + 1, + image_len, + ) + .expect("multi-tile classic RGB8 destination layout"); + // SAFETY: the fresh allocation is exclusively offered to the codec call. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("multi-tile classic RGB8 destination guard") + }; + decoder + .submit_prepared_group_into(&prepared.groups()[0], destination) + .expect("submit multi-tile classic RGB8 fixture") + .wait() + .expect("complete multi-tile classic RGB8 fixture"); + // SAFETY: completion released the exclusive destination owner. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, image_len) + .expect("read multi-tile classic RGB8 destination") + }; + assert_eq!(actual, expected); +} diff --git a/crates/j2k-metal/tests/device/multitile_color/gray12.rs b/crates/j2k-metal/tests/device/multitile_color/gray12.rs new file mode 100644 index 00000000..763e5dd0 --- /dev/null +++ b/crates/j2k-metal/tests/device/multitile_color/gray12.rs @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::batch_inputs::independent_multitile_inputs; +use super::*; + +fn assert_multitile_gray12_batch_request( + decoder: &mut MetalBatchDecoder, + encoded: &[u8], + options: BatchDecodeOptions, + request: DecodeRequest, +) { + let inputs = independent_multitile_inputs(encoded, request); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(inputs.clone()) + .unwrap_or_else(|error| panic!("CPU multi-tile Gray12 {request:?} oracle: {error}")); + assert!(expected.errors().is_empty(), "CPU request {request:?}"); + let CpuBatchSamples::U16(expected) = expected.groups()[0].samples() else { + panic!("OpenJPH Gray12 must use U16 batch storage") + }; + + let prepared = decoder + .prepare(inputs) + .unwrap_or_else(|error| panic!("prepare multi-tile Gray12 {request:?}: {error}")); + assert!(prepared.errors().is_empty(), "Metal request {request:?}"); + assert_eq!(prepared.groups().len(), 1, "Metal request {request:?}"); + let group = &prepared.groups()[0]; + assert_eq!(group.images().len(), 2, "Metal request {request:?}"); + assert!(group + .images() + .iter() + .all(|image| image.preparation_depth() == PreparationDepth::Htj2kOffsetPlan)); + let (width, height) = group.info().dimensions; + let samples_per_image = width as usize * height as usize; + let output_len = samples_per_image * group.images().len(); + assert_eq!(expected.len(), output_len, "CPU request {request:?}"); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + output_len * 2 + 4, + ) + .expect("multi-tile Gray12 batch destination"); + let layout = MetalImageLayout::new_batch( + 4, + (width, height), + width as usize * 2, + PixelFormat::Gray16, + group.images().len(), + samples_per_image * 2, + ) + .expect("multi-tile Gray12 batch destination layout"); + // SAFETY: the fresh allocation is exclusively offered to the codec call. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("multi-tile Gray12 batch destination guard") + }; + decoder + .submit_prepared_group_into(group, destination) + .unwrap_or_else(|error| panic!("submit multi-tile Gray12 {request:?}: {error}")) + .wait() + .unwrap_or_else(|error| panic!("complete multi-tile Gray12 {request:?}: {error}")); + + // SAFETY: completion released the exclusive destination owner. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, output_len) + .expect("read multi-tile Gray12 batch destination") + }; + assert_eq!( + actual.as_slice(), + expected.as_slice(), + "request {request:?}" + ); +} +#[test] +fn independent_openjph_multitile_gray12_decodes_exactly_on_metal() { + if !should_run_metal_runtime() { + return; + } + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-gray-u12-53-raw") + .expect("checked-in OpenJPH Gray12 fixture"); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::full(Arc::from(fixture.encoded))]) + .expect("prepare independent OpenJPH Gray12 fixture"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + assert_eq!( + prepared.groups()[0].images()[0].preparation_depth(), + PreparationDepth::Htj2kOffsetPlan + ); + + let image_len = fixture.width as usize * fixture.height as usize * 2; + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + image_len + 4, + ) + .expect("independent Gray12 destination"); + let layout = MetalImageLayout::new_batch( + 4, + (fixture.width, fixture.height), + fixture.width as usize * 2, + PixelFormat::Gray16, + 1, + image_len, + ) + .expect("independent Gray12 destination layout"); + // SAFETY: the fresh allocation is exclusively offered to the codec call. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("independent Gray12 destination guard") + }; + decoder + .submit_prepared_group_into(&prepared.groups()[0], destination) + .expect("submit independent multi-tile Gray12 fixture") + .wait() + .expect("complete independent multi-tile Gray12 fixture"); + // SAFETY: completion released the exclusive destination owner. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, image_len) + .expect("read independent multi-tile Gray12 destination") + }; + assert_eq!(actual.as_slice(), fixture.oracle); +} + +#[test] +fn independent_openjph_multitile_gray12_batch_matches_cpu_for_all_requests() { + if !should_run_metal_runtime() { + return; + } + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-gray-u12-53-raw") + .expect("checked-in OpenJPH multi-tile Gray12 fixture"); + let roi = Rect { + x: 8, + y: 5, + w: 8, + h: 6, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + let options = BatchDecodeOptions::default(); + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + + for request in requests { + assert_multitile_gray12_batch_request(&mut decoder, fixture.encoded, options, request); + } +} + +#[test] +fn independent_openjph_multitile_gray12_alternating_requests_reuse_stacked_scratch_exactly() { + if !should_run_metal_runtime() { + return; + } + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-gray-u12-53-raw") + .expect("checked-in OpenJPH multi-tile Gray12 fixture"); + let roi = Rect { + x: 8, + y: 5, + w: 8, + h: 6, + }; + let options = BatchDecodeOptions::default(); + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + + for request in [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Full, + DecodeRequest::Region { roi }, + ] { + assert_multitile_gray12_batch_request(&mut decoder, fixture.encoded, options, request); + } +} + +#[test] +fn independent_openjph_multitile_gray12_roi_and_reduction_match_cpu_on_metal() { + if !should_run_metal_runtime() { + return; + } + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-gray-u12-53-raw") + .expect("checked-in OpenJPH Gray12 fixture"); + let encoded = Arc::<[u8]>::from(fixture.encoded); + let roi = Rect { + x: 8, + y: 5, + w: 8, + h: 6, + }; + let requests = [ + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + let options = BatchDecodeOptions::default(); + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + + for request in requests { + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(vec![EncodedImage::new(encoded.clone(), request)]) + .expect("CPU multi-tile Gray12 request oracle"); + assert!(expected.errors().is_empty(), "CPU request {request:?}"); + assert_eq!(expected.groups().len(), 1, "CPU request {request:?}"); + let CpuBatchSamples::U16(expected) = expected.groups()[0].samples() else { + panic!("OpenJPH Gray12 must use U16 batch storage") + }; + + let prepared = decoder + .prepare(vec![EncodedImage::new(encoded.clone(), request)]) + .expect("prepare multi-tile Gray12 request"); + assert!(prepared.errors().is_empty(), "Metal request {request:?}"); + assert_eq!(prepared.groups().len(), 1, "Metal request {request:?}"); + assert_eq!( + prepared.groups()[0].images()[0].preparation_depth(), + PreparationDepth::Htj2kOffsetPlan + ); + let group = &prepared.groups()[0]; + let (width, height) = group.info().dimensions; + let sample_len = width as usize * height as usize; + assert_eq!(expected.len(), sample_len); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + sample_len * 2 + 4, + ) + .expect("multi-tile Gray12 request destination"); + let layout = MetalImageLayout::new_batch( + 4, + (width, height), + width as usize * 2, + PixelFormat::Gray16, + 1, + sample_len * 2, + ) + .expect("multi-tile Gray12 request destination layout"); + // SAFETY: the fresh allocation is exclusively offered to the codec call. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("multi-tile Gray12 request destination guard") + }; + decoder + .submit_prepared_group_into(group, destination) + .expect("submit multi-tile Gray12 request") + .wait() + .expect("complete multi-tile Gray12 request"); + // SAFETY: completion released the exclusive destination owner. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, sample_len) + .expect("read multi-tile Gray12 request destination") + }; + assert_eq!( + actual.as_slice(), + expected.as_slice(), + "request {request:?}" + ); + } +} diff --git a/crates/j2k-metal/tests/device/multitile_color/rgb.rs b/crates/j2k-metal/tests/device/multitile_color/rgb.rs new file mode 100644 index 00000000..beb693b0 --- /dev/null +++ b/crates/j2k-metal/tests/device/multitile_color/rgb.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::batch_inputs::independent_multitile_inputs; +use super::*; + +fn assert_multitile_rgb8_batch_request( + decoder: &mut MetalBatchDecoder, + encoded: &[u8], + options: BatchDecodeOptions, + request: DecodeRequest, +) { + let inputs = independent_multitile_inputs(encoded, request); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(inputs.clone()) + .unwrap_or_else(|error| panic!("CPU multi-tile RGB8 {request:?} oracle: {error}")); + assert!(expected.errors().is_empty(), "CPU request {request:?}"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("OpenJPH RGB8 must use U8 batch storage") + }; + + let prepared = decoder + .prepare(inputs) + .unwrap_or_else(|error| panic!("prepare multi-tile RGB8 {request:?}: {error}")); + assert!(prepared.errors().is_empty(), "Metal request {request:?}"); + assert_eq!(prepared.groups().len(), 1, "Metal request {request:?}"); + let group = &prepared.groups()[0]; + assert_eq!(group.images().len(), 2, "Metal request {request:?}"); + assert!(group + .images() + .iter() + .all(|image| image.preparation_depth() == PreparationDepth::Htj2kOffsetPlan)); + let (width, height) = group.info().dimensions; + let samples_per_image = width as usize * height as usize * 3; + let output_len = samples_per_image * group.images().len(); + assert_eq!(expected.len(), output_len, "CPU request {request:?}"); + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + output_len + 4, + ) + .expect("multi-tile RGB8 batch destination"); + let layout = MetalImageLayout::new_batch( + 4, + (width, height), + width as usize * 3, + PixelFormat::Rgb8, + group.images().len(), + samples_per_image, + ) + .expect("multi-tile RGB8 batch destination layout"); + // SAFETY: the fresh allocation is exclusively offered to the codec call. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("multi-tile RGB8 batch destination guard") + }; + decoder + .submit_prepared_group_into(group, destination) + .unwrap_or_else(|error| panic!("submit multi-tile RGB8 {request:?}: {error}")) + .wait() + .unwrap_or_else(|error| panic!("complete multi-tile RGB8 {request:?}: {error}")); + + // SAFETY: completion released the exclusive destination owner. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, output_len) + .expect("read multi-tile RGB8 batch destination") + }; + assert_eq!( + actual.as_slice(), + expected.as_slice(), + "request {request:?}" + ); +} +#[test] +fn independent_openjph_multitile_rgb_decodes_exactly_on_metal() { + if !should_run_metal_runtime() { + return; + } + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-rgb-u8-53-raw") + .expect("checked-in OpenJPH RGB8 fixture"); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::full(Arc::from(fixture.encoded))]) + .expect("prepare independent OpenJPH RGB fixture"); + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + assert_eq!( + prepared.groups()[0].images()[0].preparation_depth(), + PreparationDepth::Htj2kOffsetPlan + ); + + let image_len = fixture.width as usize * fixture.height as usize * 3; + let buffer = j2k_metal_support::checked_shared_buffer_for_len::( + decoder.backend_session().device(), + image_len + 4, + ) + .expect("independent RGB destination"); + let layout = MetalImageLayout::new_batch( + 4, + (fixture.width, fixture.height), + fixture.width as usize * 3, + PixelFormat::Rgb8, + 1, + image_len, + ) + .expect("independent RGB destination layout"); + // SAFETY: the fresh allocation is exclusively offered to the codec call. + let destination = unsafe { + MetalImageDestination::from_exclusive_buffer(buffer.clone(), layout) + .expect("independent RGB destination guard") + }; + decoder + .submit_prepared_group_into(&prepared.groups()[0], destination) + .expect("submit independent multi-tile RGB fixture") + .wait() + .expect("complete independent multi-tile RGB fixture"); + // SAFETY: completion released the exclusive destination owner. + let actual = unsafe { + j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, image_len) + .expect("read independent multi-tile RGB destination") + }; + assert_eq!(actual.as_slice(), fixture.oracle); +} + +#[test] +fn independent_openjph_multitile_rgb_batch_matches_cpu_for_all_requests() { + if !should_run_metal_runtime() { + return; + } + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-rgb-u8-53-raw") + .expect("checked-in OpenJPH multi-tile RGB8 fixture"); + let roi = Rect { + x: 8, + y: 5, + w: 8, + h: 6, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + + for request in requests { + assert_multitile_rgb8_batch_request(&mut decoder, fixture.encoded, options, request); + } +} diff --git a/crates/j2k-metal/tests/device/multitile_color/signed.rs b/crates/j2k-metal/tests/device/multitile_color/signed.rs new file mode 100644 index 00000000..5ddd5461 --- /dev/null +++ b/crates/j2k-metal/tests/device/multitile_color/signed.rs @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn prepared_metal_color_preserves_signed_rgb_in_resident_output() { + if !should_run_metal_runtime() { + return; + } + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let fixture = j2k_test_support::openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-rgb-s12-53-single-raw") + .expect("checked-in single-tile OpenJPH signed RGB12 fixture"); + let inputs = vec![EncodedImage::full(Arc::from(fixture.encoded))]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU signed RGB12 oracle"); + let CpuBatchSamples::I16(expected) = expected.groups()[0].samples() else { + panic!("signed RGB12 must use I16 batch storage") + }; + let expected_bytes = expected + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder.prepare(inputs).expect("prepare signed RGB group"); + assert!(prepared.errors().is_empty()); + assert_eq!( + prepared.groups()[0].info().sample_type, + NativeSampleType::I16 + ); + assert_eq!( + prepared.groups()[0].images()[0].preparation_depth(), + PreparationDepth::Htj2kOffsetPlan + ); + let result = decoder + .decode_prepared(&prepared) + .expect("decode signed RGB resident group"); + assert!(result.group_errors().is_empty()); + let surface = &result.groups()[0].surfaces()[0]; + assert_eq!(surface.pixel_format(), PixelFormat::RgbI16); + assert_eq!( + surface + .as_bytes() + .expect("resident signed RGB bytes") + .as_ref(), + expected_bytes + ); +} diff --git a/crates/j2k-metal/tests/device/resident_batch.rs b/crates/j2k-metal/tests/device/resident_batch.rs new file mode 100644 index 00000000..a3c70571 --- /dev/null +++ b/crates/j2k-metal/tests/device/resident_batch.rs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn prepared_ht_rgb_u8_nhwc_resident_group_is_exact_and_uses_one_allocation() { + if !should_run_metal_runtime() { + return; + } + + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let roi = Rect { + x: 2, + y: 2, + w: 4, + h: 4, + }; + let inputs = vec![ + EncodedImage::new( + Arc::from(fixture_ht_rgb_u8_sized(8, 8, 0)), + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ), + EncodedImage::new( + Arc::from(fixture_ht_rgb_u8_sized(8, 8, 17)), + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU RGB U8 oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("six-bit RGB must use U8 batch storage") + }; + + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(inputs) + .expect("prepare resident HT RGB U8 group"); + let result = decoder + .decode_prepared(&prepared) + .expect("decode resident HT RGB U8 group"); + assert!(result.group_errors().is_empty()); + let group = &result.groups()[0]; + assert_eq!(group.info().layout, BatchLayout::Nhwc); + assert_eq!(group.surfaces().len(), 2); + let image_bytes = expected.len() / group.surfaces().len(); + for (index, surface) in group.surfaces().iter().enumerate() { + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + assert_eq!( + surface.as_bytes().expect("resident RGB U8 bytes").as_ref(), + &expected[index * image_bytes..(index + 1) * image_bytes] + ); + } + let first_range = group.surfaces()[0] + .memory_range() + .expect("first resident RGB U8 range"); + let second_range = group.surfaces()[1] + .memory_range() + .expect("second resident RGB U8 range"); + assert_eq!(first_range.allocation, second_range.allocation); + assert_eq!(first_range.offset, 0); + assert_eq!(second_range.offset, image_bytes); + assert_eq!(first_range.len, image_bytes); + assert_eq!(second_range.len, image_bytes); +} + +#[test] +fn prepared_ht_rgb_u16_nhwc_resident_group_is_exact_and_uses_one_allocation() { + if !should_run_metal_runtime() { + return; + } + + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::full(Arc::from(fixture_ht_rgb_u16_sized(8, 8, 0))), + EncodedImage::full(Arc::from(fixture_ht_rgb_u16_sized(8, 8, 257))), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU RGB U16 oracle"); + let CpuBatchSamples::U16(expected) = expected.groups()[0].samples() else { + panic!("twelve-bit RGB must use U16 batch storage") + }; + let expected_bytes = expected + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(inputs) + .expect("prepare resident HT RGB U16 group"); + let result = decoder + .decode_prepared(&prepared) + .expect("decode resident HT RGB U16 group"); + assert!(result.group_errors().is_empty()); + let group = &result.groups()[0]; + assert_eq!(group.info().layout, BatchLayout::Nhwc); + assert_eq!(group.surfaces().len(), 2); + let image_bytes = expected_bytes.len() / group.surfaces().len(); + for (index, surface) in group.surfaces().iter().enumerate() { + assert_eq!(surface.pixel_format(), PixelFormat::Rgb16); + assert_eq!( + surface.as_bytes().expect("resident RGB U16 bytes").as_ref(), + &expected_bytes[index * image_bytes..(index + 1) * image_bytes] + ); + } + let first_range = group.surfaces()[0] + .memory_range() + .expect("first resident RGB U16 range"); + let second_range = group.surfaces()[1] + .memory_range() + .expect("second resident RGB U16 range"); + assert_eq!(first_range.allocation, second_range.allocation); + assert_eq!(first_range.offset, 0); + assert_eq!(second_range.offset, image_bytes); + assert_eq!(first_range.len, image_bytes); + assert_eq!(second_range.len, image_bytes); +} + +#[test] +fn metal_prepared_batch_continues_after_one_group_execution_failure() { + if !should_run_metal_runtime() { + return; + } + + let options = BatchDecodeOptions { + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(unsupported_classic_roi_rgb()), + EncodedImage::full(Arc::from(fixture_ht_gray8())), + ]) + .expect("prepare two distinct Metal groups"); + assert_eq!(prepared.groups().len(), 2); + + let result = decoder + .decode_prepared(&prepared) + .expect("unsupported RGN group must not abort the reusable Metal session"); + assert!(result.errors().is_empty()); + assert_eq!(result.groups().len(), 1); + assert_eq!(result.groups()[0].source_indices(), &[1]); + assert_eq!(result.group_errors().len(), 1); + assert_eq!(result.group_errors()[0].source_indices(), &[0]); + assert!(matches!( + result.group_errors()[0].source(), + Error::UnsupportedMetalRequest { .. } + )); +} + +#[test] +fn prepared_ht_rgb_resident_session_reuse_soak_stabilizes_scratch_retention() { + if !should_run_metal_runtime() { + return; + } + + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::full(Arc::from( + fixture_ht_rgb_u8_sized(8, 8, 0), + ))]) + .expect("prepare reusable resident HT RGB group"); + + for _ in 0..16 { + let result = decoder + .decode_prepared(&prepared) + .expect("warm resident HT RGB session"); + assert!(result.group_errors().is_empty()); + } + let stable = decoder + .backend_session() + .buffer_pool_diagnostics() + .expect("warm scratch-pool diagnostics"); + let retention = |diagnostics: j2k_metal::MetalBufferPoolsDiagnostics| { + ( + diagnostics.private.cached_bytes, + diagnostics.private.cached_buffers, + diagnostics.private.metadata_capacity, + diagnostics.private.peak_cached_bytes, + diagnostics.private.peak_cached_buffers, + diagnostics.shared.cached_bytes, + diagnostics.shared.cached_buffers, + diagnostics.shared.metadata_capacity, + diagnostics.shared.peak_cached_bytes, + diagnostics.shared.peak_cached_buffers, + ) + }; + let expected_retention = retention(stable); + + for iteration in 0..1_000 { + let result = decoder + .decode_prepared(&prepared) + .expect("soak resident HT RGB session"); + assert!(result.group_errors().is_empty()); + drop(result); + if iteration % 100 == 99 { + let diagnostics = decoder + .backend_session() + .buffer_pool_diagnostics() + .expect("periodic scratch-pool diagnostics"); + assert_eq!( + retention(diagnostics), + expected_retention, + "scratch retention grew after warmup at submission {}", + iteration + 1 + ); + } + } +} diff --git a/crates/j2k-metal/tests/device/tile_batch.rs b/crates/j2k-metal/tests/device/tile_batch.rs new file mode 100644 index 00000000..fff3a6c1 --- /dev/null +++ b/crates/j2k-metal/tests/device/tile_batch.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn tile_batch_decode_many_device_preserves_full_tile_order() { + if !should_run_metal_runtime() { + return; + } + + let classic_bytes = fixture_gray8(); + let reversed_bytes = fixture_gray8_reversed(); + let mut ctx = J2kContext::default(); + let mut pool = J2kScratchPool::new(); + let inputs = [classic_bytes.as_slice(), reversed_bytes.as_slice()]; + + let surfaces = Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Gray8, + BackendRequest::Metal, + ) + .expect("decode full-tile batch"); + + let mut classic_host_decoder = J2kDecoder::new(&classic_bytes).expect("classic host decoder"); + let mut classic_host = [0u8; 16]; + classic_host_decoder + .decode_into(&mut classic_host, 4, PixelFormat::Gray8) + .expect("classic host decode"); + + let mut reversed_host_decoder = + J2kDecoder::new(&reversed_bytes).expect("reversed host decoder"); + let mut reversed_host = [0u8; 16]; + reversed_host_decoder + .decode_into(&mut reversed_host, 4, PixelFormat::Gray8) + .expect("reversed host decode"); + + assert_eq!(surfaces.len(), 2); + assert_eq!(surfaces[0].backend_kind(), BackendKind::Metal); + assert_eq!(surfaces[1].backend_kind(), BackendKind::Metal); + assert_eq!( + surfaces[0].as_bytes().expect("surface byte access"), + classic_host.as_slice() + ); + assert_eq!( + surfaces[1].as_bytes().expect("surface byte access"), + reversed_host.as_slice() + ); +} + +#[test] +fn metal_tile_batch_supports_region_and_scaled_requests() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let roi = Rect { + x: 0, + y: 0, + w: 2, + h: 2, + }; + let mut batch = MetalTileBatch::with_capacity(2); + + assert_eq!( + batch + .push_tile_request( + &bytes, + MetalDecodeRequest::region(PixelFormat::Gray8, roi, BackendRequest::Metal) + ) + .expect("push region tile"), + 0 + ); + assert_eq!( + batch + .push_tile_request( + &bytes, + MetalDecodeRequest::scaled( + PixelFormat::Gray8, + Downscale::Half, + BackendRequest::Metal + ) + ) + .expect("push scaled tile"), + 1 + ); + + let surfaces = batch.decode_all().expect("batch decode"); + assert_eq!(surfaces.len(), 2); + assert_eq!(surfaces[0].dimensions(), (2, 2)); + assert_eq!(surfaces[1].dimensions(), (2, 2)); + assert_eq!(surfaces[0].backend_kind(), BackendKind::Metal); + assert_eq!(surfaces[1].backend_kind(), BackendKind::Metal); +} + +#[test] +fn metal_tile_batch_supports_region_scaled_requests() { + if !should_run_metal_runtime() { + return; + } + + let bytes = fixture_gray8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut batch = MetalTileBatch::with_capacity(1); + + assert_eq!( + batch + .push_tile_request( + &bytes, + MetalDecodeRequest::region_scaled( + PixelFormat::Gray8, + roi, + scale, + BackendRequest::Metal + ) + ) + .expect("push region scaled tile"), + 0 + ); + + let surfaces = batch.decode_all().expect("batch decode"); + assert_eq!(surfaces.len(), 1); + assert_eq!(surfaces[0].dimensions(), (scaled.w, scaled.h)); + assert_eq!(surfaces[0].backend_kind(), BackendKind::Metal); +} + +#[test] +fn submitted_distinct_region_scaled_htj2k_grayscale_tiles_flush_as_one_device_batch() { + if !should_run_metal_runtime() { + return; + } + + let ht_bytes = fixture_ht_gray8(); + let reversed_bytes = fixture_ht_gray8_reversed(); + assert_ne!(ht_bytes, reversed_bytes, "HTJ2K batch fixtures must differ"); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + + let ht_submission = submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &ht_bytes, + PixelFormat::Gray8, + roi, + scale, + BackendRequest::Metal, + ) + .expect("submit ht region-scaled tile"); + let reversed_submission = submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &reversed_bytes, + PixelFormat::Gray8, + roi, + scale, + BackendRequest::Metal, + ) + .expect("submit reversed ht region-scaled tile"); + + assert_eq!( + session.submissions().expect("session submissions"), + 0, + "region-scaled submitted tile surfaces should stay queued until wait" + ); + + let expected = [&ht_bytes, &reversed_bytes] + .into_iter() + .map(|bytes| { + let mut decoder = J2kDecoder::new(bytes).expect("host decoder"); + let stride = scaled.w as usize; + let mut host = vec![0u8; stride * scaled.h as usize]; + decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region-scaled decode"); + host + }) + .collect::>(); + + let ht_surface = ht_submission.wait().expect("ht region-scaled surface"); + let reversed_surface = reversed_submission + .wait() + .expect("reversed ht region-scaled surface"); + assert_eq!(ht_surface.backend_kind(), BackendKind::Metal); + assert_eq!(reversed_surface.backend_kind(), BackendKind::Metal); + assert_eq!(ht_surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(reversed_surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + ht_surface.as_bytes().expect("surface byte access"), + expected[0].as_slice() + ); + assert_eq!( + reversed_surface.as_bytes().expect("surface byte access"), + expected[1].as_slice() + ); + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "distinct queued HTJ2K region-scaled grayscale tiles should flush through one Metal command buffer" + ); +} + +#[test] +fn submitted_distinct_region_scaled_htj2k_gray16_tiles_flush_as_one_device_batch() { + if !should_run_metal_runtime() { + return; + } + + let first_bytes = fixture_ht_gray12_offset(0); + let second_bytes = fixture_ht_gray12_offset(37); + assert_ne!( + first_bytes, second_bytes, + "HTJ2K Gray16 batch fixtures must differ" + ); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut ctx = J2kContext::default(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + + let first_submission = submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &first_bytes, + PixelFormat::Gray16, + roi, + scale, + BackendRequest::Metal, + ) + .expect("submit first ht gray16 region-scaled tile"); + let second_submission = submit_tile_region_scaled_to_device!( + &mut ctx, + &mut session, + &mut pool, + &second_bytes, + PixelFormat::Gray16, + roi, + scale, + BackendRequest::Metal, + ) + .expect("submit second ht gray16 region-scaled tile"); + + let expected = [&first_bytes, &second_bytes] + .into_iter() + .map(|bytes| { + let mut decoder = J2kDecoder::new(bytes).expect("host decoder"); + let stride = scaled.w as usize * PixelFormat::Gray16.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Gray16, + roi, + scale, + ) + .expect("host region-scaled gray16 decode"); + host + }) + .collect::>(); + + let first_surface = first_submission.wait().expect("first gray16 surface"); + let second_surface = second_submission.wait().expect("second gray16 surface"); + assert_eq!(first_surface.backend_kind(), BackendKind::Metal); + assert_eq!(second_surface.backend_kind(), BackendKind::Metal); + assert_eq!(first_surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(second_surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!( + first_surface.as_bytes().expect("surface byte access"), + expected[0].as_slice() + ); + assert_eq!( + second_surface.as_bytes().expect("surface byte access"), + expected[1].as_slice() + ); + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "distinct queued HTJ2K region-scaled Gray16 tiles should flush through one Metal command buffer" + ); +} diff --git a/crates/j2k-metal/tests/shader_integrity.rs b/crates/j2k-metal/tests/shader_integrity.rs index 12671551..562cb138 100644 --- a/crates/j2k-metal/tests/shader_integrity.rs +++ b/crates/j2k-metal/tests/shader_integrity.rs @@ -5,6 +5,7 @@ const HOST_SOURCE: &str = concat!( "\n", include_str!("../src/compute/runtime.rs"), ); +const NATIVE_COLOR_BATCH_SOURCE: &str = include_str!("../src/store_native_color_batch.metal"); const SHADER_SOURCES: &[&str] = &[ HOST_SOURCE, include_str!("../src/classic.metal"), @@ -22,8 +23,14 @@ const SHADER_SOURCES: &[&str] = &[ include_str!("../src/mct.metal"), include_str!("../src/quantize.metal"), include_str!("../src/store.metal"), + NATIVE_COLOR_BATCH_SOURCE, ]; +#[test] +fn shader_inventory_includes_batch_capable_native_color_store() { + assert!(SHADER_SOURCES.contains(&NATIVE_COLOR_BATCH_SOURCE)); +} + #[test] fn metal_kernels_are_wired_to_host_pipelines() { let unused = unwired_metal_kernels(SHADER_SOURCES.iter().copied(), HOST_SOURCE); @@ -33,3 +40,63 @@ fn metal_kernels_are_wired_to_host_pipelines() { "Metal kernels must be compiled by host pipeline setup or removed: {unused:?}" ); } + +fn metal_function_body<'a>(source: &'a str, signature: &str) -> &'a str { + let start = source + .find(signature) + .unwrap_or_else(|| panic!("missing Metal function `{signature}`")); + let open = source[start..].find('{').map_or_else( + || panic!("Metal function `{signature}` has no body"), + |offset| start + offset, + ); + let mut depth = 0usize; + for (offset, byte) in source.as_bytes()[open..].iter().copied().enumerate() { + match byte { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + return &source[open + 1..open + offset]; + } + } + _ => {} + } + } + panic!("Metal function `{signature}` has an unterminated body"); +} + +#[test] +fn cleanup_only_ht_pipelines_cannot_instantiate_refinement_state() { + const SOURCE: &str = include_str!("../src/ht_cleanup.metal"); + + let common = metal_function_body(SOURCE, "inline void decode_ht_cleanup_common("); + let strict_pass_rejection = common + .find("params.number_of_coding_passes > 3u") + .expect("common HT decode must reject coding-pass counts above three"); + let empty_refinement_normalization = common + .find("params.refinement_length == 0u") + .expect("common HT decode must retain empty-refinement normalization"); + assert!(strict_pass_rejection < empty_refinement_normalization); + assert!(!common.contains("J2K_HT_MAX_SIGMA")); + assert!(!common.contains("J2K_HT_MAX_PREV_ROW_SIG")); + + let cleanup = metal_function_body(SOURCE, "inline void decode_ht_cleanup_only_impl("); + assert!(cleanup.contains("decode_ht_cleanup_common(")); + assert!(!cleanup.contains("decode_ht_refinement_impl(")); + assert!(!cleanup.contains("sigma")); + assert!(!cleanup.contains("prev_row_sig")); + + let refinement = metal_function_body(SOURCE, "inline bool decode_ht_refinement_impl("); + assert!(refinement.contains("thread ushort sigma[J2K_HT_MAX_SIGMA]")); + assert!(refinement.contains("thread ushort prev_row_sig[J2K_HT_MAX_PREV_ROW_SIG]")); + + for kernel in [ + "kernel void j2k_decode_ht_cleanup_batched_cleanup_only(", + "kernel void j2k_decode_ht_cleanup_repeated_batched_cleanup_only(", + ] { + let body = metal_function_body(SOURCE, kernel); + assert!(body.contains("decode_ht_cleanup_only_impl(")); + assert!(!body.contains("decode_ht_cleanup_impl(")); + assert!(!body.contains("decode_ht_refinement_impl(")); + } +} diff --git a/crates/j2k-ml/Cargo.toml b/crates/j2k-ml/Cargo.toml index 27abe129..f097bb82 100644 --- a/crates/j2k-ml/Cargo.toml +++ b/crates/j2k-ml/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "j2k-ml" -description = "Independent Burn tensor integration for JPEG 2000 and HTJ2K" +description = "Thin Burn tensor adapter for JPEG 2000 and HTJ2K batch decoding" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -8,12 +8,12 @@ license.workspace = true repository.workspace = true publish = false readme = "README.md" +autobenches = false [features] default = [] cpu = [] cuda = [ - "cpu", "dep:burn-cuda", "dep:burn-cubecl", "dep:burn-fusion", @@ -21,7 +21,15 @@ cuda = [ "dep:j2k-cuda", "dep:j2k-cuda-runtime", ] -metal = ["cpu", "dep:burn-wgpu", "dep:j2k-metal"] +metal = [ + "dep:burn-cubecl", + "dep:burn-fusion", + "dep:burn-ir", + "dep:burn-wgpu", + "dep:j2k-metal", + "dep:metal", + "dep:wgpu-hal", +] [dependencies] burn-core = { workspace = true } @@ -32,13 +40,18 @@ burn-ir = { workspace = true, optional = true } burn-wgpu = { workspace = true, optional = true } j2k = { path = "../j2k", version = "=0.7.4" } j2k-cuda = { path = "../j2k-cuda", version = "=0.7.4", features = ["cuda-runtime"], optional = true } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.7.4", features = ["cuda-oxide-j2k-ml"], optional = true } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.7.4", optional = true } j2k-metal = { path = "../j2k-metal", version = "=0.7.4", optional = true } thiserror = { workspace = true } +[target.'cfg(target_os = "macos")'.dependencies] +metal = { workspace = true, optional = true } +wgpu-hal = { version = "29.0.4", default-features = false, features = ["metal"], optional = true } + [dev-dependencies] -burn-autodiff = { workspace = true } criterion = { workspace = true } +j2k-core = { path = "../j2k-core" } +j2k-native = { path = "../j2k-native" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [target.'cfg(all(target_arch = "aarch64", target_os = "linux"))'.dev-dependencies] @@ -67,8 +80,28 @@ required-features = ["metal"] name = "cuda" required-features = ["cuda"] +[[test]] +name = "cuda_rgba" +required-features = ["cuda"] + +[[test]] +name = "cuda_batch_sessions" +required-features = ["cuda"] + [[bench]] -name = "tensor_decode" +name = "batch_decode" harness = false test = false required-features = ["cpu"] + +[[bench]] +name = "batch_decode_metal" +harness = false +test = false +required-features = ["cpu", "metal"] + +[[bench]] +name = "batch_decode_cuda" +harness = false +test = false +required-features = ["cpu", "cuda"] diff --git a/crates/j2k-ml/README.md b/crates/j2k-ml/README.md index 0d20b126..f41d24a5 100644 --- a/crates/j2k-ml/README.md +++ b/crates/j2k-ml/README.md @@ -1,24 +1,16 @@ # j2k-ml -Experimental JPEG 2000 and HTJ2K tensor decoding for Burn 0.21. +Experimental thin Burn 0.21 adapter for the `j2k` owned batch codec. `j2k-ml` is an independent integration maintained by the `j2k` project. It is not an official Tracel or Burn crate. It remains unpublished during the 0.7 release cycle. -It decodes JP2, JPH, raw J2K, and raw HTJ2K inputs into rank-3 or rank-4 -Burn tensors. Defaults are channels-first layout, automatic Gray/RGB channel -selection, and unit-scaled `f32` output. +The codec crates own parsing, preparation, grouping, decoding, and device +execution. This crate only allocates ordinary rank-4 Burn integer tensors and +connects their storage to persistent CPU, CUDA, or Metal codec sessions. It +does not own normalization, augmentation, dataset policy, or another decoding +pipeline. -Enable one or more explicit routes: - -- `cpu`: portable host decode into any Burn backend. -- `metal`: strict resident J2K Metal decode, one packed batch readback, and one - compact upload to Burn Metal. -- `cuda`: strict direct decode and conversion into Burn's default fused CUDA - allocation. Device conversion kernels are Rust `cuda-oxide`; CUDA C, NVCC, - and C wrappers are not used. - -Every real decode API is fallible. Accelerator routes fail instead of silently -falling back, and batches reject corrupt items and shape mismatches with their -input index. See the [workspace integration guide](../../docs/j2k-ml.md). +See the single [Burn integration guide](../../docs/j2k-ml.md) for the API, +support matrix, safety boundary, validation commands, and benchmark status. diff --git a/crates/j2k-ml/benches/batch_decode.rs b/crates/j2k-ml/benches/batch_decode.rs new file mode 100644 index 00000000..6f943238 --- /dev/null +++ b/crates/j2k-ml/benches/batch_decode.rs @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use burn_core::tensor::backend::Backend; +#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))] +use burn_flex::{Flex as CpuBackend, FlexDevice as CpuDevice}; +#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +use burn_ndarray::NdArrayDevice::Cpu as CpuDevice; +use criterion::{BenchmarkId, Criterion, Throughput}; +use j2k::{BatchDecodeOptions, CpuBatchDecodeResult, CpuBatchDecoder}; +use j2k_ml::{BurnBatchDecode, CpuBurnDecoder}; + +mod support; + +use support::{ + decode_case::{decoded_pixels_per_batch, requests, require_prepared_success}, + input_selection::InputMode, + workload::{materialize_workload, workload_specs, WorkloadSpec}, + BATCH_SIZES, +}; + +#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +type CpuBackend = burn_ndarray::NdArray; + +fn main() { + let input_mode = InputMode::from_env().unwrap_or_else(|error| panic!("{error}")); + let workload_specs = workload_specs(); + let mut criterion = Criterion::default().configure_from_args(); + bench_codec(&mut criterion, &workload_specs, input_mode); + bench_burn(&mut criterion, &workload_specs, input_mode); + criterion.final_summary(); +} + +fn bench_codec(criterion: &mut Criterion, workload_specs: &[WorkloadSpec], input_mode: InputMode) { + let mut group = criterion.benchmark_group(format!( + "j2k_owned_batch_codec_cpu/input_{}", + input_mode.label() + )); + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + group.throughput(Throughput::Elements( + u64::try_from(batch_size).expect("benchmark batch size fits u64"), + )); + let preflight_session = CpuBatchDecoder::new(BatchDecodeOptions::default()); + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/prepare_images", workload.name), + batch_size, + ), + &inputs, + |bencher, inputs| { + bencher.iter(|| { + let prepared = preflight_session + .prepare(std::hint::black_box(inputs.clone())) + .expect("prepare CPU codec batch"); + require_prepared_success(&prepared); + std::hint::black_box(prepared) + }); + }, + ); + group.throughput(Throughput::Elements(decoded_pixels_per_batch( + output_pixels, + batch_size, + ))); + + let mut one_shot = CpuBatchDecoder::new(BatchDecodeOptions::default()); + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/end_to_end_pixels", workload.name), + batch_size, + ), + &inputs, + |bencher, inputs| { + bencher.iter(|| { + let decoded = one_shot + .decode(std::hint::black_box(inputs.clone())) + .expect("CPU batch decode"); + std::hint::black_box(require_codec_success(decoded)) + }); + }, + ); + + let mut prepared_decoder = CpuBatchDecoder::new(BatchDecodeOptions::default()); + let prepared = prepared_decoder + .prepare(inputs.clone()) + .expect("prepare CPU batch benchmark"); + require_prepared_success(&prepared); + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/prepared_pixels", workload.name), + batch_size, + ), + &prepared, + |bencher, prepared| { + bencher.iter(|| { + let decoded = prepared_decoder + .decode_prepared(std::hint::black_box(prepared)) + .expect("prepared CPU batch decode"); + std::hint::black_box(require_codec_success(decoded)) + }); + }, + ); + } + } + } + group.finish(); +} + +fn bench_burn(criterion: &mut Criterion, workload_specs: &[WorkloadSpec], input_mode: InputMode) { + let mut group = criterion.benchmark_group(format!( + "j2k_owned_batch_burn_cpu/input_{}", + input_mode.label() + )); + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + group.throughput(Throughput::Elements(decoded_pixels_per_batch( + output_pixels, + batch_size, + ))); + + let mut one_shot = + CpuBurnDecoder::::new(CpuDevice, BatchDecodeOptions::default()); + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/end_to_end_pixels", workload.name), + batch_size, + ), + &inputs, + |bencher, inputs| { + bencher.iter(|| { + let decoded = one_shot + .decode(std::hint::black_box(inputs.clone())) + .expect("CPU Burn batch decode"); + let decoded = require_burn_success(decoded); + CpuBackend::sync(&CpuDevice).expect("sync CPU Burn backend"); + std::hint::black_box(decoded) + }); + }, + ); + + let mut prepared_decoder = + CpuBurnDecoder::::new(CpuDevice, BatchDecodeOptions::default()); + let prepared = prepared_decoder + .prepare(inputs.clone()) + .expect("prepare CPU Burn benchmark"); + require_prepared_success(&prepared); + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/prepared_pixels", workload.name), + batch_size, + ), + &prepared, + |bencher, prepared| { + bencher.iter(|| { + let decoded = prepared_decoder + .decode_prepared(std::hint::black_box(prepared)) + .expect("prepared CPU Burn batch decode"); + let decoded = require_burn_success(decoded); + CpuBackend::sync(&CpuDevice).expect("sync CPU Burn backend"); + std::hint::black_box(decoded) + }); + }, + ); + } + } + } + group.finish(); +} + +fn require_codec_success(decoded: CpuBatchDecodeResult) -> CpuBatchDecodeResult { + assert!( + decoded.errors().is_empty(), + "benchmark decode returned indexed errors: {:?}", + decoded.errors() + ); + assert_eq!( + decoded.groups().len(), + 1, + "benchmark workload must decode to exactly one homogeneous group" + ); + decoded +} + +fn require_burn_success(decoded: BurnBatchDecode) -> BurnBatchDecode { + assert!( + decoded.errors.is_empty(), + "benchmark adapter returned indexed errors: {:?}", + decoded.errors + ); + assert!( + decoded.group_errors.is_empty(), + "benchmark adapter returned group errors: {:?}", + decoded.group_errors + ); + assert_eq!( + decoded.groups.len(), + 1, + "benchmark workload must materialize exactly one Burn tensor group" + ); + decoded +} diff --git a/crates/j2k-ml/benches/batch_decode_cuda.rs b/crates/j2k-ml/benches/batch_decode_cuda.rs new file mode 100644 index 00000000..a21937dc --- /dev/null +++ b/crates/j2k-ml/benches/batch_decode_cuda.rs @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use burn_core::tensor::backend::Backend; +use burn_cuda::{Cuda, CudaDevice}; +use criterion::{BenchmarkId, Criterion, Throughput}; +use j2k::{BatchDecodeOptions, EncodedImage, PreparedBatch}; +use j2k_cuda::{CudaBatchDecodeResult, CudaBatchDecoder}; +use j2k_ml::{BurnBatchDecode, CpuBurnDecoder, CudaBurnDecoder}; + +mod cuda_telemetry; +#[path = "support/process_policy.rs"] +mod process_policy; +mod support; + +use cuda_telemetry::{ + capture_burn_telemetry, capture_codec_telemetry, print_cuda_telemetry, CudaTelemetryCase, + CudaTelemetryRow, +}; +use process_policy::ProcessMode; +use support::{ + decode_case::{decoded_pixels_per_batch, requests, require_prepared_success}, + input_selection::InputMode, + workload::{materialize_workload, workload_specs, Workload, WorkloadSpec}, + BATCH_SIZES, LOW_BATCH_SIZES, +}; + +fn main() { + let input_mode = InputMode::from_env().unwrap_or_else(|error| panic!("{error}")); + let process_mode = ProcessMode::from_env().unwrap_or_else(|error| panic!("{error}")); + let workload_specs = workload_specs(); + match process_mode { + ProcessMode::Criterion => { + let mut criterion = Criterion::default().configure_from_args(); + bench_codec_resident(&mut criterion, &workload_specs, input_mode); + bench_burn_direct(&mut criterion, &workload_specs, input_mode); + criterion.final_summary(); + } + ProcessMode::Profile => { + let mut telemetry = profile_codec_resident(&workload_specs, input_mode); + telemetry.extend(profile_burn_direct(&workload_specs, input_mode)); + print_cuda_telemetry(&telemetry); + } + } +} + +fn bench_codec_resident( + criterion: &mut Criterion, + workload_specs: &[WorkloadSpec], + input_mode: InputMode, +) { + let options = BatchDecodeOptions::default(); + let mut group = criterion.benchmark_group(format!( + "j2k_owned_batch_codec_resident_cuda/input_{}", + input_mode.label() + )); + + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + let mut one_shot = CudaBatchDecoder::with_options(options); + let mut prepared_decoder = CudaBatchDecoder::with_options(options); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + group.throughput(Throughput::Elements( + u64::try_from(batch_size).expect("benchmark batch size fits u64"), + )); + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/prepare_images", workload.name), + batch_size, + ), + &inputs, + |bencher, inputs| { + bencher.iter(|| { + std::hint::black_box( + one_shot + .prepare(std::hint::black_box(inputs.clone())) + .expect("prepare CUDA codec batch"), + ) + }); + }, + ); + group.throughput(Throughput::Elements(decoded_pixels_per_batch( + output_pixels, + batch_size, + ))); + + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/one_shot_pixels", workload.name), + batch_size, + ), + &inputs, + |bencher, inputs| { + bencher.iter(|| { + // `decode_batch` retires the resident CUDA work before returning. + let decoded = one_shot + .decode_batch(std::hint::black_box(inputs.clone())) + .expect("CUDA codec-resident batch decode"); + std::hint::black_box(require_codec_success(decoded)) + }); + }, + ); + let prepared = prepared_decoder + .prepare(inputs.clone()) + .expect("prepare CUDA codec benchmark batch"); + require_prepared_success(&prepared); + bench_prepared_codec( + &mut group, + &workload, + request_name, + batch_size, + &prepared, + &mut prepared_decoder, + ); + } + } + } + group.finish(); +} + +fn bench_prepared_codec( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + workload: &Workload, + request_name: &str, + batch_size: usize, + prepared: &PreparedBatch, + decoder: &mut CudaBatchDecoder, +) { + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/prepared_pixels", workload.name), + batch_size, + ), + prepared, + |bencher, prepared| { + bencher.iter(|| { + // `decode_prepared` retires the resident CUDA work before returning. + let batch_result = decoder + .decode_prepared(std::hint::black_box(prepared)) + .expect("prepared CUDA codec-resident batch decode"); + std::hint::black_box(require_codec_success(batch_result)) + }); + }, + ); +} + +fn bench_burn_direct( + criterion: &mut Criterion, + workload_specs: &[WorkloadSpec], + input_mode: InputMode, +) { + let options = BatchDecodeOptions::default(); + let device = CudaDevice::default(); + let mut group = criterion.benchmark_group(format!( + "j2k_owned_batch_burn_direct_cuda/input_{}", + input_mode.label() + )); + + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + let mut one_shot = CudaBurnDecoder::new(device.clone(), options) + .expect("create CUDA Burn benchmark session"); + let mut prepared_decoder = CudaBurnDecoder::new(device.clone(), options) + .expect("create prepared CUDA Burn benchmark session"); + let mut staged = CpuBurnDecoder::::new(device.clone(), options); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + group.throughput(Throughput::Elements(decoded_pixels_per_batch( + output_pixels, + batch_size, + ))); + let case = BurnBenchCase { + workload: &workload, + request_name, + batch_size, + inputs: &inputs, + }; + bench_staged_burn(&mut group, case, &mut staged, &device); + bench_one_shot_burn(&mut group, case, &mut one_shot, &device); + bench_prepared_burn(&mut group, case, &mut prepared_decoder, &device); + } + } + } + group.finish(); +} + +fn profile_codec_resident( + workload_specs: &[WorkloadSpec], + input_mode: InputMode, +) -> Vec { + let options = BatchDecodeOptions::default(); + let mut telemetry = Vec::new(); + + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + let mut one_shot = CudaBatchDecoder::with_options(options); + let mut prepared_decoder = CudaBatchDecoder::with_options(options); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in LOW_BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + capture_codec_telemetry( + &mut telemetry, + CudaTelemetryCase::new( + "codec_resident", + "one_shot", + &workload, + request_name, + batch_size, + output_pixels, + ), + &mut one_shot, + |decoder| { + decoder + .decode_batch(inputs.clone()) + .map(require_codec_success) + }, + ); + let prepared = prepared_decoder + .prepare(inputs.clone()) + .expect("prepare CUDA codec profile batch"); + require_prepared_success(&prepared); + capture_codec_telemetry( + &mut telemetry, + CudaTelemetryCase::new( + "codec_resident", + "prepared", + &workload, + request_name, + batch_size, + output_pixels, + ), + &mut prepared_decoder, + |decoder| { + decoder + .decode_prepared(&prepared) + .map(require_codec_success) + }, + ); + } + } + } + telemetry +} + +fn profile_burn_direct( + workload_specs: &[WorkloadSpec], + input_mode: InputMode, +) -> Vec { + let options = BatchDecodeOptions::default(); + let device = CudaDevice::default(); + let mut telemetry = Vec::new(); + + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + let mut one_shot = CudaBurnDecoder::new(device.clone(), options) + .expect("create CUDA Burn profile session"); + let mut prepared_decoder = CudaBurnDecoder::new(device.clone(), options) + .expect("create prepared CUDA Burn profile session"); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in LOW_BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + capture_burn_telemetry( + &mut telemetry, + CudaTelemetryCase::new( + "burn_direct", + "one_shot", + &workload, + request_name, + batch_size, + output_pixels, + ), + &mut one_shot, + |decoder| decoder.decode(inputs.clone()).map(require_burn_success), + ); + let prepared = prepared_decoder + .prepare(inputs.clone()) + .expect("prepare CUDA Burn profile batch"); + require_prepared_success(&prepared); + capture_burn_telemetry( + &mut telemetry, + CudaTelemetryCase::new( + "burn_direct", + "prepared", + &workload, + request_name, + batch_size, + output_pixels, + ), + &mut prepared_decoder, + |decoder| decoder.decode_prepared(&prepared).map(require_burn_success), + ); + } + } + } + telemetry +} + +#[derive(Clone, Copy)] +struct BurnBenchCase<'a> { + workload: &'a Workload, + request_name: &'static str, + batch_size: usize, + inputs: &'a [EncodedImage], +} + +fn bench_staged_burn( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + case: BurnBenchCase<'_>, + staged: &mut CpuBurnDecoder, + device: &CudaDevice, +) { + group.bench_with_input( + BenchmarkId::new( + format!( + "{}/{}/staged_cpu_upload_pixels", + case.workload.name, case.request_name + ), + case.batch_size, + ), + case.inputs, + |bencher, inputs| { + bencher.iter(|| { + let completed_batch = staged + .decode(std::hint::black_box(inputs.to_vec())) + .expect("CPU-staged CUDA Burn batch decode"); + let completed_batch = require_burn_success(completed_batch); + ::sync(device) + .expect("synchronize staged CUDA Burn benchmark completion"); + std::hint::black_box(completed_batch) + }); + }, + ); +} + +fn bench_one_shot_burn( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + case: BurnBenchCase<'_>, + decoder: &mut CudaBurnDecoder, + device: &CudaDevice, +) { + group.bench_with_input( + BenchmarkId::new( + format!( + "{}/{}/one_shot_pixels", + case.workload.name, case.request_name + ), + case.batch_size, + ), + case.inputs, + |bencher, inputs| { + bencher.iter(|| { + let completed_batch = decoder + .decode(std::hint::black_box(inputs.to_vec())) + .expect("CUDA Burn-direct batch decode"); + let completed_batch = require_burn_success(completed_batch); + ::sync(device) + .expect("synchronize CUDA Burn benchmark completion"); + std::hint::black_box(completed_batch) + }); + }, + ); +} + +fn bench_prepared_burn( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + case: BurnBenchCase<'_>, + decoder: &mut CudaBurnDecoder, + device: &CudaDevice, +) { + let prepared = decoder + .prepare(case.inputs.to_vec()) + .expect("prepare CUDA Burn benchmark batch"); + require_prepared_success(&prepared); + group.bench_with_input( + BenchmarkId::new( + format!( + "{}/{}/prepared_pixels", + case.workload.name, case.request_name + ), + case.batch_size, + ), + &prepared, + |bencher, prepared| { + bencher.iter(|| { + let completed_batch = decoder + .decode_prepared(std::hint::black_box(prepared)) + .expect("prepared CUDA Burn-direct batch decode"); + let completed_batch = require_burn_success(completed_batch); + ::sync(device) + .expect("synchronize prepared CUDA Burn benchmark completion"); + std::hint::black_box(completed_batch) + }); + }, + ); +} + +fn require_codec_success(decoded: CudaBatchDecodeResult) -> CudaBatchDecodeResult { + assert!( + decoded.errors().is_empty(), + "benchmark decode returned indexed errors: {:?}", + decoded.errors() + ); + assert!( + decoded.group_errors().is_empty(), + "benchmark decode returned group errors: {:?}", + decoded.group_errors() + ); + assert_eq!( + decoded.groups().len(), + 1, + "benchmark workload must decode to exactly one homogeneous group" + ); + decoded +} + +fn require_burn_success(decoded: BurnBatchDecode) -> BurnBatchDecode { + assert!( + decoded.errors.is_empty(), + "benchmark adapter returned indexed errors: {:?}", + decoded.errors + ); + assert!( + decoded.group_errors.is_empty(), + "benchmark adapter returned group errors: {:?}", + decoded.group_errors + ); + assert_eq!( + decoded.groups.len(), + 1, + "benchmark workload must materialize exactly one Burn tensor group" + ); + decoded +} diff --git a/crates/j2k-ml/benches/batch_decode_metal.rs b/crates/j2k-ml/benches/batch_decode_metal.rs new file mode 100644 index 00000000..a2f5621e --- /dev/null +++ b/crates/j2k-ml/benches/batch_decode_metal.rs @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use burn_core::tensor::backend::Backend; +use burn_wgpu::{Wgpu, WgpuDevice}; +use criterion::{BenchmarkId, Criterion, Throughput}; +use j2k::{BatchDecodeOptions, BatchLayout, EncodedImage, PreparedBatch}; +use j2k_metal::{MetalBatchDecodeResult, MetalBatchDecoder}; +use j2k_ml::{BurnBatchDecode, CpuBurnDecoder, MetalBurnDecoder}; + +#[path = "support/metal_process_policy.rs"] +mod metal_process_policy; +mod metal_telemetry; +#[path = "support/process_policy.rs"] +mod process_policy; +mod support; + +use metal_process_policy::ensure_metal_criterion_instrumentation_disabled; +use metal_telemetry::{ + capture_burn_telemetry, capture_codec_telemetry, capture_staged_telemetry, + print_metal_telemetry, MetalTelemetryCase, MetalTelemetryRow, +}; +use process_policy::ProcessMode; +use support::{ + decode_case::{decoded_pixels_per_batch, requests, require_prepared_success}, + input_selection::InputMode, + workload::{materialize_workload, workload_specs, Workload, WorkloadSpec}, + BATCH_SIZES, LOW_BATCH_SIZES, +}; + +fn main() { + let input_mode = InputMode::from_env().unwrap_or_else(|error| panic!("{error}")); + let process_mode = ProcessMode::from_env().unwrap_or_else(|error| panic!("{error}")); + let workload_specs = workload_specs(); + match process_mode { + ProcessMode::Criterion => { + ensure_metal_criterion_instrumentation_disabled() + .unwrap_or_else(|error| panic!("{error}")); + let mut criterion = Criterion::default().configure_from_args(); + bench_codec_resident(&mut criterion, &workload_specs, input_mode); + bench_burn_direct(&mut criterion, &workload_specs, input_mode); + criterion.final_summary(); + } + ProcessMode::Profile => { + let mut telemetry = profile_codec_resident(&workload_specs, input_mode); + telemetry.extend(profile_burn_direct(&workload_specs, input_mode)); + print_metal_telemetry(&telemetry); + } + } +} + +fn bench_codec_resident( + criterion: &mut Criterion, + workload_specs: &[WorkloadSpec], + input_mode: InputMode, +) { + // Resident RGB groups expose ordinary interleaved `Surface` views, so the + // codec-resident color matrix is explicitly NHWC. Grayscale is layout + // equivalent and shares the same retained sessions. + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut group = criterion.benchmark_group(format!( + "j2k_owned_batch_codec_resident_metal/input_{}", + input_mode.label() + )); + + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + let mut one_shot = MetalBatchDecoder::system_default_with_options(options) + .expect("create persistent Metal codec benchmark session"); + let mut prepared_decoder = MetalBatchDecoder::system_default_with_options(options) + .expect("create prepared Metal codec benchmark session"); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + group.throughput(Throughput::Elements( + u64::try_from(batch_size).expect("benchmark batch size fits u64"), + )); + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/prepare_images", workload.name), + batch_size, + ), + &inputs, + |bencher, inputs| { + bencher.iter(|| { + std::hint::black_box( + one_shot + .prepare(std::hint::black_box(inputs.clone())) + .expect("prepare Metal codec batch"), + ) + }); + }, + ); + group.throughput(Throughput::Elements(decoded_pixels_per_batch( + output_pixels, + batch_size, + ))); + + bench_one_shot_codec( + &mut group, + &workload, + request_name, + batch_size, + &inputs, + &mut one_shot, + ); + + let prepared = prepared_decoder + .prepare(inputs.clone()) + .expect("prepare Metal codec benchmark batch"); + require_prepared_success(&prepared); + bench_prepared_codec( + &mut group, + &workload, + request_name, + batch_size, + &prepared, + &mut prepared_decoder, + ); + } + } + } + group.finish(); +} + +fn bench_one_shot_codec( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + workload: &Workload, + request_name: &str, + batch_size: usize, + inputs: &[EncodedImage], + decoder: &mut MetalBatchDecoder, +) { + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/one_shot_pixels", workload.name), + batch_size, + ), + inputs, + |bencher, inputs| { + bencher.iter(|| { + // `decode_batch` waits for resident Metal output before returning. + let batch_result = decoder + .decode_batch(std::hint::black_box(inputs.to_vec())) + .expect("Metal codec-resident batch decode"); + std::hint::black_box(require_codec_success(batch_result)) + }); + }, + ); +} + +fn bench_prepared_codec( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + workload: &Workload, + request_name: &str, + batch_size: usize, + prepared: &PreparedBatch, + decoder: &mut MetalBatchDecoder, +) { + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/prepared_pixels", workload.name), + batch_size, + ), + prepared, + |bencher, prepared| { + bencher.iter(|| { + // `decode_prepared` waits for the resident surfaces before returning. + let batch_result = decoder + .decode_prepared(std::hint::black_box(prepared)) + .expect("prepared Metal codec-resident batch decode"); + std::hint::black_box(require_codec_success(batch_result)) + }); + }, + ); +} + +fn bench_burn_direct( + criterion: &mut Criterion, + workload_specs: &[WorkloadSpec], + input_mode: InputMode, +) { + let options = BatchDecodeOptions::default(); + let mut group = criterion.benchmark_group(format!( + "j2k_owned_batch_burn_direct_metal/input_{}", + input_mode.label() + )); + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + let mut one_shot = MetalBurnDecoder::system_default(options) + .expect("create paired Metal Burn benchmark session"); + let mut prepared_decoder = MetalBurnDecoder::system_default(options) + .expect("create prepared paired Metal Burn benchmark session"); + let one_shot_device = one_shot.device().clone(); + let mut staged = CpuBurnDecoder::::new(one_shot_device.clone(), options); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + group.throughput(Throughput::Elements(decoded_pixels_per_batch( + output_pixels, + batch_size, + ))); + bench_staged_burn( + &mut group, + &workload, + request_name, + batch_size, + &inputs, + &mut staged, + &one_shot_device, + ); + bench_one_shot_burn( + &mut group, + &workload, + request_name, + batch_size, + &inputs, + &mut one_shot, + ); + let prepared = prepared_decoder + .prepare(inputs.clone()) + .expect("prepare Metal Burn benchmark batch"); + require_prepared_success(&prepared); + bench_prepared_burn( + &mut group, + &workload, + request_name, + batch_size, + &prepared, + &mut prepared_decoder, + ); + } + } + } + group.finish(); +} + +fn profile_codec_resident( + workload_specs: &[WorkloadSpec], + input_mode: InputMode, +) -> Vec { + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut telemetry = Vec::new(); + + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + let mut one_shot = MetalBatchDecoder::system_default_with_options(options) + .expect("create Metal codec profile session"); + let mut prepared_decoder = MetalBatchDecoder::system_default_with_options(options) + .expect("create prepared Metal codec profile session"); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in LOW_BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + capture_codec_telemetry( + &mut telemetry, + MetalTelemetryCase::new( + "codec_resident", + "one_shot", + &workload, + request_name, + batch_size, + output_pixels, + 0, + ), + &mut one_shot, + |decoder| { + decoder + .decode_batch(inputs.clone()) + .map(require_codec_success) + }, + ); + let prepared = prepared_decoder + .prepare(inputs.clone()) + .expect("prepare Metal codec profile batch"); + require_prepared_success(&prepared); + capture_codec_telemetry( + &mut telemetry, + MetalTelemetryCase::new( + "codec_resident", + "prepared", + &workload, + request_name, + batch_size, + output_pixels, + 0, + ), + &mut prepared_decoder, + |decoder| { + decoder + .decode_prepared(&prepared) + .map(require_codec_success) + }, + ); + } + } + } + telemetry +} + +fn profile_burn_direct( + workload_specs: &[WorkloadSpec], + input_mode: InputMode, +) -> Vec { + let options = BatchDecodeOptions::default(); + let mut telemetry = Vec::new(); + + for &spec in workload_specs { + let workload = materialize_workload(spec, input_mode); + let mut one_shot = MetalBurnDecoder::system_default(options) + .expect("create paired Metal Burn profile session"); + let mut prepared_decoder = MetalBurnDecoder::system_default(options) + .expect("create prepared paired Metal Burn profile session"); + let device = one_shot.device().clone(); + let mut staged = CpuBurnDecoder::::new(device.clone(), options); + for (request_name, request, output_pixels) in requests(workload.dimensions, true) { + for &batch_size in LOW_BATCH_SIZES { + let inputs = workload.inputs(request, batch_size); + capture_staged_telemetry( + &mut telemetry, + MetalTelemetryCase::new( + "burn_staged", + "cpu_upload", + &workload, + request_name, + batch_size, + output_pixels, + 1, + ), + || { + let decoded = staged + .decode(inputs.clone()) + .expect("staged Metal telemetry decode"); + let decoded = require_burn_success(decoded); + ::sync(&device) + .expect("synchronize staged Metal telemetry completion"); + decoded + }, + ); + capture_burn_telemetry( + &mut telemetry, + MetalTelemetryCase::new( + "burn_direct", + "one_shot", + &workload, + request_name, + batch_size, + output_pixels, + 0, + ), + &mut one_shot, + |decoder| decoder.decode(inputs.clone()).map(require_burn_success), + ); + let prepared = prepared_decoder + .prepare(inputs.clone()) + .expect("prepare Metal Burn profile batch"); + require_prepared_success(&prepared); + capture_burn_telemetry( + &mut telemetry, + MetalTelemetryCase::new( + "burn_direct", + "prepared", + &workload, + request_name, + batch_size, + output_pixels, + 0, + ), + &mut prepared_decoder, + |decoder| decoder.decode_prepared(&prepared).map(require_burn_success), + ); + } + } + } + telemetry +} + +fn bench_staged_burn( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + workload: &Workload, + request_name: &str, + batch_size: usize, + inputs: &[EncodedImage], + decoder: &mut CpuBurnDecoder, + device: &WgpuDevice, +) { + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/staged_cpu_upload_pixels", workload.name), + batch_size, + ), + inputs, + |bencher, inputs| { + bencher.iter(|| { + let batch_result = decoder + .decode(std::hint::black_box(inputs.to_vec())) + .expect("CPU-staged Metal Burn batch decode"); + let batch_result = require_burn_success(batch_result); + ::sync(device) + .expect("synchronize staged Metal Burn benchmark completion"); + std::hint::black_box(batch_result) + }); + }, + ); +} + +fn bench_one_shot_burn( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + workload: &Workload, + request_name: &str, + batch_size: usize, + inputs: &[EncodedImage], + decoder: &mut MetalBurnDecoder, +) { + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/one_shot_pixels", workload.name), + batch_size, + ), + inputs, + |bencher, inputs| { + bencher.iter(|| { + let batch_result = decoder + .decode(std::hint::black_box(inputs.to_vec())) + .expect("Metal Burn-direct batch decode"); + let batch_result = require_burn_success(batch_result); + std::hint::black_box(batch_result) + }); + }, + ); +} + +fn bench_prepared_burn( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + workload: &Workload, + request_name: &str, + batch_size: usize, + prepared: &PreparedBatch, + decoder: &mut MetalBurnDecoder, +) { + group.bench_with_input( + BenchmarkId::new( + format!("{}/{request_name}/prepared_pixels", workload.name), + batch_size, + ), + prepared, + |bencher, prepared| { + bencher.iter(|| { + let batch_result = decoder + .decode_prepared(std::hint::black_box(prepared)) + .expect("prepared Metal Burn-direct batch decode"); + let batch_result = require_burn_success(batch_result); + std::hint::black_box(batch_result) + }); + }, + ); +} + +fn require_codec_success(decoded: MetalBatchDecodeResult) -> MetalBatchDecodeResult { + assert!( + decoded.errors().is_empty(), + "benchmark decode returned indexed errors: {:?}", + decoded.errors() + ); + assert!( + decoded.group_errors().is_empty(), + "benchmark decode returned group errors: {:?}", + decoded.group_errors() + ); + assert_eq!( + decoded.groups().len(), + 1, + "benchmark workload must decode to exactly one homogeneous group" + ); + decoded +} + +fn require_burn_success(decoded: BurnBatchDecode) -> BurnBatchDecode { + assert!( + decoded.errors.is_empty(), + "benchmark adapter returned indexed errors: {:?}", + decoded.errors + ); + assert!( + decoded.group_errors.is_empty(), + "benchmark adapter returned group errors: {:?}", + decoded.group_errors + ); + assert_eq!( + decoded.groups.len(), + 1, + "benchmark workload must materialize exactly one Burn tensor group" + ); + decoded +} diff --git a/crates/j2k-ml/benches/cuda_telemetry.rs b/crates/j2k-ml/benches/cuda_telemetry.rs new file mode 100644 index 00000000..155a58ed --- /dev/null +++ b/crates/j2k-ml/benches/cuda_telemetry.rs @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::time::{Duration, Instant}; + +use burn_core::tensor::backend::Backend; +use burn_cuda::Cuda; +use j2k_cuda::{CudaBatchDecoder, CudaSessionDiagnostics}; +use j2k_ml::CudaBurnDecoder; + +use super::Workload; + +#[derive(Clone, Copy)] +pub(super) struct CudaTelemetryCase { + route: &'static str, + decode_mode: &'static str, + input_mode: &'static str, + workload: &'static str, + request: &'static str, + batch_size: u32, + decoded_megapixels: f64, +} + +impl CudaTelemetryCase { + pub(super) fn new( + route: &'static str, + decode_mode: &'static str, + workload: &Workload, + request: &'static str, + batch_size: usize, + output_pixels: u32, + ) -> Self { + let batch_size = u32::try_from(batch_size).expect("benchmark batch size fits u32"); + Self { + route, + decode_mode, + input_mode: workload.input_mode.label(), + workload: workload.name, + request, + batch_size, + decoded_megapixels: f64::from(output_pixels) * f64::from(batch_size) / 1_000_000.0, + } + } +} + +pub(super) struct CudaTelemetryRow { + case: CudaTelemetryCase, + elapsed: Duration, + before: CudaSessionDiagnostics, + after: CudaSessionDiagnostics, + consumer_host_synchronizations: u64, +} + +pub(super) fn capture_codec_telemetry( + telemetry: &mut Vec, + case: CudaTelemetryCase, + decoder: &mut CudaBatchDecoder, + decode: impl FnOnce(&mut CudaBatchDecoder) -> Result, +) where + E: core::fmt::Debug, +{ + let before = decoder.diagnostics().expect("pre-decode CUDA diagnostics"); + let start = Instant::now(); + let completed_batch = decode(decoder).expect("CUDA codec telemetry decode"); + let elapsed = start.elapsed(); + drop(completed_batch); + let after = decoder.diagnostics().expect("post-decode CUDA diagnostics"); + telemetry.push(CudaTelemetryRow { + case, + elapsed, + before, + after, + consumer_host_synchronizations: 0, + }); +} + +pub(super) fn capture_burn_telemetry( + telemetry: &mut Vec, + case: CudaTelemetryCase, + decoder: &mut CudaBurnDecoder, + decode: impl FnOnce(&mut CudaBurnDecoder) -> Result, +) where + E: core::fmt::Debug, +{ + let before = decoder + .codec() + .diagnostics() + .expect("pre-decode CUDA Burn diagnostics"); + let start = Instant::now(); + let completed_batch = decode(decoder).expect("CUDA Burn telemetry decode"); + ::sync(decoder.device()).expect("synchronize CUDA Burn telemetry completion"); + let elapsed = start.elapsed(); + drop(completed_batch); + let after = decoder + .codec() + .diagnostics() + .expect("post-decode CUDA Burn diagnostics"); + telemetry.push(CudaTelemetryRow { + case, + elapsed, + before, + after, + consumer_host_synchronizations: 1, + }); +} + +pub(super) fn print_cuda_telemetry(rows: &[CudaTelemetryRow]) { + println!( + "cuda_telemetry_v2,route,decode_mode,input_mode,workload,request,batch_size,probe_images_per_s,\ + probe_megapixels_per_s,h2d_ops,h2d_bytes,d2h_ops,d2h_bytes,status_d2h_ops,\ + status_d2h_bytes,codec_kernel_launches,runtime_device_allocations,\ + runtime_device_allocation_bytes,runtime_live_device_allocations_after,\ + runtime_live_device_bytes_after,session_peak_live_device_allocations_before,\ + session_peak_live_device_allocations_after,session_peak_live_device_bytes_before,\ + session_peak_live_device_bytes_after,codec_pool_retained_bytes_after,\ + session_pool_peak_retained_bytes_before,session_pool_peak_retained_bytes_after,\ + event_driver_allocations,event_reuses,\ + codec_event_host_syncs,codec_context_host_syncs,consumer_host_syncs" + ); + for row in rows { + let before = row.before.runtime.unwrap_or_default(); + let after = row + .after + .runtime + .expect("a completed CUDA telemetry decode initializes the runtime"); + let elapsed_seconds = row.elapsed.as_secs_f64(); + let images_per_second = f64::from(row.case.batch_size) / elapsed_seconds; + let megapixels_per_second = row.case.decoded_megapixels / elapsed_seconds; + println!( + concat!( + "cuda_telemetry_v2,{},{},{},{},{},{},", + "{:.3},{:.3},", + "{},{},{},{},{},{},{},{},{},{},", + "{},{},{},{},{},{},{},{},{},{},", + "{},{},{}" + ), + row.case.route, + row.case.decode_mode, + row.case.input_mode, + row.case.workload, + row.case.request, + row.case.batch_size, + images_per_second, + megapixels_per_second, + delta( + before.host_to_device_operations, + after.host_to_device_operations + ), + delta(before.host_to_device_bytes, after.host_to_device_bytes), + delta( + before.device_to_host_operations, + after.device_to_host_operations + ), + delta(before.device_to_host_bytes, after.device_to_host_bytes), + delta( + before.status_device_to_host_operations, + after.status_device_to_host_operations + ), + delta( + before.status_device_to_host_bytes, + after.status_device_to_host_bytes + ), + delta(before.kernel_launches, after.kernel_launches), + delta( + before.device_allocation_operations, + after.device_allocation_operations + ), + delta( + before.device_allocation_bytes, + after.device_allocation_bytes + ), + after.live_device_allocations, + after.live_device_bytes, + before.peak_live_device_allocations, + after.peak_live_device_allocations, + before.peak_live_device_bytes, + after.peak_live_device_bytes, + row.after.pools.retained_bytes(), + row.before.pools.peak_retained_bytes_upper_bound(), + row.after.pools.peak_retained_bytes_upper_bound(), + delta( + before.event_driver_allocations, + after.event_driver_allocations + ), + delta(before.event_reuses, after.event_reuses), + delta( + before.event_host_synchronizations, + after.event_host_synchronizations + ), + delta( + before.context_host_synchronizations, + after.context_host_synchronizations + ), + row.consumer_host_synchronizations, + ); + } +} + +const fn delta(before: u64, after: u64) -> u64 { + after.saturating_sub(before) +} diff --git a/crates/j2k-ml/benches/metal_telemetry.rs b/crates/j2k-ml/benches/metal_telemetry.rs new file mode 100644 index 00000000..4ef40ed5 --- /dev/null +++ b/crates/j2k-ml/benches/metal_telemetry.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::time::{Duration, Instant}; + +use j2k_metal::{MetalBatchDecoder, MetalBufferPoolsDiagnostics}; +use j2k_ml::MetalBurnDecoder; + +use super::Workload; + +#[derive(Clone, Copy)] +pub(super) struct MetalTelemetryCase { + route: &'static str, + decode_mode: &'static str, + input_mode: &'static str, + workload: &'static str, + request: &'static str, + batch_size: usize, + decoded_megapixels: f64, + asserted_decoded_host_uploads: u64, +} + +impl MetalTelemetryCase { + pub(super) fn new( + route: &'static str, + decode_mode: &'static str, + workload: &Workload, + request: &'static str, + batch_size: usize, + output_pixels: u32, + asserted_decoded_host_uploads: u64, + ) -> Self { + Self { + route, + decode_mode, + input_mode: workload.input_mode.label(), + workload: workload.name, + request, + batch_size, + decoded_megapixels: f64::from(output_pixels) + * f64::from(u32::try_from(batch_size).expect("benchmark batch size fits u32")) + / 1_000_000.0, + asserted_decoded_host_uploads, + } + } +} + +#[derive(Clone, Copy)] +struct MetalTelemetrySnapshot { + submissions: u64, + pools: MetalBufferPoolsDiagnostics, +} + +pub(super) struct MetalTelemetryRow { + case: MetalTelemetryCase, + elapsed: Duration, + before: Option, + after: Option, + asserted_codec_group_waits: u64, + asserted_consumer_host_synchronizations: u64, +} + +fn metal_snapshot(decoder: &MetalBatchDecoder) -> MetalTelemetrySnapshot { + MetalTelemetrySnapshot { + submissions: decoder + .submissions() + .expect("Metal telemetry submission count"), + pools: decoder + .backend_session() + .buffer_pool_diagnostics() + .expect("Metal telemetry buffer-pool diagnostics"), + } +} + +pub(super) fn capture_codec_telemetry( + telemetry: &mut Vec, + case: MetalTelemetryCase, + decoder: &mut MetalBatchDecoder, + decode: impl FnOnce(&mut MetalBatchDecoder) -> Result, +) where + E: core::fmt::Debug, +{ + let before = metal_snapshot(decoder); + let start = Instant::now(); + let completed = decode(decoder).expect("Metal codec telemetry decode"); + let elapsed = start.elapsed(); + drop(completed); + let after = metal_snapshot(decoder); + telemetry.push(MetalTelemetryRow { + case, + elapsed, + before: Some(before), + after: Some(after), + asserted_codec_group_waits: 1, + asserted_consumer_host_synchronizations: 0, + }); +} + +pub(super) fn capture_burn_telemetry( + telemetry: &mut Vec, + case: MetalTelemetryCase, + decoder: &mut MetalBurnDecoder, + decode: impl FnOnce(&mut MetalBurnDecoder) -> Result, +) where + E: core::fmt::Debug, +{ + let before = metal_snapshot(decoder.codec()); + let start = Instant::now(); + let completed = decode(decoder).expect("Metal Burn telemetry decode"); + let elapsed = start.elapsed(); + drop(completed); + let after = metal_snapshot(decoder.codec()); + telemetry.push(MetalTelemetryRow { + case, + elapsed, + before: Some(before), + after: Some(after), + asserted_codec_group_waits: 1, + asserted_consumer_host_synchronizations: 0, + }); +} + +pub(super) fn capture_staged_telemetry( + telemetry: &mut Vec, + case: MetalTelemetryCase, + decode: impl FnOnce() -> T, +) { + let start = Instant::now(); + let completed = decode(); + let elapsed = start.elapsed(); + drop(completed); + telemetry.push(MetalTelemetryRow { + case, + elapsed, + before: None, + after: None, + asserted_codec_group_waits: 0, + asserted_consumer_host_synchronizations: 1, + }); +} + +pub(super) fn print_metal_telemetry(rows: &[MetalTelemetryRow]) { + println!( + "metal_telemetry_v2,route,decode_mode,input_mode,workload,request,batch_size,probe_images_per_s,\ + probe_megapixels_per_s,asserted_decoded_host_uploads,asserted_decoded_host_readbacks,\ + asserted_final_output_allocations,asserted_final_device_copies,measured_codec_group_launches,\ + asserted_codec_group_waits,asserted_consumer_host_syncs,private_cached_buffers_after,\ + shared_cached_buffers_after,private_cached_bytes_after,shared_cached_bytes_after,\ + session_pool_peak_cached_bytes_before,session_pool_peak_cached_bytes_after" + ); + for row in rows { + let elapsed_seconds = row.elapsed.as_secs_f64(); + let images_per_second = + f64::from(u32::try_from(row.case.batch_size).expect("benchmark batch size fits u32")) + / elapsed_seconds; + let megapixels_per_second = row.case.decoded_megapixels / elapsed_seconds; + let before = row.before.unwrap_or(MetalTelemetrySnapshot { + submissions: 0, + pools: MetalBufferPoolsDiagnostics::default(), + }); + let after = row.after.unwrap_or(before); + println!( + concat!( + "metal_telemetry_v2,{},{},{},{},{},{},", + "{:.3},{:.3},{},{},{},{},{},{},{},", + "{},{},{},{},{},{}" + ), + row.case.route, + row.case.decode_mode, + row.case.input_mode, + row.case.workload, + row.case.request, + row.case.batch_size, + images_per_second, + megapixels_per_second, + row.case.asserted_decoded_host_uploads, + 0, + 1, + 0, + after.submissions.saturating_sub(before.submissions), + row.asserted_codec_group_waits, + row.asserted_consumer_host_synchronizations, + after.pools.private.cached_buffers, + after.pools.shared.cached_buffers, + after.pools.private.cached_bytes, + after.pools.shared.cached_bytes, + before + .pools + .private + .peak_cached_bytes + .saturating_add(before.pools.shared.peak_cached_bytes), + after + .pools + .private + .peak_cached_bytes + .saturating_add(after.pools.shared.peak_cached_bytes), + ); + } +} diff --git a/crates/j2k-ml/benches/support/decode_case.rs b/crates/j2k-ml/benches/support/decode_case.rs new file mode 100644 index 00000000..859fe203 --- /dev/null +++ b/crates/j2k-ml/benches/support/decode_case.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::{DecodeRequest, Downscale, PreparedBatch, Rect}; + +pub(crate) fn requests( + dimensions: (u32, u32), + include_region_reduced: bool, +) -> Vec<(&'static str, DecodeRequest, u32)> { + let (width, height) = dimensions; + let roi = centered_roi(dimensions); + let mut requests = vec![ + ("full", DecodeRequest::Full, width.saturating_mul(height)), + ( + "roi", + DecodeRequest::Region { roi }, + roi.w.saturating_mul(roi.h), + ), + ( + "reduced", + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + width.div_ceil(2).saturating_mul(height.div_ceil(2)), + ), + ]; + if include_region_reduced { + requests.push(( + "roi_reduced", + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + roi.w.div_ceil(2).saturating_mul(roi.h.div_ceil(2)), + )); + } + requests +} + +pub(crate) fn decoded_pixels_per_batch(output_pixels: u32, batch_size: usize) -> u64 { + u64::from(output_pixels) + .saturating_mul(u64::try_from(batch_size).expect("benchmark batch size fits u64")) +} + +pub(crate) fn require_prepared_success(prepared: &PreparedBatch) { + assert!( + prepared.errors().is_empty(), + "benchmark preparation returned indexed errors: {:?}", + prepared.errors() + ); + assert_eq!( + prepared.groups().len(), + 1, + "benchmark workload must prepare exactly one homogeneous group" + ); +} + +fn centered_roi((width, height): (u32, u32)) -> Rect { + Rect { + x: width / 4, + y: height / 4, + w: (width / 2).max(1), + h: (height / 2).max(1), + } +} diff --git a/crates/j2k-ml/benches/support/fixture.rs b/crates/j2k-ml/benches/support/fixture.rs new file mode 100644 index 00000000..e4f45636 --- /dev/null +++ b/crates/j2k-ml/benches/support/fixture.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::{ + encode_j2k_lossless, wrap_j2k_codestream, J2kBlockCodingMode, J2kChannelAssociation, + J2kChannelDefinition, J2kChannelType, J2kEncodeValidation, J2kFileBoxMetadata, + J2kFileColorSpec, J2kFileWrapOptions, J2kLosslessEncodeOptions, J2kLosslessSamples, +}; +use j2k_core::Colorspace; + +pub(super) fn encode_ht_fixture( + width: u32, + height: u32, + components: u16, + precision: u8, + signed: bool, + variant: usize, +) -> Vec { + let sample_count = usize::try_from(width).expect("benchmark width fits usize") + * usize::try_from(height).expect("benchmark height fits usize") + * usize::from(components); + let mask = if precision == 16 { + u16::MAX + } else { + (1_u16 << precision) - 1 + }; + let bytes_per_sample = if precision <= 8 { 1 } else { 2 }; + let mut bytes = Vec::with_capacity(sample_count * bytes_per_sample); + let variant = u32::try_from(variant).expect("benchmark fixture variant fits u32"); + for index in 0..sample_count { + let value = u16::try_from( + u32::try_from(index) + .expect("benchmark sample index fits u32") + .wrapping_mul(2_653) + .wrapping_add(variant.wrapping_mul(4_051)) + .wrapping_add(17) + & u32::from(u16::MAX), + ) + .expect("masked benchmark value fits u16") + & mask; + if signed { + let sign_bit = 1_u16 << (precision - 1); + let sign_extended = if value & sign_bit == 0 { + value + } else { + value | !mask + }; + if precision <= 8 { + bytes.push(sign_extended.to_le_bytes()[0]); + } else { + bytes.extend_from_slice(&sign_extended.to_le_bytes()); + } + } else if precision <= 8 { + bytes.push(u8::try_from(value).expect("8-bit benchmark sample")); + } else { + bytes.extend_from_slice(&value.to_le_bytes()); + } + } + let samples = J2kLosslessSamples::new(&bytes, width, height, components, precision, signed) + .expect("valid HT benchmark samples"); + let codestream = encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External), + ) + .expect("encode HT benchmark fixture") + .codestream; + if components == 4 { + wrap_benchmark_rgba_jph(&codestream) + } else { + codestream + } +} + +fn wrap_benchmark_rgba_jph(codestream: &[u8]) -> Vec { + let channel_definitions = [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 3, + channel_type: J2kChannelType::Opacity, + association: J2kChannelAssociation::WholeImage, + }, + ]; + wrap_j2k_codestream( + codestream, + J2kFileWrapOptions::jph() + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &channel_definitions, + }), + ) + .expect("wrap benchmark RGBA HTJ2K as JPH with straight alpha") +} diff --git a/crates/j2k-ml/benches/support/input_selection.rs b/crates/j2k-ml/benches/support/input_selection.rs new file mode 100644 index 00000000..a0f0b27a --- /dev/null +++ b/crates/j2k-ml/benches/support/input_selection.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +const INPUT_MODE_ENV: &str = "J2K_ML_BATCH_INPUT_MODE"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum InputMode { + Distinct, + Repeated, +} + +impl InputMode { + pub(crate) fn from_env() -> Result { + match std::env::var(INPUT_MODE_ENV) { + Ok(value) => Self::parse(Some(&value)), + Err(std::env::VarError::NotPresent) => Self::parse(None), + Err(std::env::VarError::NotUnicode(_)) => { + Err(format!("{INPUT_MODE_ENV} must be valid UTF-8")) + } + } + } + + pub(crate) fn parse(value: Option<&str>) -> Result { + match value { + None | Some("distinct") => Ok(Self::Distinct), + Some("repeated") => Ok(Self::Repeated), + Some(value) => Err(format!( + "unsupported {INPUT_MODE_ENV}={value:?}; expected distinct or repeated" + )), + } + } + + pub(crate) const fn label(self) -> &'static str { + match self { + Self::Distinct => "distinct", + Self::Repeated => "repeated", + } + } +} diff --git a/crates/j2k-ml/benches/support/metal_process_policy.rs b/crates/j2k-ml/benches/support/metal_process_policy.rs new file mode 100644 index 00000000..4b143474 --- /dev/null +++ b/crates/j2k-ml/benches/support/metal_process_policy.rs @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::process_policy::PROCESS_MODE_ENV; + +pub(crate) fn ensure_metal_criterion_instrumentation_disabled() -> Result<(), String> { + const INSTRUMENTATION_FLAGS: &[&str] = &[ + "J2K_PROFILE_STAGES", + "J2K_METAL_PROFILE_STAGES", + "J2K_METAL_PROFILE_SIGNPOSTS", + "J2K_METAL_PROFILE_DECODE_SPLIT_COMMANDS", + "J2K_METAL_PROFILE_COEFFICIENT_PREP_SPLIT_COMMANDS", + "J2K_METAL_PROFILE_CLASSIC_TIER1_DENSITY", + "J2K_METAL_PROFILE_CLASSIC_TIER1_RAW_PACK", + "J2K_METAL_PROFILE_CLASSIC_TIER1_ARITHMETIC_PACK", + "J2K_METAL_PROFILE_CLASSIC_TIER1_SYMBOL_PLAN", + "J2K_METAL_PROFILE_CLASSIC_TIER1_PASS_PLAN", + "J2K_METAL_PROFILE_CLASSIC_TIER1_TOKEN_EMIT", + "J2K_METAL_PROFILE_CLASSIC_TIER1_SPLIT_TOKEN_EMIT", + "J2K_METAL_PROFILE_CLASSIC_TIER1_TOKEN_PACK", + "MTL_CAPTURE_ENABLED", + ]; + let enabled = INSTRUMENTATION_FLAGS + .iter() + .copied() + .filter(|name| { + std::env::var(name) + .ok() + .is_some_and(|value| instrumentation_value_enabled(&value)) + }) + .collect::>(); + if enabled.is_empty() { + Ok(()) + } else { + Err(format!( + "Criterion acceptance measurements require profiling, signposts, and capture to be disabled; unset {} or run {PROCESS_MODE_ENV}=profile", + enabled.join(", ") + )) + } +} + +fn instrumentation_value_enabled(value: &str) -> bool { + !matches!( + value.trim().to_ascii_lowercase().as_str(), + "" | "0" | "false" | "off" | "no" + ) +} diff --git a/crates/j2k-ml/benches/support/mod.rs b/crates/j2k-ml/benches/support/mod.rs new file mode 100644 index 00000000..11ad2a5d --- /dev/null +++ b/crates/j2k-ml/benches/support/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +pub(super) mod decode_case; +mod fixture; +pub(super) mod input_selection; +pub(super) mod workload; + +pub(super) const LOW_BATCH_SIZES: &[usize] = &[1, 8]; +pub(super) const BATCH_SIZES: &[usize] = &[LOW_BATCH_SIZES[0], LOW_BATCH_SIZES[1], 32, 64]; +pub(super) const GENERATED_BATCH_SIZE: usize = 64; diff --git a/crates/j2k-ml/benches/support/process_policy.rs b/crates/j2k-ml/benches/support/process_policy.rs new file mode 100644 index 00000000..adbb2bd4 --- /dev/null +++ b/crates/j2k-ml/benches/support/process_policy.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +pub(crate) const PROCESS_MODE_ENV: &str = "J2K_ML_BATCH_PROCESS_MODE"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ProcessMode { + Criterion, + Profile, +} + +impl ProcessMode { + pub(crate) fn from_env() -> Result { + match std::env::var(PROCESS_MODE_ENV) { + Ok(value) => match value.as_str() { + "criterion" => Ok(Self::Criterion), + "profile" => Ok(Self::Profile), + _ => Err(format!( + "unsupported {PROCESS_MODE_ENV}={value:?}; expected criterion or profile" + )), + }, + Err(std::env::VarError::NotPresent) => Ok(Self::Criterion), + Err(std::env::VarError::NotUnicode(_)) => { + Err(format!("{PROCESS_MODE_ENV} must be valid UTF-8")) + } + } + } +} diff --git a/crates/j2k-ml/benches/support/workload.rs b/crates/j2k-ml/benches/support/workload.rs new file mode 100644 index 00000000..a3c85867 --- /dev/null +++ b/crates/j2k-ml/benches/support/workload.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{DecodeRequest, EncodedImage}; + +use super::{fixture::encode_ht_fixture, input_selection::InputMode, GENERATED_BATCH_SIZE}; + +#[derive(Clone, Copy)] +pub(crate) struct WorkloadSpec { + pub(crate) name: &'static str, + pub(crate) dimensions: (u32, u32), + components: u16, + precision: u8, + signed: bool, +} + +impl WorkloadSpec { + pub(crate) const fn new( + name: &'static str, + width: u32, + height: u32, + components: u16, + precision: u8, + signed: bool, + ) -> Self { + Self { + name, + dimensions: (width, height), + components, + precision, + signed, + } + } +} + +pub(crate) struct Workload { + pub(crate) name: &'static str, + pub(crate) dimensions: (u32, u32), + pub(crate) input_mode: InputMode, + encoded: Vec>, +} + +impl Workload { + pub(crate) fn inputs(&self, request: DecodeRequest, count: usize) -> Vec { + assert!( + count <= GENERATED_BATCH_SIZE, + "benchmark batch size exceeds generated input set" + ); + match self.input_mode { + InputMode::Distinct => self + .encoded + .iter() + .take(count) + .map(|bytes| EncodedImage::new(Arc::clone(bytes), request)) + .collect(), + InputMode::Repeated => (0..count) + .map(|_| EncodedImage::new(Arc::clone(&self.encoded[0]), request)) + .collect(), + } + } +} + +pub(crate) fn workload_specs() -> Vec { + let mut workloads = accelerator_workload_specs(); + workloads.extend(color_workload_specs()); + workloads +} + +fn accelerator_workload_specs() -> Vec { + [ + ("gray12_512", 512, 512, 1, 12, false), + ("gray12_1024", 1024, 1024, 1, 12, false), + ("gray16_512", 512, 512, 1, 16, false), + ("gray16_1024", 1024, 1024, 1, 16, false), + ("gray_i12_512", 512, 512, 1, 12, true), + ("gray_i12_1024", 1024, 1024, 1, 12, true), + ("gray_i16_512", 512, 512, 1, 16, true), + ("gray_i16_1024", 1024, 1024, 1, 16, true), + ] + .into_iter() + .map(WorkloadSpec::from) + .collect() +} + +fn color_workload_specs() -> impl Iterator { + [ + ("rgb8_256", 256, 256, 3, 8, false), + ("rgb8_512", 512, 512, 3, 8, false), + ("rgb16_256", 256, 256, 3, 16, false), + ("rgb16_512", 512, 512, 3, 16, false), + ("rgba8_256", 256, 256, 4, 8, false), + ("rgba8_512", 512, 512, 4, 8, false), + ("rgba16_256", 256, 256, 4, 16, false), + ("rgba16_512", 512, 512, 4, 16, false), + ] + .into_iter() + .map(WorkloadSpec::from) +} + +impl From<(&'static str, u32, u32, u16, u8, bool)> for WorkloadSpec { + fn from( + (name, width, height, components, precision, signed): ( + &'static str, + u32, + u32, + u16, + u8, + bool, + ), + ) -> Self { + Self::new(name, width, height, components, precision, signed) + } +} + +pub(crate) fn materialize_workload(spec: WorkloadSpec, input_mode: InputMode) -> Workload { + let encoded_count = match input_mode { + InputMode::Distinct => GENERATED_BATCH_SIZE, + InputMode::Repeated => 1, + }; + let encoded = (0..encoded_count) + .map(|variant| { + Arc::from(encode_ht_fixture( + spec.dimensions.0, + spec.dimensions.1, + spec.components, + spec.precision, + spec.signed, + variant, + )) + }) + .collect(); + Workload { + name: spec.name, + dimensions: spec.dimensions, + input_mode, + encoded, + } +} diff --git a/crates/j2k-ml/benches/tensor_decode.rs b/crates/j2k-ml/benches/tensor_decode.rs deleted file mode 100644 index 2bc819bb..00000000 --- a/crates/j2k-ml/benches/tensor_decode.rs +++ /dev/null @@ -1,275 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use burn_core::tensor::backend::Backend; -#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))] -use burn_flex::{Flex as CpuBackend, FlexDevice as CpuDevice}; -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] -use burn_ndarray::NdArrayDevice::Cpu as CpuDevice; -use criterion::{BenchmarkId, Criterion, Throughput}; -use j2k::{DeviceDecodeRequest, Downscale, Rect}; -use j2k_ml::{cpu, TensorDecodeOptions, TensorInput}; -#[cfg(all(feature = "cuda", not(target_os = "macos")))] -use j2k_test_support::htj2k_gray8_large_fixture; -use j2k_test_support::{classic_j2k_gray8_fixture, openhtj2k_refinement_fixture}; - -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] -type CpuBackend = burn_ndarray::NdArray; - -const BATCH_SIZES: &[usize] = &[1, 8, 32]; -#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))] -const CPU_STAGED_BENCHMARK_GROUP: &str = "j2k_ml_decode_to_ready_tensor_cpu_staged_flex"; -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] -const CPU_STAGED_BENCHMARK_GROUP: &str = - "j2k_ml_decode_to_ready_tensor_cpu_staged_ndarray_arm_linux"; - -fn main() { - let mut criterion = Criterion::default().configure_from_args(); - bench_cpu_staged(&mut criterion); - #[cfg(all(feature = "cuda", not(target_os = "macos")))] - bench_cuda_routes(&mut criterion); - #[cfg(all(feature = "metal", target_os = "macos"))] - bench_metal_staged(&mut criterion); - criterion.final_summary(); -} - -fn bench_cpu_staged(criterion: &mut Criterion) { - let gray8 = classic_j2k_gray8_fixture(128, 128); - let gray16 = openhtj2k_refinement_fixture(); - let options = TensorDecodeOptions::default(); - let mut group = criterion.benchmark_group(CPU_STAGED_BENCHMARK_GROUP); - - for (label, encoded) in [("gray8", gray8.as_slice()), ("gray16", gray16)] { - let item_bytes = compact_item_bytes(encoded); - for &batch_size in BATCH_SIZES { - let inputs = repeated_inputs(encoded, batch_size, DeviceDecodeRequest::Full); - let transferred = batch_bytes(item_bytes, batch_size); - group.throughput(Throughput::Bytes(transferred)); - group.bench_with_input( - BenchmarkId::new(format!("{label}/compact_upload_bytes"), transferred), - &inputs, - |bencher, inputs| { - bencher.iter(|| { - let decoded = cpu::decode_float_batch::( - std::hint::black_box(inputs), - &options, - &CpuDevice, - ) - .expect("CPU-staged tensor decode"); - CpuBackend::sync(&CpuDevice).expect("sync CPU backend"); - std::hint::black_box(decoded.tensor) - }); - }, - ); - } - } - - let roi = Rect { - x: 32, - y: 32, - w: 64, - h: 64, - }; - let request = DeviceDecodeRequest::RegionScaled { - roi, - scale: Downscale::Half, - }; - let inputs = repeated_inputs(&gray8, 8, request); - let compact_bytes = 8_u64 * 32 * 32; - group.throughput(Throughput::Bytes(compact_bytes)); - group.bench_function("roi_tiles_8/compact_upload_bytes_8192", |bencher| { - bencher.iter(|| { - let decoded = cpu::decode_float_batch::( - std::hint::black_box(&inputs), - &options, - &CpuDevice, - ) - .expect("CPU-staged ROI batch"); - CpuBackend::sync(&CpuDevice).expect("sync CPU backend"); - std::hint::black_box(decoded.tensor) - }); - }); - group.finish(); -} - -#[cfg(all(feature = "cuda", not(target_os = "macos")))] -fn bench_cuda_routes(criterion: &mut Criterion) { - use burn_cuda::{Cuda, CudaDevice}; - use j2k_ml::cuda; - - type CudaBackend = Cuda; - - let gray8 = htj2k_gray8_large_fixture(128, 128); - let gray16 = openhtj2k_refinement_fixture(); - let options = TensorDecodeOptions::default(); - let device = CudaDevice::default(); - let mut group = criterion.benchmark_group("j2k_ml_decode_to_ready_tensor_cuda"); - - for (label, encoded) in [("gray8", gray8.as_slice()), ("gray16", gray16)] { - let item_bytes = compact_item_bytes(encoded); - for &batch_size in BATCH_SIZES { - let inputs = repeated_inputs(encoded, batch_size, DeviceDecodeRequest::Full); - let transferred = batch_bytes(item_bytes, batch_size); - group.throughput(Throughput::Bytes(transferred)); - group.bench_with_input( - BenchmarkId::new( - format!("{label}/staged_baseline_compact_upload_bytes"), - transferred, - ), - &inputs, - |bencher, inputs| { - bencher.iter(|| { - let decoded = cpu::decode_float_batch::( - std::hint::black_box(inputs), - &options, - &device, - ) - .expect("CUDA-staged baseline"); - CudaBackend::sync(&device).expect("sync CUDA"); - std::hint::black_box(decoded.tensor) - }); - }, - ); - group.bench_with_input( - BenchmarkId::new(format!("{label}/direct_transferred_bytes_0"), batch_size), - &inputs, - |bencher, inputs| { - bencher.iter(|| { - let decoded = cuda::decode_float_batch( - std::hint::black_box(inputs), - &options, - &device, - ) - .expect("CUDA-direct tensor decode"); - CudaBackend::sync(&device).expect("sync CUDA"); - std::hint::black_box(decoded.tensor) - }); - }, - ); - } - } - - let roi_request = DeviceDecodeRequest::RegionScaled { - roi: Rect { - x: 32, - y: 32, - w: 64, - h: 64, - }, - scale: Downscale::Half, - }; - let roi_inputs = repeated_inputs(&gray8, 8, roi_request); - let roi_bytes = 8_u64 * 32 * 32; - group.throughput(Throughput::Bytes(roi_bytes)); - group.bench_function( - "roi_tiles_8/staged_baseline_compact_upload_bytes_8192", - |bencher| { - bencher.iter(|| { - let decoded = - cpu::decode_float_batch::(&roi_inputs, &options, &device) - .expect("CUDA ROI baseline"); - CudaBackend::sync(&device).expect("sync CUDA"); - std::hint::black_box(decoded.tensor) - }); - }, - ); - group.bench_function("roi_tiles_8/direct_transferred_bytes_0", |bencher| { - bencher.iter(|| { - let decoded = cuda::decode_float_batch(&roi_inputs, &options, &device) - .expect("CUDA-direct ROI decode"); - CudaBackend::sync(&device).expect("sync CUDA"); - std::hint::black_box(decoded.tensor) - }); - }); - group.finish(); -} - -#[cfg(all(feature = "metal", target_os = "macos"))] -fn bench_metal_staged(criterion: &mut Criterion) { - use burn_wgpu::{Wgpu, WgpuDevice}; - use j2k_ml::metal; - - let gray8 = classic_j2k_gray8_fixture(128, 128); - let gray16 = openhtj2k_refinement_fixture(); - let options = TensorDecodeOptions::default(); - let device = WgpuDevice::DefaultDevice; - let mut group = criterion.benchmark_group("j2k_ml_decode_to_ready_tensor_metal_staged"); - for (label, encoded) in [("gray8", gray8.as_slice()), ("gray16", gray16)] { - let item_bytes = compact_item_bytes(encoded); - for &batch_size in BATCH_SIZES { - let inputs = repeated_inputs(encoded, batch_size, DeviceDecodeRequest::Full); - let transferred = batch_bytes(item_bytes, batch_size) - .checked_mul(2) - .expect("benchmark transfer byte count fits u64"); - group.throughput(Throughput::Bytes(transferred)); - group.bench_with_input( - BenchmarkId::new( - format!("{label}/packed_readback_plus_upload_bytes"), - transferred, - ), - &inputs, - |bencher, inputs| { - bencher.iter(|| { - let decoded = metal::decode_float_batch( - std::hint::black_box(inputs), - &options, - &device, - ) - .expect("Metal-staged tensor decode"); - as Backend>::sync(&device).expect("sync Metal"); - std::hint::black_box(decoded.tensor) - }); - }, - ); - } - } - - let roi_inputs = repeated_inputs( - &gray8, - 8, - DeviceDecodeRequest::RegionScaled { - roi: Rect { - x: 32, - y: 32, - w: 64, - h: 64, - }, - scale: Downscale::Half, - }, - ); - let roi_transferred = 8_u64 * 32 * 32 * 2; - group.throughput(Throughput::Bytes(roi_transferred)); - group.bench_function( - "roi_tiles_8/packed_readback_plus_upload_bytes_16384", - |bencher| { - bencher.iter(|| { - let decoded = metal::decode_float_batch(&roi_inputs, &options, &device) - .expect("Metal-staged ROI tensor decode"); - as Backend>::sync(&device).expect("sync Metal"); - std::hint::black_box(decoded.tensor) - }); - }, - ); - group.finish(); -} - -fn compact_item_bytes(encoded: &[u8]) -> u64 { - let info = j2k::J2kDecoder::inspect(encoded).expect("inspect benchmark fixture"); - let sample_bytes = if info.bit_depth <= 8 { 1 } else { 2 }; - u64::from(info.dimensions.0) * u64::from(info.dimensions.1) * sample_bytes -} - -fn batch_bytes(item_bytes: u64, batch_size: usize) -> u64 { - item_bytes - .checked_mul(u64::try_from(batch_size).expect("benchmark batch size fits u64")) - .expect("benchmark transfer byte count fits u64") -} - -fn repeated_inputs( - encoded: &[u8], - batch_size: usize, - request: DeviceDecodeRequest, -) -> Vec> { - (0..batch_size) - .map(|_| TensorInput { encoded, request }) - .collect() -} diff --git a/crates/j2k-ml/src/batch_contract.rs b/crates/j2k-ml/src/batch_contract.rs new file mode 100644 index 00000000..f7a33662 --- /dev/null +++ b/crates/j2k-ml/src/batch_contract.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use burn_core::tensor::DType; +use j2k::{BatchGroupInfo, BatchLayout, NativeSampleType}; + +use crate::BurnDecodeError; + +pub(crate) fn tensor_shape( + batch: usize, + info: &BatchGroupInfo, +) -> Result<[usize; 4], BurnDecodeError> { + let width = usize::try_from(info.dimensions.0).map_err(|_| BurnDecodeError::SizeOverflow)?; + let height = usize::try_from(info.dimensions.1).map_err(|_| BurnDecodeError::SizeOverflow)?; + let channels = info.color.channels(); + let samples = batch + .checked_mul(width) + .and_then(|value| value.checked_mul(height)) + .and_then(|value| value.checked_mul(channels)) + .ok_or(BurnDecodeError::SizeOverflow)?; + if samples == 0 { + return Err(BurnDecodeError::SizeOverflow); + } + Ok(match info.layout { + BatchLayout::Nchw => [batch, channels, height, width], + BatchLayout::Nhwc => [batch, height, width, channels], + _ => return Err(BurnDecodeError::UnsupportedCodecContract), + }) +} + +pub(crate) const fn dtype(sample_type: NativeSampleType) -> Result { + Ok(match sample_type { + NativeSampleType::U8 => DType::U8, + NativeSampleType::U16 => DType::U16, + NativeSampleType::I16 => DType::I16, + _ => return Err(BurnDecodeError::UnsupportedCodecContract), + }) +} diff --git a/crates/j2k-ml/src/completion.rs b/crates/j2k-ml/src/completion.rs new file mode 100644 index 00000000..ae6c4aee --- /dev/null +++ b/crates/j2k-ml/src/completion.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{BurnBatchGroupError, BurnDecodeError}; + +pub(crate) fn finish_submitted_groups( + groups: Vec, + mut group_errors: Vec, + mut finish: impl FnMut(G) -> (Vec, Result), +) -> Result<(Vec, Vec), BurnDecodeError> { + let mut outputs = Vec::new(); + outputs + .try_reserve_exact(groups.len()) + .map_err(|_| BurnDecodeError::SizeOverflow)?; + group_errors + .try_reserve_exact(groups.len()) + .map_err(|_| BurnDecodeError::SizeOverflow)?; + let mut fatal_error = None; + for group in groups { + let (source_indices, result) = finish(group); + match result { + Ok(output) => outputs.push(output), + Err(source) if burn_group_error_is_fatal(&source) => { + if fatal_error.is_none() { + fatal_error = Some(source); + } + } + Err(source) => group_errors.push(BurnBatchGroupError::new(source_indices, source)), + } + } + if let Some(error) = fatal_error { + return Err(error); + } + group_errors.sort_by_key(|error| { + error + .source_indices() + .first() + .copied() + .unwrap_or(usize::MAX) + }); + Ok((outputs, group_errors)) +} + +pub(crate) fn burn_group_error_is_fatal(error: &BurnDecodeError) -> bool { + match error { + BurnDecodeError::Infrastructure(_) | BurnDecodeError::AcceleratorInterop { .. } => true, + #[cfg(feature = "cuda")] + BurnDecodeError::Cuda(source) => source.session_is_unusable(), + #[cfg(feature = "metal")] + BurnDecodeError::Metal(source) => source.session_is_unusable(), + BurnDecodeError::UnsupportedDType { .. } + | BurnDecodeError::SampleTypeMismatch + | BurnDecodeError::SizeOverflow + | BurnDecodeError::UnsupportedCodecContract => false, + } +} + +#[cfg(test)] +mod tests { + use super::{burn_group_error_is_fatal, finish_submitted_groups}; + use crate::BurnDecodeError; + + #[test] + fn finishing_groups_retires_every_group_after_nonfatal_failure() { + let groups = vec![ + (0, Ok(10)), + (1, Err(BurnDecodeError::UnsupportedCodecContract)), + (2, Ok(30)), + ]; + let mut retired = Vec::new(); + + let (outputs, errors) = finish_submitted_groups(groups, Vec::new(), |(index, result)| { + retired.push(index); + (vec![index], result) + }) + .expect("a group-local failure must preserve other completed groups"); + + assert_eq!(retired, [0, 1, 2]); + assert_eq!(outputs, [10, 30]); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].source_indices(), [1]); + assert!(matches!( + errors[0].source(), + BurnDecodeError::UnsupportedCodecContract + )); + } + + #[test] + fn finishing_groups_retires_every_group_before_returning_fatal_failure() { + let groups = vec![ + ( + 0, + Err(BurnDecodeError::AcceleratorInterop { + backend: "test", + message: "session ordering failed".to_string(), + }), + ), + (1, Ok(10)), + ]; + let mut retired = Vec::new(); + + let result = finish_submitted_groups(groups, Vec::new(), |(index, result)| { + retired.push(index); + (vec![index], result) + }); + + assert_eq!(retired, [0, 1]); + assert!(matches!( + result, + Err(BurnDecodeError::AcceleratorInterop { + backend: "test", + .. + }) + )); + } + + #[cfg(feature = "metal")] + #[test] + fn metal_group_fatality_delegates_to_codec_error_classification() { + assert!(burn_group_error_is_fatal(&BurnDecodeError::Metal( + j2k_metal::Error::MetalRuntime { + message: "test runtime failure".to_string(), + } + ))); + assert!(!burn_group_error_is_fatal(&BurnDecodeError::Metal( + j2k_metal::Error::MetalKernel { + message: "test group status".to_string(), + } + ))); + } + + #[cfg(feature = "cuda")] + #[test] + fn cuda_group_fatality_delegates_to_codec_error_classification() { + let recoverable = j2k_cuda::CudaBatchError::GroupExecution { + source_indices: vec![2], + source: Box::new(j2k_cuda::Error::HtJobChunkPlan( + j2k_core::HtGpuJobChunkPlanError::InvalidCodingPassCount { + source_index: 2, + original_job_index: 4, + coding_passes: 0, + }, + )), + }; + assert!(!burn_group_error_is_fatal(&BurnDecodeError::Cuda( + recoverable + ))); + assert!(burn_group_error_is_fatal(&BurnDecodeError::Cuda( + j2k_cuda::CudaBatchError::Infrastructure( + j2k_core::BatchInfrastructureError::EmptyBatchPlan, + ) + ))); + } +} diff --git a/crates/j2k-ml/src/cpu.rs b/crates/j2k-ml/src/cpu.rs index 78e782ff..3654ec69 100644 --- a/crates/j2k-ml/src/cpu.rs +++ b/crates/j2k-ml/src/cpu.rs @@ -1,357 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -//! Portable decode into any Burn backend. +//! Portable materialization of native codec batch groups into Burn tensors. -use burn_core::tensor::{backend::Backend, DType, FloatDType, Int, Tensor}; -use j2k::{ - DecodeOutcome, DeviceDecodePlan, DeviceDecodeRequest, J2kDecodeWarning, J2kDecoder, - J2kScratchPool, PixelFormat, -}; +mod batch; -use crate::{ - ChannelSelection, TensorBatchDecode, TensorDecode, TensorDecodeError, TensorDecodeOptions, - TensorInput, TensorRoute, -}; - -mod materialization; - -use materialization::{integer_tensor_3, integer_tensor_4}; -#[cfg(feature = "metal")] -pub(crate) use materialization::{integer_tensor_3_from_bytes, integer_tensor_4_from_bytes}; -pub(crate) use materialization::{ - normalize_3, normalize_4, validate_normalization_channels, validate_normalization_values, - SampleWidth, -}; - -#[derive(Debug)] -struct PackedImage { - bytes: Vec, - shape: [usize; 3], - outcome: DecodeOutcome, -} - -/// Decode one image into a rank-3 U8 tensor. -pub fn decode_u8( - input: TensorInput<'_>, - options: &TensorDecodeOptions, - device: &B::Device, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U8)?; - let packed = decode_packed(input, options.channels, SampleWidth::U8)?; - let tensor = integer_tensor_3::(&packed, options.layout, device, DType::U8); - Ok(single_result(tensor, packed.outcome)) -} - -/// Decode one image into a rank-3 U16 tensor. -pub fn decode_u16( - input: TensorInput<'_>, - options: &TensorDecodeOptions, - device: &B::Device, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U16)?; - let packed = decode_packed(input, options.channels, SampleWidth::U16)?; - let tensor = integer_tensor_3::(&packed, options.layout, device, DType::U16); - Ok(single_result(tensor, packed.outcome)) -} - -/// Decode one image into a rank-3 F32 tensor. -pub fn decode_float( - input: TensorInput<'_>, - options: &TensorDecodeOptions, - device: &B::Device, -) -> Result>, TensorDecodeError> { - validate_normalization_values(&options.normalization)?; - ensure_dtype::(device, DType::F32)?; - - let info = J2kDecoder::inspect(input.encoded)?; - let width = if info.bit_depth <= 8 { - SampleWidth::U8 - } else { - SampleWidth::U16 - }; - ensure_dtype::(device, width.dtype())?; - validate_normalization_channels( - &options.normalization, - selected_channels(options.channels, info.components), - )?; - let packed = decode_packed(input, options.channels, width)?; - let tensor = - integer_tensor_3::(&packed, options.layout, device, width.dtype()).cast(FloatDType::F32); - let tensor = normalize_3( - tensor, - &options.normalization, - options.layout, - packed.shape[2], - width, - device, - ); - Ok(single_result(tensor, packed.outcome)) -} - -/// Decode a batch into a rank-4 U8 tensor. -pub fn decode_u8_batch( - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - device: &B::Device, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U8)?; - let packed = decode_batch(inputs, options.channels, SampleWidth::U8)?; - let tensor = integer_tensor_4::(&packed, options.layout, device, DType::U8); - Ok(batch_result(tensor, packed)) -} - -/// Decode a batch into a rank-4 U16 tensor. -pub fn decode_u16_batch( - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - device: &B::Device, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U16)?; - let packed = decode_batch(inputs, options.channels, SampleWidth::U16)?; - let tensor = integer_tensor_4::(&packed, options.layout, device, DType::U16); - Ok(batch_result(tensor, packed)) -} - -/// Decode a batch into a rank-4 F32 tensor. -pub fn decode_float_batch( - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - device: &B::Device, -) -> Result>, TensorDecodeError> { - validate_normalization_values(&options.normalization)?; - ensure_dtype::(device, DType::F32)?; - let first = inputs.first().ok_or(TensorDecodeError::EmptyBatch)?; - let first_info = - J2kDecoder::inspect(first.encoded).map_err(|source| indexed(0, source.into()))?; - let bit_depth = first_info.bit_depth; - let width = if bit_depth <= 8 { - SampleWidth::U8 - } else { - SampleWidth::U16 - }; - ensure_dtype::(device, width.dtype())?; - validate_normalization_channels( - &options.normalization, - selected_channels(options.channels, first_info.components), - ) - .map_err(|error| indexed(0, error))?; - - for (index, input) in inputs.iter().enumerate().skip(1) { - let item_info = - J2kDecoder::inspect(input.encoded).map_err(|source| indexed(index, source.into()))?; - let item_depth = item_info.bit_depth; - if (item_depth <= 8) != (bit_depth <= 8) { - return Err(TensorDecodeError::BatchItem { - index, - source: Box::new(TensorDecodeError::StrictRoute { - route: TensorRoute::CpuStaged, - message: format!( - "mixed canonical integer widths are unsupported: first item is {bit_depth}-bit, item is {item_depth}-bit" - ), - }), - }); - } - validate_normalization_channels( - &options.normalization, - selected_channels(options.channels, item_info.components), - ) - .map_err(|error| indexed(index, error))?; - } - - let packed = decode_batch(inputs, options.channels, width)?; - let tensor = - integer_tensor_4::(&packed, options.layout, device, width.dtype()).cast(FloatDType::F32); - let tensor = normalize_4( - tensor, - &options.normalization, - options.layout, - packed.shape[2], - width, - device, - ); - Ok(batch_result(tensor, packed)) -} - -pub(crate) fn ensure_dtype( - device: &B::Device, - dtype: DType, -) -> Result<(), TensorDecodeError> { - if B::supports_dtype(device, dtype) { - Ok(()) - } else { - Err(TensorDecodeError::UnsupportedDType { dtype }) - } -} - -fn decode_packed( - input: TensorInput<'_>, - selection: ChannelSelection, - width: SampleWidth, -) -> Result { - let mut decoder = J2kDecoder::new(input.encoded)?; - let plan = DeviceDecodePlan::for_image(decoder.info().dimensions, input.request)?; - let channels = selected_channels(selection, decoder.info().components); - let format = pixel_format(channels, width); - let (output_width, output_height) = plan.output_dims(); - let width_usize = usize::try_from(output_width).map_err(|_| TensorDecodeError::SizeOverflow)?; - let height_usize = - usize::try_from(output_height).map_err(|_| TensorDecodeError::SizeOverflow)?; - let stride = width_usize - .checked_mul(format.bytes_per_pixel()) - .ok_or(TensorDecodeError::SizeOverflow)?; - let byte_len = stride - .checked_mul(height_usize) - .ok_or(TensorDecodeError::SizeOverflow)?; - let mut bytes = Vec::new(); - bytes - .try_reserve_exact(byte_len) - .map_err(|_| TensorDecodeError::SizeOverflow)?; - bytes.resize(byte_len, 0); - let mut pool = J2kScratchPool::new(); - let outcome = match input.request { - DeviceDecodeRequest::Full => decoder.decode_into(&mut bytes, stride, format)?, - DeviceDecodeRequest::Region { roi } => { - decoder.decode_region_into(&mut pool, &mut bytes, stride, format, roi)? - } - DeviceDecodeRequest::Scaled { scale } => { - decoder.decode_scaled_into(&mut pool, &mut bytes, stride, format, scale)? - } - DeviceDecodeRequest::RegionScaled { roi, scale } => { - decoder.decode_region_scaled_into(&mut pool, &mut bytes, stride, format, roi, scale)? - } - }; - Ok(PackedImage { - bytes, - shape: [height_usize, width_usize, channels], - outcome, - }) -} - -pub(crate) fn selected_channels(selection: ChannelSelection, components: u16) -> usize { - match selection { - ChannelSelection::Auto if components == 1 => 1, - ChannelSelection::Auto | ChannelSelection::Rgb => 3, - ChannelSelection::Gray => 1, - ChannelSelection::Rgba => 4, - } -} - -pub(crate) fn planned_shape( - input: TensorInput<'_>, - selection: ChannelSelection, -) -> Result<[usize; 3], TensorDecodeError> { - let info = J2kDecoder::inspect(input.encoded)?; - let plan = DeviceDecodePlan::for_image(info.dimensions, input.request)?; - let (width, height) = plan.output_dims(); - Ok([ - usize::try_from(height).map_err(|_| TensorDecodeError::SizeOverflow)?, - usize::try_from(width).map_err(|_| TensorDecodeError::SizeOverflow)?, - selected_channels(selection, info.components), - ]) -} - -pub(crate) fn pixel_format(channels: usize, width: SampleWidth) -> PixelFormat { - match (channels, width) { - (1, SampleWidth::U8) => PixelFormat::Gray8, - (3, SampleWidth::U8) => PixelFormat::Rgb8, - (4, SampleWidth::U8) => PixelFormat::Rgba8, - (1, SampleWidth::U16) => PixelFormat::Gray16, - (3, SampleWidth::U16) => PixelFormat::Rgb16, - (4, SampleWidth::U16) => PixelFormat::Rgba16, - _ => unreachable!("channel selection is confined to 1, 3, or 4"), - } -} - -#[derive(Debug)] -struct PackedBatch { - bytes: Vec, - shape: [usize; 3], - outcomes: Vec>, -} - -fn decode_batch( - inputs: &[TensorInput<'_>], - channels: ChannelSelection, - width: SampleWidth, -) -> Result { - let first = inputs.first().ok_or(TensorDecodeError::EmptyBatch)?; - let expected_shape = planned_shape(*first, channels).map_err(|error| indexed(0, error))?; - for (index, input) in inputs.iter().enumerate().skip(1) { - let actual = planned_shape(*input, channels).map_err(|error| indexed(index, error))?; - if actual != expected_shape { - return Err(TensorDecodeError::BatchShapeMismatch { - index, - expected: expected_shape, - actual, - }); - } - } - let first = decode_packed(*first, channels, width).map_err(|error| indexed(0, error))?; - let shape = first.shape; - let item_bytes = shape - .iter() - .try_fold(width.bytes(), |size, dim| size.checked_mul(*dim)) - .ok_or(TensorDecodeError::SizeOverflow)?; - let capacity = item_bytes - .checked_mul(inputs.len()) - .ok_or(TensorDecodeError::SizeOverflow)?; - let mut bytes = Vec::new(); - bytes - .try_reserve_exact(capacity) - .map_err(|_| TensorDecodeError::SizeOverflow)?; - bytes.extend_from_slice(&first.bytes); - let mut outcomes = Vec::new(); - outcomes - .try_reserve_exact(inputs.len()) - .map_err(|_| TensorDecodeError::SizeOverflow)?; - outcomes.push(first.outcome); - - for (index, input) in inputs.iter().enumerate().skip(1) { - let image = - decode_packed(*input, channels, width).map_err(|error| indexed(index, error))?; - if image.shape != shape { - return Err(TensorDecodeError::BatchShapeMismatch { - index, - expected: shape, - actual: image.shape, - }); - } - bytes.extend_from_slice(&image.bytes); - outcomes.push(image.outcome); - } - Ok(PackedBatch { - bytes, - shape, - outcomes, - }) -} - -fn indexed(index: usize, source: TensorDecodeError) -> TensorDecodeError { - TensorDecodeError::BatchItem { - index, - source: Box::new(source), - } -} - -fn single_result(tensor: T, outcome: DecodeOutcome) -> TensorDecode { - TensorDecode { - tensor, - decoded: outcome.decoded, - warnings: outcome.warnings, - route: TensorRoute::CpuStaged, - } -} - -fn batch_result(tensor: T, packed: PackedBatch) -> TensorBatchDecode { - let (decoded, warnings) = packed - .outcomes - .into_iter() - .map(|outcome| (outcome.decoded, outcome.warnings)) - .unzip(); - TensorBatchDecode { - tensor, - decoded, - warnings, - route: TensorRoute::CpuStaged, - } -} +pub use batch::CpuBurnDecoder; diff --git a/crates/j2k-ml/src/cpu/batch.rs b/crates/j2k-ml/src/cpu/batch.rs new file mode 100644 index 00000000..e445a6a5 --- /dev/null +++ b/crates/j2k-ml/src/cpu/batch.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::marker::PhantomData; + +use burn_core::tensor::{backend::Backend, DType, Tensor, TensorData}; +use j2k::{ + BatchDecodeOptions, CpuBatchDecodeResult, CpuBatchSamples, EncodedImage, NativeSampleType, + PreparedBatch, PreparedImage, +}; + +use crate::batch_contract::{dtype, tensor_shape}; +use crate::{BurnBatchDecode, BurnBatchGroup, BurnBatchTensor, BurnDecodeError}; + +/// Persistent CPU codec session followed by one Burn materialization per output group. +#[derive(Debug)] +pub struct CpuBurnDecoder { + codec: j2k::CpuBatchDecoder, + device: B::Device, + backend: PhantomData, +} + +impl CpuBurnDecoder { + /// Construct a CPU codec session for one Burn device. + #[must_use] + pub fn new(device: B::Device, options: BatchDecodeOptions) -> Self { + Self { + codec: j2k::CpuBatchDecoder::new(options), + device, + backend: PhantomData, + } + } + + /// Borrow the retained codec session. + #[must_use] + pub const fn codec(&self) -> &j2k::CpuBatchDecoder { + &self.codec + } + + /// Borrow the Burn device receiving decoded tensors. + #[must_use] + pub const fn device(&self) -> &B::Device { + &self.device + } + + /// Parse and group owned inputs once for repeated training epochs. + pub fn prepare(&self, inputs: Vec) -> Result { + Ok(self.codec.prepare(inputs)?) + } + + /// Regroup caller-supplied prepared images without reparsing encoded bytes. + /// + /// Source indices in the returned batch are positions in `images`; each + /// [`PreparedImage::source_index`] remains its original preparation index. + pub fn prepare_prepared_images( + &self, + images: Vec, + ) -> Result { + Ok(self.codec.prepare_prepared_images(images)?) + } + + /// Prepare, decode, and materialize one owned batch. + pub fn decode( + &mut self, + inputs: Vec, + ) -> Result, BurnDecodeError> { + let prepared = self.prepare(inputs)?; + self.decode_prepared(&prepared) + } + + /// Regroup, decode, and materialize caller-supplied prepared images. + pub fn decode_prepared_images( + &mut self, + images: Vec, + ) -> Result, BurnDecodeError> { + let prepared = self.prepare_prepared_images(images)?; + self.decode_prepared(&prepared) + } + + /// Decode a reusable prepared batch into ordinary rank-4 Burn integer tensors. + pub fn decode_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result, BurnDecodeError> { + self.ensure_prepared_dtypes(prepared)?; + let decoded = self.codec.decode_prepared(prepared)?; + materialize::(decoded, &self.device) + } + + fn ensure_prepared_dtypes(&self, prepared: &PreparedBatch) -> Result<(), BurnDecodeError> { + for group in prepared.groups() { + let dtype = dtype(group.info().sample_type)?; + if !B::supports_dtype(&self.device, dtype) { + return Err(BurnDecodeError::UnsupportedDType { dtype }); + } + } + Ok(()) + } +} + +fn materialize( + decoded: CpuBatchDecodeResult, + device: &B::Device, +) -> Result, BurnDecodeError> { + let (codec_groups, errors) = decoded.into_parts(); + let mut groups = Vec::new(); + groups + .try_reserve_exact(codec_groups.len()) + .map_err(|_| BurnDecodeError::SizeOverflow)?; + + for group in codec_groups { + let (info, source_indices, decoded_rects, warnings, samples) = group.into_parts(); + let shape = tensor_shape(source_indices.len(), &info)?; + let tensor = match (info.sample_type, samples) { + (NativeSampleType::U8, CpuBatchSamples::U8(samples)) => BurnBatchTensor::U8( + Tensor::from_data(TensorData::new(samples, shape), (device, DType::U8)), + ), + (NativeSampleType::U16, CpuBatchSamples::U16(samples)) => BurnBatchTensor::U16( + Tensor::from_data(TensorData::new(samples, shape), (device, DType::U16)), + ), + (NativeSampleType::I16, CpuBatchSamples::I16(samples)) => BurnBatchTensor::I16( + Tensor::from_data(TensorData::new(samples, shape), (device, DType::I16)), + ), + _ => return Err(BurnDecodeError::SampleTypeMismatch), + }; + groups.push(BurnBatchGroup { + tensor, + info, + source_indices, + decoded_rects, + warnings, + }); + } + + Ok(BurnBatchDecode { + groups, + errors, + group_errors: Vec::new(), + }) +} diff --git a/crates/j2k-ml/src/cpu/materialization.rs b/crates/j2k-ml/src/cpu/materialization.rs deleted file mode 100644 index ed6ed883..00000000 --- a/crates/j2k-ml/src/cpu/materialization.rs +++ /dev/null @@ -1,222 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use burn_core::tensor::{backend::Backend, DType, Int, Tensor, TensorData}; - -use super::{PackedBatch, PackedImage}; -use crate::{FloatNormalization, TensorDecodeError, TensorLayout}; - -#[derive(Debug, Clone, Copy)] -pub(crate) enum SampleWidth { - U8, - U16, -} - -impl SampleWidth { - pub(crate) const fn bytes(self) -> usize { - match self { - Self::U8 => 1, - Self::U16 => 2, - } - } - - pub(crate) const fn dtype(self) -> DType { - match self { - Self::U8 => DType::U8, - Self::U16 => DType::U16, - } - } - - const fn unit_denominator(self) -> f32 { - match self { - Self::U8 => 255.0, - Self::U16 => 65_535.0, - } - } -} - -pub(super) fn integer_tensor_3( - packed: &PackedImage, - layout: TensorLayout, - device: &B::Device, - dtype: DType, -) -> Tensor { - integer_tensor_3_from_bytes(packed.bytes.clone(), packed.shape, layout, device, dtype) -} - -pub(crate) fn integer_tensor_3_from_bytes( - bytes: Vec, - shape: [usize; 3], - layout: TensorLayout, - device: &B::Device, - dtype: DType, -) -> Tensor { - let tensor = match dtype { - DType::U8 => Tensor::from_data(TensorData::new(bytes, shape), (device, DType::U8)), - DType::U16 => Tensor::from_data( - TensorData::new(bytes_to_u16(&bytes), shape), - (device, DType::U16), - ), - _ => unreachable!("portable integer decode only uses U8 or U16"), - }; - match layout { - TensorLayout::ChannelsFirst => tensor.permute([2, 0, 1]), - TensorLayout::ChannelsLast => tensor, - } -} - -pub(super) fn integer_tensor_4( - packed: &PackedBatch, - layout: TensorLayout, - device: &B::Device, - dtype: DType, -) -> Tensor { - integer_tensor_4_from_bytes( - packed.bytes.clone(), - packed.outcomes.len(), - packed.shape, - layout, - device, - dtype, - ) -} - -pub(crate) fn integer_tensor_4_from_bytes( - bytes: Vec, - batch: usize, - item_shape: [usize; 3], - layout: TensorLayout, - device: &B::Device, - dtype: DType, -) -> Tensor { - let shape = [batch, item_shape[0], item_shape[1], item_shape[2]]; - let tensor = match dtype { - DType::U8 => Tensor::from_data(TensorData::new(bytes, shape), (device, DType::U8)), - DType::U16 => Tensor::from_data( - TensorData::new(bytes_to_u16(&bytes), shape), - (device, DType::U16), - ), - _ => unreachable!("portable integer decode only uses U8 or U16"), - }; - match layout { - TensorLayout::ChannelsFirst => tensor.permute([0, 3, 1, 2]), - TensorLayout::ChannelsLast => tensor, - } -} - -fn bytes_to_u16(bytes: &[u8]) -> Vec { - bytes - .chunks_exact(2) - .map(|sample| u16::from_ne_bytes([sample[0], sample[1]])) - .collect() -} - -pub(crate) fn validate_normalization_values( - normalization: &FloatNormalization, -) -> Result<(), TensorDecodeError> { - let FloatNormalization::MeanStd { mean, std } = normalization else { - return Ok(()); - }; - if mean.iter().chain(std).any(|value| !value.is_finite()) { - return Err(invalid_normalization("mean and std values must be finite")); - } - if std.contains(&0.0) { - return Err(invalid_normalization("standard deviations must be nonzero")); - } - Ok(()) -} - -pub(crate) fn validate_normalization_channels( - normalization: &FloatNormalization, - channels: usize, -) -> Result<(), TensorDecodeError> { - let FloatNormalization::MeanStd { mean, std } = normalization else { - return Ok(()); - }; - if mean.len() != channels || std.len() != channels { - return Err(invalid_normalization(format!( - "mean and std must each contain {channels} values; got {} and {}", - mean.len(), - std.len() - ))); - } - Ok(()) -} - -fn invalid_normalization(message: impl Into) -> TensorDecodeError { - TensorDecodeError::InvalidNormalization { - message: message.into(), - } -} - -pub(crate) fn normalize_3( - tensor: Tensor, - normalization: &FloatNormalization, - layout: TensorLayout, - channels: usize, - width: SampleWidth, - device: &B::Device, -) -> Tensor { - match normalization { - FloatNormalization::Raw => tensor, - FloatNormalization::Unit => tensor.div_scalar(width.unit_denominator()), - FloatNormalization::MeanStd { mean, std } => { - let unit = tensor.div_scalar(width.unit_denominator()); - let (mean, std) = normalization_tensors(mean, std, channels, layout, device); - unit.sub(mean.reshape(shape_3(channels, layout))) - .div(std.reshape(shape_3(channels, layout))) - } - } -} - -pub(crate) fn normalize_4( - tensor: Tensor, - normalization: &FloatNormalization, - layout: TensorLayout, - channels: usize, - width: SampleWidth, - device: &B::Device, -) -> Tensor { - match normalization { - FloatNormalization::Raw => tensor, - FloatNormalization::Unit => tensor.div_scalar(width.unit_denominator()), - FloatNormalization::MeanStd { mean, std } => { - let unit = tensor.div_scalar(width.unit_denominator()); - let (mean, std) = normalization_tensors(mean, std, channels, layout, device); - unit.sub(mean.reshape(shape_4(channels, layout))) - .div(std.reshape(shape_4(channels, layout))) - } - } -} - -fn normalization_tensors( - mean: &[f32], - std: &[f32], - channels: usize, - _layout: TensorLayout, - device: &B::Device, -) -> (Tensor, Tensor) { - ( - Tensor::from_data( - TensorData::new(mean.to_vec(), [channels]), - (device, DType::F32), - ), - Tensor::from_data( - TensorData::new(std.to_vec(), [channels]), - (device, DType::F32), - ), - ) -} - -const fn shape_3(channels: usize, layout: TensorLayout) -> [usize; 3] { - match layout { - TensorLayout::ChannelsFirst => [channels, 1, 1], - TensorLayout::ChannelsLast => [1, 1, channels], - } -} - -const fn shape_4(channels: usize, layout: TensorLayout) -> [usize; 4] { - match layout { - TensorLayout::ChannelsFirst => [1, channels, 1, 1], - TensorLayout::ChannelsLast => [1, 1, 1, channels], - } -} diff --git a/crates/j2k-ml/src/cuda.rs b/crates/j2k-ml/src/cuda.rs index 49bd2f4e..47a17ea9 100644 --- a/crates/j2k-ml/src/cuda.rs +++ b/crates/j2k-ml/src/cuda.rs @@ -1,371 +1,8 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -//! Strict decode directly into Burn's default fused CUDA backend. +//! CUDA codec output written into Burn-owned allocations. -use burn_core::tensor::{DType, Int, Tensor}; -use burn_cuda::{Cuda, CudaDevice}; -use j2k::{DeviceDecodePlan, J2kDecodeWarning, Rect}; -use j2k_cuda::{CudaSession, J2kDecoder as CudaDecoder, Surface}; -use j2k_cuda_runtime::{CudaContext, CudaExternalDeviceBufferViewMut}; - -use crate::cpu::{ - ensure_dtype, pixel_format, selected_channels, validate_normalization_channels, - validate_normalization_values, SampleWidth, -}; -use crate::{ - TensorBatchDecode, TensorDecode, TensorDecodeError, TensorDecodeOptions, TensorInput, - TensorRoute, -}; - -mod config; +mod batch; mod interop; -use config::{kernel_config, tensor_shape_3, tensor_shape_4}; -use interop::{fill_float_tensor, fill_int_tensor}; - -type CudaTensor = Tensor; -type CudaIntTensor = Tensor; - -#[derive(Debug, Clone)] -struct PlannedImage<'a> { - input: TensorInput<'a>, - pub(super) shape: [usize; 3], - decoded: Rect, -} - -/// Decode one image into a rank-3 U8 tensor without decoded-pixel host transfer. -pub fn decode_u8( - input: TensorInput<'_>, - options: &TensorDecodeOptions, - device: &CudaDevice, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U8)?; - let plan = plan_image(input, options, SampleWidth::U8)?; - let shape = tensor_shape_3(plan.shape, options.layout); - let context = - CudaContext::retain_primary(device.index).map_err(|error| cuda_runtime_error(&error))?; - let tensor = fill_int_tensor::<3>(shape, DType::U8, device, &context, |destination| { - decode_plans_into( - std::slice::from_ref(&plan), - options, - SampleWidth::U8, - true, - false, - &context, - destination, - ) - })?; - Ok(single_result(tensor, &plan)) -} - -/// Decode one image into a rank-3 U16 tensor without decoded-pixel host transfer. -pub fn decode_u16( - input: TensorInput<'_>, - options: &TensorDecodeOptions, - device: &CudaDevice, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U16)?; - let plan = plan_image(input, options, SampleWidth::U16)?; - let shape = tensor_shape_3(plan.shape, options.layout); - let context = - CudaContext::retain_primary(device.index).map_err(|error| cuda_runtime_error(&error))?; - let tensor = fill_int_tensor::<3>(shape, DType::U16, device, &context, |destination| { - decode_plans_into( - std::slice::from_ref(&plan), - options, - SampleWidth::U16, - true, - false, - &context, - destination, - ) - })?; - Ok(single_result(tensor, &plan)) -} - -/// Decode one image into a rank-3 F32 tensor without decoded-pixel host transfer. -pub fn decode_float( - input: TensorInput<'_>, - options: &TensorDecodeOptions, - device: &CudaDevice, -) -> Result>, TensorDecodeError> { - validate_normalization_values(&options.normalization)?; - ensure_dtype::(device, DType::F32)?; - let width = canonical_width(input.encoded)?; - let plan = plan_image(input, options, width)?; - validate_normalization_channels(&options.normalization, plan.shape[2])?; - let shape = tensor_shape_3(plan.shape, options.layout); - let context = - CudaContext::retain_primary(device.index).map_err(|error| cuda_runtime_error(&error))?; - let tensor = fill_float_tensor::<3>(shape, device, &context, |destination| { - decode_plans_into( - std::slice::from_ref(&plan), - options, - width, - false, - false, - &context, - destination, - ) - })?; - Ok(single_result(tensor, &plan)) -} - -/// Decode a batch into one rank-4 U8 tensor without decoded-pixel host transfer. -pub fn decode_u8_batch( - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - device: &CudaDevice, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U8)?; - let plans = plan_batch(inputs, options, SampleWidth::U8)?; - let shape = tensor_shape_4(plans.len(), plans[0].shape, options.layout); - let context = - CudaContext::retain_primary(device.index).map_err(|error| cuda_runtime_error(&error))?; - let tensor = fill_int_tensor::<4>(shape, DType::U8, device, &context, |destination| { - decode_plans_into( - &plans, - options, - SampleWidth::U8, - true, - true, - &context, - destination, - ) - })?; - Ok(batch_result(tensor, &plans)) -} - -/// Decode a batch into one rank-4 U16 tensor without decoded-pixel host transfer. -pub fn decode_u16_batch( - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - device: &CudaDevice, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U16)?; - let plans = plan_batch(inputs, options, SampleWidth::U16)?; - let shape = tensor_shape_4(plans.len(), plans[0].shape, options.layout); - let context = - CudaContext::retain_primary(device.index).map_err(|error| cuda_runtime_error(&error))?; - let tensor = fill_int_tensor::<4>(shape, DType::U16, device, &context, |destination| { - decode_plans_into( - &plans, - options, - SampleWidth::U16, - true, - true, - &context, - destination, - ) - })?; - Ok(batch_result(tensor, &plans)) -} - -/// Decode a batch into one rank-4 F32 tensor without decoded-pixel host transfer. -pub fn decode_float_batch( - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - device: &CudaDevice, -) -> Result>, TensorDecodeError> { - validate_normalization_values(&options.normalization)?; - ensure_dtype::(device, DType::F32)?; - let first = inputs.first().ok_or(TensorDecodeError::EmptyBatch)?; - let width = canonical_width(first.encoded).map_err(|error| indexed(0, error))?; - for (index, input) in inputs.iter().enumerate().skip(1) { - let item_width = canonical_width(input.encoded).map_err(|error| indexed(index, error))?; - if item_width.dtype() != width.dtype() { - return Err(indexed( - index, - strict(format!( - "mixed canonical integer widths are unsupported: first item uses {:?}, item uses {:?}", - width.dtype(), - item_width.dtype() - )), - )); - } - } - let plans = plan_batch(inputs, options, width)?; - validate_normalization_channels(&options.normalization, plans[0].shape[2])?; - let shape = tensor_shape_4(plans.len(), plans[0].shape, options.layout); - let context = - CudaContext::retain_primary(device.index).map_err(|error| cuda_runtime_error(&error))?; - let tensor = fill_float_tensor::<4>(shape, device, &context, |destination| { - decode_plans_into(&plans, options, width, false, true, &context, destination) - })?; - Ok(batch_result(tensor, &plans)) -} - -fn canonical_width(encoded: &[u8]) -> Result { - Ok(if j2k::J2kDecoder::inspect(encoded)?.bit_depth <= 8 { - SampleWidth::U8 - } else { - SampleWidth::U16 - }) -} - -fn plan_image<'a>( - input: TensorInput<'a>, - options: &TensorDecodeOptions, - width: SampleWidth, -) -> Result, TensorDecodeError> { - let info = j2k::J2kDecoder::inspect(input.encoded)?; - let plan = DeviceDecodePlan::for_image(info.dimensions, input.request)?; - let channels = selected_channels(options.channels, info.components); - let (width_px, height_px) = plan.output_dims(); - let _ = pixel_format(channels, width); - Ok(PlannedImage { - input, - shape: [ - usize::try_from(height_px).map_err(|_| TensorDecodeError::SizeOverflow)?, - usize::try_from(width_px).map_err(|_| TensorDecodeError::SizeOverflow)?, - channels, - ], - decoded: plan.output_rect(), - }) -} - -fn plan_batch<'a>( - inputs: &[TensorInput<'a>], - options: &TensorDecodeOptions, - width: SampleWidth, -) -> Result>, TensorDecodeError> { - if inputs.is_empty() { - return Err(TensorDecodeError::EmptyBatch); - } - let mut plans: Vec> = Vec::new(); - plans - .try_reserve_exact(inputs.len()) - .map_err(|_| TensorDecodeError::SizeOverflow)?; - for (index, input) in inputs.iter().enumerate() { - let plan = plan_image(*input, options, width).map_err(|error| indexed(index, error))?; - if let Some(first) = plans.first() { - if plan.shape != first.shape { - return Err(TensorDecodeError::BatchShapeMismatch { - index, - expected: first.shape, - actual: plan.shape, - }); - } - } - plans.push(plan); - } - Ok(plans) -} - -fn decode_plans_into( - plans: &[PlannedImage<'_>], - options: &TensorDecodeOptions, - width: SampleWidth, - integer_output: bool, - index_errors: bool, - context: &CudaContext, - destination: &mut CudaExternalDeviceBufferViewMut<'_>, -) -> Result<(), TensorDecodeError> { - let mut session = CudaSession::with_context(context.clone()); - let item_elements = plans[0] - .shape - .iter() - .try_fold(1usize, |size, dim| size.checked_mul(*dim)) - .ok_or(TensorDecodeError::SizeOverflow)?; - for (index, plan) in plans.iter().enumerate() { - let mut decoder = CudaDecoder::new(plan.input.encoded) - .map_err(|error| route_item_error(index_errors, index, strict(error.to_string())))?; - let format = pixel_format(plan.shape[2], width); - let surface = decoder - .decode_request_to_device_with_session(format, plan.input.request, &mut session) - .map_err(|error| route_item_error(index_errors, index, strict(error.to_string())))?; - validate_surface(&surface, plan.shape, width) - .map_err(|error| route_item_error(index_errors, index, error))?; - let source = surface.cuda_surface().ok_or_else(|| { - route_item_error( - index_errors, - index, - strict("strict CUDA decode returned a host surface"), - ) - })?; - let source_len = item_elements - .checked_mul(match width { - SampleWidth::U8 => 1, - SampleWidth::U16 => 2, - }) - .ok_or(TensorDecodeError::SizeOverflow)?; - context - .j2k_ml_convert_into_external( - source.device_ptr(), - source_len, - destination, - kernel_config(plan, options, width, integer_output, index, item_elements)?, - ) - .map_err(|error| route_item_error(index_errors, index, cuda_runtime_error(&error)))?; - } - Ok(()) -} - -fn validate_surface( - surface: &Surface, - shape: [usize; 3], - width: SampleWidth, -) -> Result<(), TensorDecodeError> { - let sample_bytes = match width { - SampleWidth::U8 => 1, - SampleWidth::U16 => 2, - }; - let expected_pitch = shape[1] - .checked_mul(shape[2]) - .and_then(|value| value.checked_mul(sample_bytes)) - .ok_or(TensorDecodeError::SizeOverflow)?; - if surface.pitch_bytes() != expected_pitch { - return Err(strict(format!( - "CUDA surface pitch {} does not match compact pitch {expected_pitch}", - surface.pitch_bytes() - ))); - } - Ok(()) -} - -pub(super) fn strict(message: impl Into) -> TensorDecodeError { - TensorDecodeError::StrictRoute { - route: TensorRoute::CudaDirect, - message: message.into(), - } -} - -pub(super) fn cuda_runtime_error(error: &j2k_cuda_runtime::CudaError) -> TensorDecodeError { - strict(error.to_string()) -} - -fn indexed(index: usize, source: TensorDecodeError) -> TensorDecodeError { - TensorDecodeError::BatchItem { - index, - source: Box::new(source), - } -} - -fn route_item_error(is_batch: bool, index: usize, source: TensorDecodeError) -> TensorDecodeError { - if is_batch { - indexed(index, source) - } else { - source - } -} - -fn single_result(tensor: T, plan: &PlannedImage<'_>) -> TensorDecode { - TensorDecode { - tensor, - decoded: plan.decoded, - warnings: vec![J2kDecodeWarning::LenientDecodeMode], - route: TensorRoute::CudaDirect, - } -} - -fn batch_result(tensor: T, plans: &[PlannedImage<'_>]) -> TensorBatchDecode { - TensorBatchDecode { - tensor, - decoded: plans.iter().map(|plan| plan.decoded).collect(), - warnings: plans - .iter() - .map(|_| vec![J2kDecodeWarning::LenientDecodeMode]) - .collect(), - route: TensorRoute::CudaDirect, - } -} +pub use batch::{CudaBurnDecoder, SubmittedCudaBurnBatch}; diff --git a/crates/j2k-ml/src/cuda/batch.rs b/crates/j2k-ml/src/cuda/batch.rs new file mode 100644 index 00000000..4385dd38 --- /dev/null +++ b/crates/j2k-ml/src/cuda/batch.rs @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use burn_core::tensor::backend::Backend; +use burn_cuda::{Cuda, CudaDevice}; +use j2k::{ + BatchDecodeOptions, EncodedImage, IndexedBatchError, NativeSampleType, PreparedBatch, + PreparedBatchGroup, PreparedImage, +}; +use j2k_cuda::{CudaBatchDecoder as CodecDecoder, SubmittedCudaExternalBatch}; + +use super::interop::{fill_batch_int_tensor, register_int_tensor, SubmittedBatchIntTensor}; +use crate::batch_contract::{dtype, tensor_shape}; +use crate::{ + BurnBatchDecode, BurnBatchGroup, BurnBatchGroupError, BurnBatchTensor, BurnDecodeError, +}; + +/// Pending CUDA decode whose output allocation is not exposed to Burn until +/// group-level codec status validation succeeds. +#[must_use = "submitted CUDA Burn batches must be waited or dropped"] +pub struct SubmittedCudaBurnBatch { + groups: Vec, + errors: Vec, + group_errors: Vec, +} + +impl core::fmt::Debug for SubmittedCudaBurnBatch { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("SubmittedCudaBurnBatch") + .field("groups", &self.groups.len()) + .field("errors", &self.errors) + .field("group_errors", &self.group_errors) + .finish_non_exhaustive() + } +} + +impl SubmittedCudaBurnBatch { + /// Number of asynchronously submitted homogeneous groups. + #[must_use] + pub fn len(&self) -> usize { + self.groups.len() + } + + /// Whether no accelerator work was submitted. + #[must_use] + pub fn is_empty(&self) -> bool { + self.groups.is_empty() + } + + /// Wait once per group, validate its batched status, and only then expose + /// ordinary rank-4 Burn tensors. + pub fn wait(self) -> Result, BurnDecodeError> { + let Self { + groups, + errors, + group_errors, + } = self; + let (groups, group_errors) = + crate::completion::finish_submitted_groups(groups, group_errors, |group| { + let source_indices = group.source_indices.clone(); + (source_indices, group.wait()) + })?; + Ok(BurnBatchDecode { + groups, + errors, + group_errors, + }) + } +} + +struct SubmittedCudaBurnGroup { + sample_type: NativeSampleType, + source_indices: Vec, + allocation: SubmittedBatchIntTensor, +} + +impl SubmittedCudaBurnGroup { + fn wait(self) -> Result, BurnDecodeError> { + let sample_type = self.sample_type; + let (cube, shape, dtype, device, pending) = self.allocation.into_parts(); + let decoded = match pending.wait() { + Ok(decoded) => decoded, + Err(source) => { + if source.completion_is_uncertain() { + std::mem::forget(cube); + } + return Err(BurnDecodeError::Cuda(source)); + } + }; + let tensor = register_int_tensor(cube, shape, dtype, &device); + let tensor = match sample_type { + NativeSampleType::U8 => BurnBatchTensor::U8(tensor), + NativeSampleType::U16 => BurnBatchTensor::U16(tensor), + NativeSampleType::I16 => BurnBatchTensor::I16(tensor), + _ => return Err(BurnDecodeError::UnsupportedCodecContract), + }; + Ok(BurnBatchGroup { + tensor, + info: decoded.info().clone(), + source_indices: decoded.source_indices().to_vec(), + decoded_rects: decoded.decoded_rects().to_vec(), + warnings: decoded.warnings().to_vec(), + }) + } +} + +/// Persistent CUDA codec session writing directly into Burn-owned allocations. +#[derive(Debug)] +pub struct CudaBurnDecoder { + codec: CodecDecoder, + device: CudaDevice, +} + +impl CudaBurnDecoder { + /// Create a decoder for one Burn CUDA device. The retained primary context + /// is initialized lazily when the first nonempty valid group is submitted. + pub fn new(device: CudaDevice, options: BatchDecodeOptions) -> Result { + Ok(Self { + codec: CodecDecoder::with_options(options), + device, + }) + } + + /// Borrow the retained codec session. + #[must_use] + pub const fn codec(&self) -> &CodecDecoder { + &self.codec + } + + /// Burn CUDA device receiving decoded tensors. + #[must_use] + pub const fn device(&self) -> &CudaDevice { + &self.device + } + + /// Parse and group owned inputs once for repeated decode calls. + pub fn prepare(&self, inputs: Vec) -> Result { + Ok(self.codec.prepare(inputs)?) + } + + /// Regroup caller-supplied prepared images without reparsing codestream bytes. + pub fn prepare_prepared_images( + &self, + images: Vec, + ) -> Result { + Ok(self.codec.prepare_prepared_images(images)?) + } + + /// Prepare and synchronously finish one owned batch. Use [`Self::submit`] + /// to overlap caller work with GPU execution. + pub fn decode( + &mut self, + inputs: Vec, + ) -> Result, BurnDecodeError> { + self.submit(inputs)?.wait() + } + + /// Regroup and synchronously decode caller-supplied prepared images. + pub fn decode_prepared_images( + &mut self, + images: Vec, + ) -> Result, BurnDecodeError> { + let prepared = self.prepare_prepared_images(images)?; + self.decode_prepared(&prepared) + } + + /// Finish a reusable codec preparation and return validated Burn tensors. + pub fn decode_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result, BurnDecodeError> { + self.submit_prepared(prepared)?.wait() + } + + /// Prepare and asynchronously submit one owned batch without exposing + /// partially written tensor storage. + pub fn submit( + &mut self, + inputs: Vec, + ) -> Result { + let prepared = self.prepare(inputs)?; + self.submit_prepared(&prepared) + } + + /// Asynchronously submit a reusable preparation. The returned guard owns + /// every fresh Burn allocation and codec completion resource until + /// [`SubmittedCudaBurnBatch::wait`] validates the whole group. + pub fn submit_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result { + let mut groups = Vec::new(); + groups + .try_reserve_exact(prepared.groups().len()) + .map_err(|_| BurnDecodeError::SizeOverflow)?; + let mut group_errors = Vec::new(); + group_errors + .try_reserve_exact(prepared.groups().len()) + .map_err(|_| BurnDecodeError::SizeOverflow)?; + for group in prepared.groups() { + match self.submit_group(group) { + Ok(submitted) => groups.push(submitted), + Err(source) if crate::completion::burn_group_error_is_fatal(&source) => { + return Err(source); + } + Err(source) => group_errors.push(BurnBatchGroupError::new( + group.source_indices().to_vec(), + source, + )), + } + } + Ok(SubmittedCudaBurnBatch { + groups, + errors: prepared.errors().to_vec(), + group_errors, + }) + } + + fn submit_group( + &mut self, + group: &PreparedBatchGroup, + ) -> Result { + let sample_type = group.info().sample_type; + let dtype = dtype(sample_type)?; + if ! as Backend>::supports_dtype(&self.device, dtype) { + return Err(BurnDecodeError::UnsupportedDType { dtype }); + } + let shape = tensor_shape(group.source_indices().len(), group.info())?; + let device = self.device.clone(); + let context = self + .codec + .session_mut() + .context_for_device_interop(self.device.index) + .map_err(|error| BurnDecodeError::AcceleratorInterop { + backend: "CUDA", + message: error.to_string(), + })?; + let allocation = fill_batch_int_tensor(shape, dtype, &device, &context, |destination| { + // SAFETY: the interop owner keeps the unique Burn allocation live, + // establishes CubeCL-to-codec-to-CubeCL event ordering, and stores + // the returned completion guard ahead of that allocation. + Ok(unsafe { self.codec.submit_batch_into(group, destination) }?) + })?; + if allocation.payload().group().info().sample_type != sample_type { + return Err(BurnDecodeError::SampleTypeMismatch); + } + Ok(SubmittedCudaBurnGroup { + sample_type, + source_indices: group.source_indices().to_vec(), + allocation, + }) + } +} diff --git a/crates/j2k-ml/src/cuda/config.rs b/crates/j2k-ml/src/cuda/config.rs deleted file mode 100644 index 1f515344..00000000 --- a/crates/j2k-ml/src/cuda/config.rs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use j2k_cuda_runtime::{ - CudaJ2kMlKernelConfig, CudaJ2kMlLayout, CudaJ2kMlNormalization, CudaJ2kMlSample, -}; - -use super::PlannedImage; -use crate::cpu::SampleWidth; -use crate::{FloatNormalization, TensorDecodeError, TensorDecodeOptions, TensorLayout}; - -pub(super) fn kernel_config( - plan: &PlannedImage<'_>, - options: &TensorDecodeOptions, - width: SampleWidth, - integer_output: bool, - index: usize, - item_elements: usize, -) -> Result { - let sample = match width { - SampleWidth::U8 => CudaJ2kMlSample::U8, - SampleWidth::U16 => CudaJ2kMlSample::U16, - }; - let layout = match options.layout { - TensorLayout::ChannelsFirst => CudaJ2kMlLayout::ChannelsFirst, - TensorLayout::ChannelsLast => CudaJ2kMlLayout::ChannelsLast, - }; - let normalization = if integer_output { - CudaJ2kMlNormalization::Integer - } else { - match &options.normalization { - FloatNormalization::Raw => CudaJ2kMlNormalization::Raw, - FloatNormalization::Unit => CudaJ2kMlNormalization::Unit, - FloatNormalization::MeanStd { mean, std } => { - let mut means = [0.0; 4]; - let mut deviations = [1.0; 4]; - means[..plan.shape[2]].copy_from_slice(mean); - deviations[..plan.shape[2]].copy_from_slice(std); - CudaJ2kMlNormalization::MeanStd { - mean: means, - std: deviations, - } - } - } - }; - Ok(CudaJ2kMlKernelConfig { - width: u32::try_from(plan.shape[1]).map_err(|_| TensorDecodeError::SizeOverflow)?, - height: u32::try_from(plan.shape[0]).map_err(|_| TensorDecodeError::SizeOverflow)?, - channels: u32::try_from(plan.shape[2]).map_err(|_| TensorDecodeError::SizeOverflow)?, - sample, - layout, - destination_offset_elements: index - .checked_mul(item_elements) - .ok_or(TensorDecodeError::SizeOverflow)?, - normalization, - }) -} - -pub(super) fn tensor_shape_3(shape: [usize; 3], layout: TensorLayout) -> [usize; 3] { - match layout { - TensorLayout::ChannelsFirst => [shape[2], shape[0], shape[1]], - TensorLayout::ChannelsLast => shape, - } -} - -pub(super) fn tensor_shape_4(batch: usize, shape: [usize; 3], layout: TensorLayout) -> [usize; 4] { - match layout { - TensorLayout::ChannelsFirst => [batch, shape[2], shape[0], shape[1]], - TensorLayout::ChannelsLast => [batch, shape[0], shape[1], shape[2]], - } -} diff --git a/crates/j2k-ml/src/cuda/interop.rs b/crates/j2k-ml/src/cuda/interop.rs index 330b56a3..f8b30755 100644 --- a/crates/j2k-ml/src/cuda/interop.rs +++ b/crates/j2k-ml/src/cuda/interop.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use burn_core::tensor::{DType, Int, Shape, Tensor, TensorPrimitive}; +use burn_core::tensor::{DType, Int, Shape, Tensor}; use burn_cubecl::cubecl::{cuda::CudaRuntime, Runtime}; use burn_cubecl::{ops::numeric::empty_device_contiguous_dtype, CubeBackend}; use burn_cuda::{Cuda, CudaDevice}; @@ -8,102 +8,150 @@ use burn_fusion::{get_client, stream::OperationStreams, NoOp}; use burn_ir::{BackendIr, InitOperationIr, OperationIr}; use j2k_cuda_runtime::{CudaContext, CudaExternalDeviceBufferViewMut}; -use super::{cuda_runtime_error, strict}; -use crate::TensorDecodeError; +use crate::BurnDecodeError; type InnerCuda = CubeBackend; -pub(super) fn fill_int_tensor( - shape: [usize; D], +/// Fresh Burn allocation plus an asynchronously submitted codec owner. +/// +/// `payload` is declared first so dropping an unfinished value retires codec +/// work before the `CubeCL` allocation is released. +pub(super) struct SubmittedBatchIntTensor { + payload: R, + cube: burn_cubecl::tensor::CubeTensor, + shape: Shape, dtype: DType, - device: &CudaDevice, - context: &CudaContext, - fill: impl FnOnce(&mut CudaExternalDeviceBufferViewMut<'_>) -> Result<(), TensorDecodeError>, -) -> Result, TensorDecodeError> { - let cube = fill_cube_tensor(shape, dtype, device, context, fill)?; - Ok(register_int_tensor( - cube, - Shape::from(shape.to_vec()), - dtype, - device, - )) + device: CudaDevice, } -pub(super) fn fill_float_tensor( - shape: [usize; D], - device: &CudaDevice, - context: &CudaContext, - fill: impl FnOnce(&mut CudaExternalDeviceBufferViewMut<'_>) -> Result<(), TensorDecodeError>, -) -> Result, TensorDecodeError> { - let cube = fill_cube_tensor(shape, DType::F32, device, context, fill)?; - Ok(register_float_tensor( - cube, - Shape::from(shape.to_vec()), - device, - )) +impl SubmittedBatchIntTensor { + pub(super) fn payload(&self) -> &R { + &self.payload + } + + pub(super) fn into_parts( + self, + ) -> ( + burn_cubecl::tensor::CubeTensor, + Shape, + DType, + CudaDevice, + R, + ) { + let Self { + payload, + cube, + shape, + dtype, + device, + } = self; + (cube, shape, dtype, device, payload) + } } -fn fill_cube_tensor( +pub(super) fn fill_batch_int_tensor( shape: [usize; D], dtype: DType, device: &CudaDevice, context: &CudaContext, - fill: impl FnOnce(&mut CudaExternalDeviceBufferViewMut<'_>) -> Result<(), TensorDecodeError>, -) -> Result, TensorDecodeError> { + fill: impl FnOnce(&mut CudaExternalDeviceBufferViewMut<'_>) -> Result, +) -> Result, BurnDecodeError> { let logical_len = tensor_byte_len(&shape, dtype)?; - let shape = Shape::from(shape.to_vec()); + let burn_shape = Shape::from(shape.to_vec()); let client = CudaRuntime::client(device); - let cube = empty_device_contiguous_dtype(client, device.clone(), shape, dtype); - // CubeCL allocates asynchronously. Its stream must finish before j2k writes - // through the default CUDA stream because no event interop is exposed. - burn_cubecl::cubecl::future::block_on(cube.client.sync()) - .map_err(|error| strict(format!("CubeCL CUDA allocation handoff failed: {error}")))?; + let cube = empty_device_contiguous_dtype(client, device.clone(), burn_shape.clone(), dtype); let handle_len = - usize::try_from(cube.handle.size_in_used()).map_err(|_| TensorDecodeError::SizeOverflow)?; + usize::try_from(cube.handle.size_in_used()).map_err(|_| BurnDecodeError::SizeOverflow)?; if handle_len != logical_len { - return Err(strict(format!( - "CubeCL CUDA tensor handle exposes {handle_len} bytes; expected {logical_len}" + return Err(interop(format!( + "CubeCL tensor handle exposes {handle_len} bytes; expected {logical_len}" ))); } + + // SAFETY: the stream token stays inside this audited CUDA bridge. The + // client, allocation, managed resource, and codec completion owner remain + // live until both cross-stream event dependencies have been registered. + let raw_stream = unsafe { cube.client.external_write_stream(&cube.handle) } + .map_err(|error| interop(format!("CubeCL stream handoff failed: {error}")))?; + let mut stream_owner = cube.client.clone(); let mut resource = cube .client .get_resource(cube.handle.clone()) - .map_err(|error| strict(format!("CubeCL CUDA resource access failed: {error}")))?; + .map_err(|error| interop(format!("CubeCL resource access failed: {error}")))?; let raw = resource.resource(); - let available = usize::try_from(raw.size).map_err(|_| TensorDecodeError::SizeOverflow)?; + let available = usize::try_from(raw.size).map_err(|_| BurnDecodeError::SizeOverflow)?; if logical_len > available { - return Err(strict(format!( - "CubeCL CUDA resource exposes {available} bytes for a {logical_len}-byte tensor" + return Err(interop(format!( + "CubeCL resource exposes {available} bytes for a {logical_len}-byte tensor" ))); } - let pointer = raw.ptr; - // SAFETY: `resource` is CubeCL's managed allocation guard for `pointer`. - // Its exclusive borrow spans `fill`; context, extent, and alignment are - // validated by the non-owning view constructor. - let mut destination = unsafe { - CudaExternalDeviceBufferViewMut::from_raw_parts( + + // SAFETY: this is a fresh, unregistered CubeCL tensor. Its managed + // resource is exclusively borrowed for the complete codec submission; + // the view validates retained-primary-context identity, extent, and + // alignment. `with_primary_stream_ordering` orders allocation before the + // codec and the Burn stream after the codec without a CPU synchronization. + let submission = unsafe { + let mut destination = CudaExternalDeviceBufferViewMut::from_raw_parts( context, - pointer, + raw.ptr, logical_len, dtype.size(), &mut resource, ) - } - .map_err(|error| cuda_runtime_error(&error))?; - fill(&mut destination)?; - drop(destination); + .map_err(|error| interop(error.to_string()))?; + let mut work_invoked = false; + let ordered = context.with_primary_stream_ordering(raw_stream, &mut stream_owner, || { + work_invoked = true; + fill(&mut destination) + }); + drop(destination); + match ordered { + Ok(Ok(output)) => Ok(output), + Ok(Err(error)) => { + let quarantine = burn_error_completion_is_uncertain(&error); + Err((error, quarantine)) + } + Err(error) => { + let quarantine = work_invoked && error.completion_is_uncertain(); + Err(( + interop(format!("CUDA stream event handoff failed: {error}")), + quarantine, + )) + } + } + }; drop(resource); - Ok(cube) + let output = match submission { + Ok(output) => output, + Err((error, quarantine)) => { + if quarantine { + // CUDA could not prove that the external allocation is no + // longer referenced. Quarantine it rather than letting CubeCL + // recycle or free storage still reachable by the driver. + std::mem::forget(cube); + } + return Err(error); + } + }; + + Ok(SubmittedBatchIntTensor { + payload: output, + cube, + shape: burn_shape, + dtype, + device: device.clone(), + }) } -pub(super) fn tensor_byte_len(shape: &[usize], dtype: DType) -> Result { +fn tensor_byte_len(shape: &[usize], dtype: DType) -> Result { shape .iter() .try_fold(dtype.size(), |size, dim| size.checked_mul(*dim)) - .ok_or(TensorDecodeError::SizeOverflow) + .ok_or(BurnDecodeError::SizeOverflow) } -fn register_int_tensor( +pub(super) fn register_int_tensor( cube: burn_cubecl::tensor::CubeTensor, shape: Shape, dtype: DType, @@ -122,37 +170,51 @@ fn register_int_tensor( Tensor::::from_primitive(primitive) } -fn register_float_tensor( - cube: burn_cubecl::tensor::CubeTensor, - shape: Shape, - device: &CudaDevice, -) -> Tensor { - let fusion = get_client::(device); - let handle = ::float_tensor_handle(cube); - let desc = InitOperationIr::create(shape, DType::F32, || fusion.register_tensor_handle(handle)); - let primitive = fusion - .register( - OperationStreams::default(), - OperationIr::Init(desc), - NoOp::::new(), - ) - .remove(0); - Tensor::::from_primitive(TensorPrimitive::Float(primitive)) +fn interop(message: impl Into) -> BurnDecodeError { + BurnDecodeError::AcceleratorInterop { + backend: "CUDA", + message: message.into(), + } +} + +fn burn_error_completion_is_uncertain(error: &BurnDecodeError) -> bool { + matches!(error, BurnDecodeError::Cuda(source) if source.completion_is_uncertain()) } #[cfg(test)] mod tests { use burn_core::tensor::DType; - use super::tensor_byte_len; - use crate::TensorDecodeError; + use super::{burn_error_completion_is_uncertain, tensor_byte_len}; + use crate::BurnDecodeError; #[test] - fn tensor_byte_length_is_exact_and_overflow_checked_before_cubecl_allocation() { + fn tensor_byte_length_is_exact_and_overflow_checked_before_allocation() { assert_eq!(tensor_byte_len(&[2, 3, 4], DType::U16).unwrap(), 48); assert!(matches!( - tensor_byte_len(&[usize::MAX, 2], DType::F32), - Err(TensorDecodeError::SizeOverflow) + tensor_byte_len(&[usize::MAX, 2], DType::U16), + Err(BurnDecodeError::SizeOverflow) )); } + + #[test] + fn only_uncertain_cuda_completion_requires_allocation_quarantine() { + let validation = BurnDecodeError::Cuda(j2k_cuda::CudaBatchError::GroupExecution { + source_indices: vec![0], + source: Box::new(j2k_cuda::Error::UnsupportedCudaRequest { + reason: "pre-submit validation", + }), + }); + assert!(!burn_error_completion_is_uncertain(&validation)); + + let uncertain = BurnDecodeError::Cuda(j2k_cuda::CudaBatchError::GroupExecution { + source_indices: vec![0], + source: Box::new(j2k_cuda::Error::CudaRuntime { + source: j2k_cuda_runtime::CudaError::StatePoisoned { + message: "completion unknown".to_string(), + }, + }), + }); + assert!(burn_error_completion_is_uncertain(&uncertain)); + } } diff --git a/crates/j2k-ml/src/lib.rs b/crates/j2k-ml/src/lib.rs index fbfc955c..4656f887 100644 --- a/crates/j2k-ml/src/lib.rs +++ b/crates/j2k-ml/src/lib.rs @@ -1,11 +1,21 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -//! Independent Burn tensor integration for JPEG 2000 and HTJ2K. +//! Thin Burn tensor adapter for the `j2k` owned batch codec. +//! +//! The codec crates own parsing, grouping, decoding, and accelerator execution. +//! This crate only materializes CPU groups or lends unique Burn allocations to +//! the CUDA and Metal external-destination APIs. Casting and normalization stay +//! in ordinary Burn tensor operations after decode. #![deny(missing_docs)] -use burn_core::tensor::Tensor; -use j2k::{DeviceDecodeRequest, J2kDecodeWarning, Rect}; +use burn_core::tensor::{backend::Backend, Int, Tensor}; +use j2k::{BatchGroupInfo, IndexedBatchError, J2kDecodeWarning, Rect}; + +#[cfg(any(feature = "cpu", feature = "cuda", feature = "metal"))] +mod batch_contract; +#[cfg(any(feature = "cuda", feature = "metal", test))] +mod completion; #[cfg(feature = "cpu")] pub mod cpu; @@ -14,195 +24,144 @@ pub mod cuda; #[cfg(feature = "metal")] pub mod metal; -/// Tensor memory layout. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum TensorLayout { - /// Channels precede spatial dimensions (`CHW` or `NCHW`). - #[default] - ChannelsFirst, - /// Channels follow spatial dimensions (`HWC` or `NHWC`). - ChannelsLast, -} +#[cfg(feature = "cpu")] +pub use cpu::CpuBurnDecoder; +#[cfg(feature = "cuda")] +pub use cuda::{CudaBurnDecoder, SubmittedCudaBurnBatch}; +#[cfg(feature = "metal")] +pub use metal::{MetalBurnDecoder, SubmittedMetalBurnBatch}; -/// Output channel selection. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum ChannelSelection { - /// Preserve grayscale as one channel and otherwise produce RGB. - #[default] - Auto, - /// Produce one grayscale channel. - Gray, - /// Produce three RGB channels. - Rgb, - /// Produce four RGBA channels. - Rgba, +/// Ordinary rank-4 Burn integer tensor tagged with its exact codec sample type. +#[derive(Debug)] +pub enum BurnBatchTensor { + /// Unsigned samples with precision at most eight bits. + U8(Tensor), + /// Unsigned samples with precision from nine through sixteen bits. + U16(Tensor), + /// Signed samples with precision at most sixteen bits. + I16(Tensor), } -/// Floating-point sample normalization. -#[derive(Debug, Clone, Default, PartialEq)] -pub enum FloatNormalization { - /// Scale integer samples into the inclusive range `0..=1`. - #[default] - Unit, - /// Cast integer samples without scaling. - Raw, - /// Unit-scale, then apply per-channel `(x - mean) / std`. - MeanStd { - /// Per-channel means. - mean: Vec, - /// Per-channel standard deviations. - std: Vec, - }, -} +impl BurnBatchTensor { + /// Borrow the ordinary Burn integer tensor regardless of its codec dtype tag. + #[must_use] + pub const fn tensor(&self) -> &Tensor { + match self { + Self::U8(tensor) | Self::U16(tensor) | Self::I16(tensor) => tensor, + } + } -/// Options shared by all tensor decode routes. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct TensorDecodeOptions { - /// Requested tensor layout. - pub layout: TensorLayout, - /// Requested output channels. - pub channels: ChannelSelection, - /// Floating-point normalization. - pub normalization: FloatNormalization, + /// Consume the codec dtype tag and return the ordinary Burn integer tensor. + #[must_use] + pub fn into_tensor(self) -> Tensor { + match self { + Self::U8(tensor) | Self::U16(tensor) | Self::I16(tensor) => tensor, + } + } } -/// Route that produced a tensor. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TensorRoute { - /// Host decode followed by a Burn upload. - CpuStaged, - /// Decode and conversion directly into a CUDA tensor allocation. - CudaDirect, - /// Metal decode followed by one packed readback and Burn upload. - MetalStaged, +/// One homogeneous decoded tensor group and its codec metadata. +#[derive(Debug)] +pub struct BurnBatchGroup { + /// Decoded rank-4 integer tensor. + pub tensor: BurnBatchTensor, + /// Exact codec and output metadata shared by the group. + pub info: BatchGroupInfo, + /// Original caller indices in tensor batch order. + pub source_indices: Vec, + /// Actual decoded source rectangle for each tensor item. + pub decoded_rects: Vec, + /// Non-fatal codec warnings for each tensor item. + pub warnings: Vec>, } -/// Borrowed compressed input and its decode geometry request. -#[derive(Debug, Clone, Copy)] -pub struct TensorInput<'a> { - /// JP2, JPH, raw J2K, or raw HTJ2K bytes. - pub encoded: &'a [u8], - /// Full-frame, ROI, scaled, or ROI-scaled request. - pub request: DeviceDecodeRequest, +/// Failure while submitting or completing one homogeneous Burn tensor group. +/// +/// No partially written tensor from the affected group is exposed. Other +/// homogeneous groups may still succeed when the retained codec and framework +/// sessions remain usable. +#[derive(Debug, thiserror::Error)] +#[error("Burn batch group containing source indices {source_indices:?} failed: {source}")] +pub struct BurnBatchGroupError { + source_indices: Vec, + #[source] + source: BurnDecodeError, } -impl<'a> TensorInput<'a> { - /// Construct a full-resolution input. - #[must_use] - pub const fn full(encoded: &'a [u8]) -> Self { +impl BurnBatchGroupError { + #[cfg(any(feature = "cuda", feature = "metal", test))] + pub(crate) fn new(source_indices: Vec, source: BurnDecodeError) -> Self { Self { - encoded, - request: DeviceDecodeRequest::Full, + source_indices, + source, } } -} -/// Successful single-image tensor decode. -#[derive(Debug)] -pub struct TensorDecode { - /// Decoded Burn tensor. - pub tensor: T, - /// Rectangle actually decoded. - pub decoded: Rect, - /// Non-fatal codec warnings. - pub warnings: Vec, - /// Route actually used. - pub route: TensorRoute, + /// Original input indices whose dense tensor group was discarded. + #[must_use] + pub fn source_indices(&self) -> &[usize] { + &self.source_indices + } + + /// Structured codec, framework, or interop failure for this group. + #[must_use] + pub const fn source(&self) -> &BurnDecodeError { + &self.source + } + + /// Consume the group failure into affected indices and its source. + #[must_use] + pub fn into_parts(self) -> (Vec, BurnDecodeError) { + (self.source_indices, self.source) + } } -/// Successful batch tensor decode. +/// Successful tensor groups plus indexed preparation and homogeneous execution failures. #[derive(Debug)] -pub struct TensorBatchDecode { - /// Decoded Burn tensor. - pub tensor: T, - /// Rectangle decoded for each item, in input order. - pub decoded: Vec, - /// Codec warnings for each item, in input order. - pub warnings: Vec>, - /// Route actually used. - pub route: TensorRoute, +pub struct BurnBatchDecode { + /// Successfully decoded homogeneous tensor groups. + pub groups: Vec>, + /// Structured preparation or decode failures in original input order. + pub errors: Vec, + /// Homogeneous groups discarded after adapter submission or completion failed. + pub group_errors: Vec, } -/// Tensor decode failure. +/// Failure at the codec-to-Burn ownership boundary. #[derive(Debug, thiserror::Error)] -pub enum TensorDecodeError { - /// The codec rejected the compressed input or decode request. - #[error("JPEG 2000 decode failed: {0}")] - Codec(#[from] j2k::J2kError), - /// A requested integer dtype is unsupported by the selected Burn backend. - #[error("Burn backend does not support requested dtype {dtype:?}")] +pub enum BurnDecodeError { + /// The codec could not allocate or schedule the requested batch. + #[error("JPEG 2000 batch infrastructure failed: {0}")] + Infrastructure(#[from] j2k::BatchInfrastructureError), + /// The selected Burn backend cannot represent the codec's exact integer type. + #[error("Burn backend does not support exact codec dtype {dtype:?}")] UnsupportedDType { - /// Unsupported dtype. + /// Required Burn storage dtype. dtype: burn_core::tensor::DType, }, - /// Normalization parameters are invalid. - #[error("invalid float normalization: {message}")] - InvalidNormalization { - /// Actionable validation detail. - message: String, - }, - /// A batch contained no inputs. - #[error("cannot decode an empty tensor batch")] - EmptyBatch, - /// A batch item shape differs from the first item. - #[error("batch item {index} has shape {actual:?}; expected {expected:?}")] - BatchShapeMismatch { - /// Index of the mismatching item. - index: usize, - /// Expected HWC shape. - expected: [usize; 3], - /// Actual HWC shape. - actual: [usize; 3], - }, - /// A particular batch item failed to decode or convert. - #[error("batch item {index} failed: {source}")] - BatchItem { - /// Input index. - index: usize, - /// Item-specific failure. - #[source] - source: Box, - }, - /// A requested allocation size overflowed `usize`. - #[error("tensor size overflow")] + /// Codec group metadata and the returned native sample owner disagreed. + #[error("codec batch sample owner did not match its declared sample type")] + SampleTypeMismatch, + /// Tensor shape arithmetic overflowed the host index type. + #[error("Burn tensor shape overflow")] SizeOverflow, - /// Accelerator route failed without falling back. - #[error("strict {route:?} route failed: {message}")] - StrictRoute { - /// Requested route. - route: TensorRoute, - /// Backend failure detail. + /// A newer codec contract cannot be represented by this adapter version. + #[error("unsupported codec batch layout or sample type")] + UnsupportedCodecContract, + /// CUDA rejected or could not complete one homogeneous codec group. + #[cfg(feature = "cuda")] + #[error(transparent)] + Cuda(#[from] j2k_cuda::CudaBatchError), + /// Metal rejected or could not complete one homogeneous codec group. + #[cfg(feature = "metal")] + #[error(transparent)] + Metal(#[from] j2k_metal::Error), + /// A framework allocation could not be handed to an accelerator safely. + #[error("{backend} tensor interop failed: {message}")] + AcceleratorInterop { + /// Accelerator runtime at the failing boundary. + backend: &'static str, + /// Actionable ownership, context, bounds, or ordering detail. message: String, }, } - -/// Infallible Burn batcher that intentionally panics on float decode errors. -#[derive(Debug, Clone)] -pub struct PanicOnDecodeError { - options: TensorDecodeOptions, - backend: core::marker::PhantomData, -} - -impl PanicOnDecodeError { - /// Construct an adapter with explicit decode options. - #[must_use] - pub fn new(options: TensorDecodeOptions) -> Self { - Self { - options, - backend: core::marker::PhantomData, - } - } -} - -#[cfg(feature = "cpu")] -impl<'a, B> burn_core::data::dataloader::batcher::Batcher, Tensor> - for PanicOnDecodeError -where - B: burn_core::tensor::backend::Backend, -{ - fn batch(&self, items: Vec>, device: &B::Device) -> Tensor { - cpu::decode_float_batch(&items, &self.options, device) - .unwrap_or_else(|error| panic!("j2k-ml batch decode failed: {error}")) - .tensor - } -} diff --git a/crates/j2k-ml/src/metal.rs b/crates/j2k-ml/src/metal.rs index c47bf465..52a7246e 100644 --- a/crates/j2k-ml/src/metal.rs +++ b/crates/j2k-ml/src/metal.rs @@ -1,409 +1,9 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -//! Strict J2K Metal decode followed by a compact staged Burn Metal upload. +//! Metal codec output written into Burn-owned allocations. -// The non-macOS Metal session is intentionally a zero-sized compatibility -// stub. Keep these private helper signatures identical across targets. -#![cfg_attr( - not(target_os = "macos"), - allow( - clippy::trivially_copy_pass_by_ref, - reason = "the non-macOS Metal compatibility session is intentionally zero-sized" - ) -)] - -use burn_core::tensor::{DType, FloatDType, Int, Tensor}; -use burn_wgpu::{Wgpu, WgpuDevice}; -use j2k::{ - BackendRequest, DeviceDecodePlan, DeviceDecodeRequest, J2kDecodeWarning, J2kDecoder, Rect, -}; +mod batch; #[cfg(target_os = "macos")] -use j2k_metal::download_surfaces_packed; -use j2k_metal::{ - J2kDecoder as MetalDecoder, MetalBackendSession, MetalDecodeRequest, Surface, SurfaceResidency, -}; - -use crate::cpu::{ - ensure_dtype, integer_tensor_3_from_bytes, integer_tensor_4_from_bytes, normalize_3, - normalize_4, pixel_format, planned_shape, selected_channels, validate_normalization_channels, - validate_normalization_values, SampleWidth, -}; -use crate::{ - TensorBatchDecode, TensorDecode, TensorDecodeError, TensorDecodeOptions, TensorInput, - TensorRoute, -}; - -type MetalTensor = Tensor; -type MetalIntTensor = Tensor; - -struct MetalImage { - surface: Surface, - shape: [usize; 3], - sample_width: SampleWidth, - decoded: Rect, - warnings: Vec, -} - -/// Decode one image into a rank-3 U8 Burn Metal tensor. -pub fn decode_u8( - input: TensorInput<'_>, - options: &TensorDecodeOptions, - device: &WgpuDevice, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U8)?; - let session = metal_session()?; - let image = decode_surface(&session, input, options, SampleWidth::U8)?; - let bytes = packed_readback(&session, &[(&image.surface, image_byte_len(&image)?)])?; - let tensor = - integer_tensor_3_from_bytes::(bytes, image.shape, options.layout, device, DType::U8); - Ok(single_result(tensor, image)) -} - -/// Decode one image into a rank-3 U16 Burn Metal tensor. -pub fn decode_u16( - input: TensorInput<'_>, - options: &TensorDecodeOptions, - device: &WgpuDevice, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U16)?; - let session = metal_session()?; - let image = decode_surface(&session, input, options, SampleWidth::U16)?; - let bytes = packed_readback(&session, &[(&image.surface, image_byte_len(&image)?)])?; - let tensor = - integer_tensor_3_from_bytes::(bytes, image.shape, options.layout, device, DType::U16); - Ok(single_result(tensor, image)) -} - -/// Decode one image into a rank-3 F32 Burn Metal tensor. -pub fn decode_float( - input: TensorInput<'_>, - options: &TensorDecodeOptions, - device: &WgpuDevice, -) -> Result>, TensorDecodeError> { - validate_normalization_values(&options.normalization)?; - ensure_dtype::(device, DType::F32)?; - let info = J2kDecoder::inspect(input.encoded)?; - let width = sample_width(info.bit_depth); - ensure_dtype::(device, width.dtype())?; - validate_normalization_channels( - &options.normalization, - selected_channels(options.channels, info.components), - )?; - let session = metal_session()?; - let image = decode_surface(&session, input, options, width)?; - let bytes = packed_readback(&session, &[(&image.surface, image_byte_len(&image)?)])?; - let tensor = integer_tensor_3_from_bytes::( - bytes, - image.shape, - options.layout, - device, - width.dtype(), - ) - .cast(FloatDType::F32); - let tensor = normalize_3( - tensor, - &options.normalization, - options.layout, - image.shape[2], - width, - device, - ); - Ok(single_result(tensor, image)) -} - -/// Decode a batch into one rank-4 U8 Burn Metal tensor. -pub fn decode_u8_batch( - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - device: &WgpuDevice, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U8)?; - let session = metal_session()?; - let images = decode_surfaces(&session, inputs, options, SampleWidth::U8)?; - let (bytes, shape) = readback_batch(&session, &images)?; - let tensor = integer_tensor_4_from_bytes::( - bytes, - images.len(), - shape, - options.layout, - device, - DType::U8, - ); - Ok(batch_result(tensor, images)) -} - -/// Decode a batch into one rank-4 U16 Burn Metal tensor. -pub fn decode_u16_batch( - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - device: &WgpuDevice, -) -> Result>, TensorDecodeError> { - ensure_dtype::(device, DType::U16)?; - let session = metal_session()?; - let images = decode_surfaces(&session, inputs, options, SampleWidth::U16)?; - let (bytes, shape) = readback_batch(&session, &images)?; - let tensor = integer_tensor_4_from_bytes::( - bytes, - images.len(), - shape, - options.layout, - device, - DType::U16, - ); - Ok(batch_result(tensor, images)) -} - -/// Decode a batch into one rank-4 F32 Burn Metal tensor. -pub fn decode_float_batch( - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - device: &WgpuDevice, -) -> Result>, TensorDecodeError> { - validate_normalization_values(&options.normalization)?; - ensure_dtype::(device, DType::F32)?; - let first = inputs.first().ok_or(TensorDecodeError::EmptyBatch)?; - let first_info = - J2kDecoder::inspect(first.encoded).map_err(|error| indexed(0, error.into()))?; - let width = sample_width(first_info.bit_depth); - ensure_dtype::(device, width.dtype())?; - validate_normalization_channels( - &options.normalization, - selected_channels(options.channels, first_info.components), - ) - .map_err(|error| indexed(0, error))?; - for (index, input) in inputs.iter().enumerate().skip(1) { - let item_info = - J2kDecoder::inspect(input.encoded).map_err(|error| indexed(index, error.into()))?; - let item_width = sample_width(item_info.bit_depth); - if item_width.dtype() != width.dtype() { - return Err(indexed( - index, - strict(format!( - "mixed canonical integer widths are unsupported: first item uses {:?}, item uses {:?}", - width.dtype(), - item_width.dtype() - )), - )); - } - validate_normalization_channels( - &options.normalization, - selected_channels(options.channels, item_info.components), - ) - .map_err(|error| indexed(index, error))?; - } - let session = metal_session()?; - let images = decode_surfaces(&session, inputs, options, width)?; - let (bytes, shape) = readback_batch(&session, &images)?; - let tensor = integer_tensor_4_from_bytes::( - bytes, - images.len(), - shape, - options.layout, - device, - width.dtype(), - ) - .cast(FloatDType::F32); - let tensor = normalize_4( - tensor, - &options.normalization, - options.layout, - shape[2], - width, - device, - ); - Ok(batch_result(tensor, images)) -} - -fn sample_width(bit_depth: u8) -> SampleWidth { - if bit_depth <= 8 { - SampleWidth::U8 - } else { - SampleWidth::U16 - } -} - -fn metal_session() -> Result { - MetalBackendSession::system_default().map_err(|error| strict(error.to_string())) -} - -fn decode_surfaces( - session: &MetalBackendSession, - inputs: &[TensorInput<'_>], - options: &TensorDecodeOptions, - width: SampleWidth, -) -> Result, TensorDecodeError> { - let first = inputs.first().ok_or(TensorDecodeError::EmptyBatch)?; - let expected_shape = - planned_shape(*first, options.channels).map_err(|error| indexed(0, error))?; - for (index, input) in inputs.iter().enumerate().skip(1) { - let actual = - planned_shape(*input, options.channels).map_err(|error| indexed(index, error))?; - if actual != expected_shape { - return Err(TensorDecodeError::BatchShapeMismatch { - index, - expected: expected_shape, - actual, - }); - } - } - let mut images: Vec = Vec::new(); - images - .try_reserve_exact(inputs.len()) - .map_err(|_| TensorDecodeError::SizeOverflow)?; - for (index, input) in inputs.iter().enumerate() { - let image = decode_surface(session, *input, options, width) - .map_err(|error| indexed(index, error))?; - if let Some(first) = images.first() { - if image.shape != first.shape { - return Err(TensorDecodeError::BatchShapeMismatch { - index, - expected: first.shape, - actual: image.shape, - }); - } - } - images.push(image); - } - Ok(images) -} - -fn decode_surface( - session: &MetalBackendSession, - input: TensorInput<'_>, - options: &TensorDecodeOptions, - width: SampleWidth, -) -> Result { - let mut decoder = - MetalDecoder::new(input.encoded).map_err(|error| strict(error.to_string()))?; - let info = decoder.inner().info(); - let plan = DeviceDecodePlan::for_image(info.dimensions, input.request)?; - let channels = selected_channels(options.channels, info.components); - let format = pixel_format(channels, width); - let request = metal_request(input.request, format); - let surface = decoder - .decode_request_to_device_with_session(request, session) - .map_err(|error| strict(error.to_string()))?; - if surface.residency() != SurfaceResidency::MetalResidentDecode { - return Err(strict(format!( - "resident decode returned unexpected residency {:?}", - surface.residency() - ))); - } - let (output_width, output_height) = plan.output_dims(); - Ok(MetalImage { - surface, - shape: [ - usize::try_from(output_height).map_err(|_| TensorDecodeError::SizeOverflow)?, - usize::try_from(output_width).map_err(|_| TensorDecodeError::SizeOverflow)?, - channels, - ], - sample_width: width, - decoded: plan.output_rect(), - warnings: vec![J2kDecodeWarning::LenientDecodeMode], - }) -} - -fn metal_request(request: DeviceDecodeRequest, format: j2k::PixelFormat) -> MetalDecodeRequest { - match request { - DeviceDecodeRequest::Full => MetalDecodeRequest::full(format, BackendRequest::Metal), - DeviceDecodeRequest::Region { roi } => { - MetalDecodeRequest::region(format, roi, BackendRequest::Metal) - } - DeviceDecodeRequest::Scaled { scale } => { - MetalDecodeRequest::scaled(format, scale, BackendRequest::Metal) - } - DeviceDecodeRequest::RegionScaled { roi, scale } => { - MetalDecodeRequest::region_scaled(format, roi, scale, BackendRequest::Metal) - } - } -} - -fn readback_batch( - session: &MetalBackendSession, - images: &[MetalImage], -) -> Result<(Vec, [usize; 3]), TensorDecodeError> { - let first = images.first().ok_or(TensorDecodeError::EmptyBatch)?; - let mut surfaces = Vec::new(); - surfaces - .try_reserve_exact(images.len()) - .map_err(|_| TensorDecodeError::SizeOverflow)?; - for image in images { - surfaces.push((&image.surface, image_byte_len(image)?)); - } - Ok((packed_readback(session, &surfaces)?, first.shape)) -} - -fn image_byte_len(image: &MetalImage) -> Result { - let expected_pitch = image.shape[1] - .checked_mul(image.shape[2]) - .and_then(|samples| samples.checked_mul(image.sample_width.bytes())) - .ok_or(TensorDecodeError::SizeOverflow)?; - if image.surface.pitch_bytes() != expected_pitch { - return Err(strict(format!( - "Metal surface pitch {} does not match compact pitch {expected_pitch}", - image.surface.pitch_bytes() - ))); - } - expected_pitch - .checked_mul(image.shape[0]) - .ok_or(TensorDecodeError::SizeOverflow) -} - -#[cfg(target_os = "macos")] -fn packed_readback( - session: &MetalBackendSession, - surfaces: &[(&Surface, usize)], -) -> Result, TensorDecodeError> { - let mut surface_refs = Vec::new(); - surface_refs - .try_reserve_exact(surfaces.len()) - .map_err(|_| TensorDecodeError::SizeOverflow)?; - surface_refs.extend(surfaces.iter().map(|(surface, _)| *surface)); - download_surfaces_packed(session, &surface_refs).map_err(|error| strict(error.to_string())) -} - -#[cfg(not(target_os = "macos"))] -fn packed_readback( - _session: &MetalBackendSession, - _surfaces: &[(&Surface, usize)], -) -> Result, TensorDecodeError> { - Err(strict("Metal is unavailable on this platform")) -} - -fn strict(message: impl Into) -> TensorDecodeError { - TensorDecodeError::StrictRoute { - route: TensorRoute::MetalStaged, - message: message.into(), - } -} - -fn indexed(index: usize, source: TensorDecodeError) -> TensorDecodeError { - TensorDecodeError::BatchItem { - index, - source: Box::new(source), - } -} - -fn single_result(tensor: T, image: MetalImage) -> TensorDecode { - TensorDecode { - tensor, - decoded: image.decoded, - warnings: image.warnings, - route: TensorRoute::MetalStaged, - } -} +mod interop; -fn batch_result(tensor: T, images: Vec) -> TensorBatchDecode { - let mut decoded = Vec::with_capacity(images.len()); - let mut warnings = Vec::with_capacity(images.len()); - for image in images { - decoded.push(image.decoded); - warnings.push(image.warnings); - } - TensorBatchDecode { - tensor, - decoded, - warnings, - route: TensorRoute::MetalStaged, - } -} +pub use batch::{MetalBurnDecoder, SubmittedMetalBurnBatch}; diff --git a/crates/j2k-ml/src/metal/batch.rs b/crates/j2k-ml/src/metal/batch.rs new file mode 100644 index 00000000..94d5511b --- /dev/null +++ b/crates/j2k-ml/src/metal/batch.rs @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use burn_core::tensor::backend::Backend; +use burn_wgpu::{Wgpu, WgpuDevice}; +use j2k::{ + BatchDecodeOptions, BatchGroupInfo, EncodedImage, IndexedBatchError, NativeSampleType, + PreparedBatch, PreparedBatchGroup, PreparedImage, +}; +use j2k_metal::MetalBatchDecoder as CodecDecoder; + +use crate::batch_contract::{dtype, tensor_shape}; +use crate::{ + BurnBatchDecode, BurnBatchGroup, BurnBatchGroupError, BurnBatchTensor, BurnDecodeError, +}; + +#[cfg(target_os = "macos")] +use super::interop::{ + fill_batch_int_tensor, paired_metal_runtime, register_int_tensor, SubmittedBatchIntTensor, +}; +#[cfg(target_os = "macos")] +use j2k_metal::SubmittedMetalGroupDecodeInto; + +/// Pending Metal decode whose registered Burn tensors remain private until +/// group-level command and codec-status validation succeeds. +#[cfg(target_os = "macos")] +#[must_use = "submitted Metal Burn batches must be waited or dropped"] +pub struct SubmittedMetalBurnBatch { + groups: Vec, + errors: Vec, + group_errors: Vec, +} + +#[cfg(target_os = "macos")] +impl core::fmt::Debug for SubmittedMetalBurnBatch { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("SubmittedMetalBurnBatch") + .field("groups", &self.groups.len()) + .field("errors", &self.errors) + .field("group_errors", &self.group_errors) + .finish_non_exhaustive() + } +} + +#[cfg(target_os = "macos")] +impl SubmittedMetalBurnBatch { + /// Number of asynchronously submitted homogeneous groups. + #[must_use] + pub fn len(&self) -> usize { + self.groups.len() + } + + /// Whether no accelerator work was submitted. + #[must_use] + pub fn is_empty(&self) -> bool { + self.groups.is_empty() + } + + /// Wait once per group, validate its batched status, and only then expose + /// ordinary rank-4 Burn tensors. + pub fn wait(self) -> Result, BurnDecodeError> { + let Self { + groups, + errors, + group_errors, + } = self; + let (groups, group_errors) = + crate::completion::finish_submitted_groups(groups, group_errors, |group| { + let source_indices = group.source_indices.clone(); + (source_indices, group.wait()) + })?; + Ok(BurnBatchDecode { + groups, + errors, + group_errors, + }) + } +} + +#[cfg(not(target_os = "macos"))] +#[derive(Debug)] +pub struct SubmittedMetalBurnBatch { + _private: (), +} + +#[cfg(not(target_os = "macos"))] +impl SubmittedMetalBurnBatch { + /// No Metal groups can be submitted on this host. + #[must_use] + pub const fn len(&self) -> usize { + 0 + } + + /// No Metal groups can be submitted on this host. + #[must_use] + pub const fn is_empty(&self) -> bool { + true + } + + /// Return the typed Metal-unavailable error. + pub fn wait(self) -> Result, BurnDecodeError> { + Err(unavailable()) + } +} + +#[cfg(target_os = "macos")] +struct SubmittedMetalBurnGroup { + sample_type: NativeSampleType, + info: BatchGroupInfo, + source_indices: Vec, + allocation: SubmittedBatchIntTensor, +} + +#[cfg(target_os = "macos")] +impl SubmittedMetalBurnGroup { + fn wait(self) -> Result, BurnDecodeError> { + let Self { + sample_type, + info, + source_indices, + allocation, + } = self; + let (cube, shape, dtype, device, pending) = allocation.into_parts(); + let (decoded_rects, warnings) = pending.wait()?.into_parts(); + let tensor = register_int_tensor(cube, shape, dtype, &device); + let tensor = match sample_type { + NativeSampleType::U8 => BurnBatchTensor::U8(tensor), + NativeSampleType::U16 => BurnBatchTensor::U16(tensor), + NativeSampleType::I16 => BurnBatchTensor::I16(tensor), + _ => return Err(BurnDecodeError::UnsupportedCodecContract), + }; + Ok(BurnBatchGroup { + tensor, + info, + source_indices, + decoded_rects, + warnings, + }) + } +} + +/// Persistent Metal codec session writing directly into Burn-owned allocations. +pub struct MetalBurnDecoder { + codec: CodecDecoder, + device: WgpuDevice, + #[cfg(target_os = "macos")] + consumer_queue: metal::CommandQueue, +} + +impl core::fmt::Debug for MetalBurnDecoder { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MetalBurnDecoder") + .field("codec", &self.codec) + .field("device", &self.device) + .field("options", &self.codec.options()) + .finish_non_exhaustive() + } +} + +impl MetalBurnDecoder { + /// Create paired J2K and Burn sessions from the exact same Metal device and + /// retain Burn's exact consumer command queue for implicit queue ordering. + #[cfg(target_os = "macos")] + pub fn system_default(options: BatchDecodeOptions) -> Result { + let (backend, device, consumer_queue) = paired_metal_runtime()?; + Ok(Self { + codec: CodecDecoder::with_backend_session_and_options(backend, options), + device, + consumer_queue, + }) + } + + /// Return a typed unavailable error on hosts without Metal. + #[cfg(not(target_os = "macos"))] + pub fn system_default(_options: BatchDecodeOptions) -> Result { + Err(unavailable()) + } + + /// Borrow the retained codec session. + #[must_use] + pub const fn codec(&self) -> &CodecDecoder { + &self.codec + } + + /// Burn wgpu device paired with the codec's underlying Metal device. + #[must_use] + pub const fn device(&self) -> &WgpuDevice { + &self.device + } + + /// Parse and group owned inputs once for repeated decode calls. + pub fn prepare(&self, inputs: Vec) -> Result { + Ok(self.codec.prepare(inputs)?) + } + + /// Regroup caller-supplied prepared images without reparsing encoded bytes. + /// + /// Source indices in the returned batch are positions in `images`; each + /// [`PreparedImage::source_index`] remains its original preparation index. + pub fn prepare_prepared_images( + &self, + images: Vec, + ) -> Result { + Ok(self.codec.prepare_prepared_images(images)?) + } + + /// Prepare and synchronously finish one owned batch. Use [`Self::submit`] + /// to overlap caller work with GPU execution. + pub fn decode( + &mut self, + inputs: Vec, + ) -> Result, BurnDecodeError> { + self.submit(inputs)?.wait() + } + + /// Finish a reusable codec preparation and return validated Burn tensors. + pub fn decode_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result, BurnDecodeError> { + self.submit_prepared(prepared)?.wait() + } + + /// Regroup, decode, and materialize caller-supplied prepared images. + pub fn decode_prepared_images( + &mut self, + images: Vec, + ) -> Result, BurnDecodeError> { + let prepared = self.prepare_prepared_images(images)?; + self.decode_prepared(&prepared) + } + + /// Prepare and asynchronously submit one owned batch without exposing + /// partially written tensor storage. + #[cfg(target_os = "macos")] + pub fn submit( + &mut self, + inputs: Vec, + ) -> Result { + let prepared = self.prepare(inputs)?; + self.submit_prepared(&prepared) + } + + /// Return a typed unavailable error on hosts without Metal. + #[cfg(not(target_os = "macos"))] + pub fn submit( + &mut self, + _inputs: Vec, + ) -> Result { + Err(unavailable()) + } + + /// Asynchronously submit a reusable preparation. Same-queue ordering is + /// implicit; a GPU event dependency is registered only when queues differ. + #[cfg(target_os = "macos")] + pub fn submit_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result { + let mut groups = Vec::new(); + groups + .try_reserve_exact(prepared.groups().len()) + .map_err(|_| BurnDecodeError::SizeOverflow)?; + let mut group_errors = Vec::new(); + group_errors + .try_reserve_exact(prepared.groups().len()) + .map_err(|_| BurnDecodeError::SizeOverflow)?; + for group in prepared.groups() { + match self.submit_group(group) { + Ok(submitted) => groups.push(submitted), + Err(source) if crate::completion::burn_group_error_is_fatal(&source) => { + return Err(source); + } + Err(source) => group_errors.push(BurnBatchGroupError::new( + group.source_indices().to_vec(), + source, + )), + } + } + Ok(SubmittedMetalBurnBatch { + groups, + errors: prepared.errors().to_vec(), + group_errors, + }) + } + + /// Return a typed unavailable error on hosts without Metal. + #[cfg(not(target_os = "macos"))] + pub fn submit_prepared( + &mut self, + _prepared: &PreparedBatch, + ) -> Result { + Err(unavailable()) + } + + #[cfg(target_os = "macos")] + fn submit_group( + &mut self, + group: &PreparedBatchGroup, + ) -> Result { + let sample_type = group.info().sample_type; + let dtype = dtype(sample_type)?; + if ! as Backend>::supports_dtype(&self.device, dtype) { + return Err(BurnDecodeError::UnsupportedDType { dtype }); + } + let shape = tensor_shape(group.source_indices().len(), group.info())?; + let device = self.device.clone(); + let consumer_queue = self.consumer_queue.clone(); + let allocation = + fill_batch_int_tensor(shape, dtype, group.info(), &device, |destination| { + Ok(self.codec.submit_prepared_group_into_for_consumer_queue( + group, + destination, + &consumer_queue, + )?) + })?; + Ok(SubmittedMetalBurnGroup { + sample_type, + info: group.info().clone(), + source_indices: group.source_indices().to_vec(), + allocation, + }) + } +} + +#[cfg(not(target_os = "macos"))] +fn unavailable() -> BurnDecodeError { + BurnDecodeError::AcceleratorInterop { + backend: "Metal", + message: "Metal is unavailable on this platform".to_string(), + } +} diff --git a/crates/j2k-ml/src/metal/interop.rs b/crates/j2k-ml/src/metal/interop.rs new file mode 100644 index 00000000..1fd133ef --- /dev/null +++ b/crates/j2k-ml/src/metal/interop.rs @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use burn_core::tensor::{DType, Int, Shape, Tensor}; +use burn_cubecl::cubecl::{wgpu::WgpuRuntime, Runtime}; +use burn_cubecl::{tensor::CubeTensor, CubeBackend}; +use burn_fusion::{get_client, stream::OperationStreams, NoOp}; +use burn_ir::{BackendIr, InitOperationIr, OperationIr}; +use burn_wgpu::{graphics, RuntimeOptions, Wgpu, WgpuDevice, WgpuSetup}; +use j2k::{BatchGroupInfo, PixelFormat}; +use j2k_metal::{MetalBackendSession, MetalImageDestination}; +use metal::foreign_types::ForeignType; + +use crate::BurnDecodeError; + +type InnerWgpu = CubeBackend; + +pub(super) fn paired_metal_runtime( +) -> Result<(MetalBackendSession, WgpuDevice, metal::CommandQueue), BurnDecodeError> { + use std::sync::OnceLock; + + // CubeCL permits one `init_device` call for each externally created setup. + // Cache the setup and its initialized logical device together so every + // decoder shares the same client, allocator, Metal device, and queue. + static DEFAULT_RUNTIME: OnceLock<(WgpuSetup, WgpuDevice)> = OnceLock::new(); + let (setup, burn) = DEFAULT_RUNTIME.get_or_init(|| { + let setup = burn_wgpu::init_setup::( + &WgpuDevice::DefaultDevice, + RuntimeOptions::default(), + ); + let burn = burn_wgpu::init_device(setup.clone(), RuntimeOptions::default()); + (setup, burn) + }); + paired_from_setup(setup, burn.clone()) +} + +fn paired_from_setup( + setup: &WgpuSetup, + burn: WgpuDevice, +) -> Result<(MetalBackendSession, WgpuDevice, metal::CommandQueue), BurnDecodeError> { + // SAFETY: setup was explicitly created with the Metal graphics API. The + // returned destruction guard is dropped while the cached setup stays live. + let hal_device = unsafe { setup.device.as_hal::() }.ok_or_else(|| { + BurnDecodeError::AcceleratorInterop { + backend: "Metal", + message: "Burn wgpu setup did not expose a Metal device".to_string(), + } + })?; + let retained = hal_device.retained_raw_handle(); + // SAFETY: the patched HAL accessor transfers one +1 Objective-C retain; + // `metal::Device` adopts exactly that retain and releases it on drop. + let metal_device = unsafe { metal::Device::from_ptr(retained.as_ptr().cast()) }; + drop(hal_device); + // SAFETY: setup was created with the Metal graphics API. The retained + // queue is the exact queue CubeCL/wgpu will use after setup is installed. + let hal_queue = unsafe { setup.queue.as_hal::() }.ok_or_else(|| { + BurnDecodeError::AcceleratorInterop { + backend: "Metal", + message: "Burn wgpu setup did not expose a Metal command queue".to_string(), + } + })?; + let retained_queue = hal_queue.retained_raw_handle(); + // SAFETY: the patched HAL accessor transfers one +1 Objective-C retain; + // `metal::CommandQueue` adopts exactly that retain and releases it on drop. + let metal_queue = unsafe { metal::CommandQueue::from_ptr(retained_queue.as_ptr().cast()) }; + drop(hal_queue); + let codec = MetalBackendSession::with_command_queue(metal_device, metal_queue.clone())?; + Ok((codec, burn, metal_queue)) +} + +/// Fresh Burn allocation kept private behind a pending codec status owner. +/// The payload is declared first so Drop retires Metal work before the `CubeCL` +/// allocation can be released. +pub(super) struct SubmittedBatchIntTensor { + payload: R, + cube: burn_cubecl::tensor::CubeTensor, + shape: Shape, + dtype: DType, + device: WgpuDevice, +} + +impl SubmittedBatchIntTensor { + pub(super) fn into_parts( + self, + ) -> ( + burn_cubecl::tensor::CubeTensor, + Shape, + DType, + WgpuDevice, + R, + ) { + let Self { + payload, + cube, + shape, + dtype, + device, + } = self; + (cube, shape, dtype, device, payload) + } +} + +pub(super) fn fill_batch_int_tensor( + shape: [usize; D], + dtype: DType, + info: &BatchGroupInfo, + device: &WgpuDevice, + submit: impl FnOnce(MetalImageDestination) -> Result, +) -> Result, BurnDecodeError> { + let logical_len = shape + .iter() + .try_fold(dtype.size(), |size, dim| size.checked_mul(*dim)) + .ok_or(BurnDecodeError::SizeOverflow)?; + let tracked_len = logical_len + .checked_next_multiple_of(4) + .ok_or(BurnDecodeError::SizeOverflow)?; + let burn_shape = Shape::from(shape.to_vec()); + let client = WgpuRuntime::client(device); + // wgpu buffer bindings and lazy initialization tracking operate on + // four-byte-sized ranges. Reserve that rounded range as part of this + // tensor's unique CubeCL allocation, while retaining the exact logical + // shape in the tensor metadata. This makes the final 1-3 padding bytes + // ours rather than borrowing bytes from a neighboring pooled allocation. + let handle = client.empty(tracked_len); + let cube = + CubeTensor::new_contiguous(client, device.clone(), burn_shape.clone(), handle, dtype); + let handle_len = + usize::try_from(cube.handle.size_in_used()).map_err(|_| BurnDecodeError::SizeOverflow)?; + if handle_len != tracked_len { + return Err(interop(format!( + "CubeCL tensor handle exposes {handle_len} bytes; expected {tracked_len} including tracker padding" + ))); + } + let resource = cube + .client + .get_resource(cube.handle.clone()) + .map_err(|error| interop(format!("CubeCL resource access failed: {error}")))?; + let raw = resource.resource(); + let available = usize::try_from(raw.size).map_err(|_| BurnDecodeError::SizeOverflow)?; + if logical_len > available { + return Err(interop(format!( + "CubeCL resource exposes {available} bytes for a {logical_len}-byte tensor" + ))); + } + let base = usize::try_from(raw.offset).map_err(|_| BurnDecodeError::SizeOverflow)?; + let initialized_range = tracked_external_write_range( + base, + logical_len, + available, + usize::try_from(raw.buffer.size()).map_err(|_| BurnDecodeError::SizeOverflow)?, + )?; + // SAFETY: the resource is a live wgpu Metal buffer. The guard prevents + // destruction while the ownership-bearing retained handle is adopted. + let hal_buffer = unsafe { raw.buffer.as_hal::() } + .ok_or_else(|| interop("Burn tensor allocation is not Metal-backed"))?; + let retained = hal_buffer.retained_raw_handle(); + // SAFETY: the patched HAL accessor transfers one +1 Objective-C retain; + // `metal::Buffer` adopts it exactly once. + let metal_buffer = unsafe { metal::Buffer::from_ptr(retained.as_ptr().cast()) }; + drop(hal_buffer); + + let pixel_format = pixel_format(info)?; + let row_bytes = usize::try_from(info.dimensions.0) + .ok() + .and_then(|width| width.checked_mul(pixel_format.bytes_per_pixel())) + .ok_or(BurnDecodeError::SizeOverflow)?; + let image_bytes = row_bytes + .checked_mul(info.dimensions.1 as usize) + .ok_or(BurnDecodeError::SizeOverflow)?; + let image_count = shape[0]; + let layout = j2k_metal::MetalImageLayout::new_batch( + base, + info.dimensions, + row_bytes, + pixel_format, + image_count, + image_bytes, + ) + .map_err(|error| interop(error.to_string()))?; + // SAFETY: `cube` has not been registered with Burn, no tensor alias has + // escaped, and the managed resource remains live through the completed + // codec submission. The destination validates the exact suballocation. + let destination = unsafe { MetalImageDestination::from_exclusive_buffer(metal_buffer, layout) } + .map_err(|error| interop(error.to_string()))?; + let output = submit(destination)?; + // SAFETY: `cube` is still private and uniquely owned. The submission + // callback has registered the codec producer dependency on Burn's exact + // consumer queue before returning, and the codec final store initializes + // every logical byte in this dense tensor subrange. CubeCL binds the final + // 1-3 bytes as inaccessible alignment padding with shader bounds checks, + // so the tracker range mirrors its four-byte-rounded binding. Without this + // handoff wgpu's first use would zero the decoded allocation. + unsafe { + raw.buffer + .mark_external_write_initialized(initialized_range) + } + .map_err(|error| interop(error.to_string()))?; + drop(resource); + Ok(SubmittedBatchIntTensor { + payload: output, + cube, + shape: burn_shape, + dtype, + device: device.clone(), + }) +} + +fn tracked_external_write_range( + base: usize, + logical_len: usize, + allocation_len: usize, + buffer_len: usize, +) -> Result, BurnDecodeError> { + if !base.is_multiple_of(4) { + return Err(interop(format!( + "CubeCL tensor suballocation offset {base} is not four-byte aligned" + ))); + } + let tracked_len = logical_len + .checked_next_multiple_of(4) + .ok_or(BurnDecodeError::SizeOverflow)?; + let end = base + .checked_add(tracked_len) + .ok_or(BurnDecodeError::SizeOverflow)?; + let allocation_end = base + .checked_add(allocation_len) + .ok_or(BurnDecodeError::SizeOverflow)?; + if end > allocation_end { + return Err(interop(format!( + "CubeCL tensor tracker range {base}..{end} exceeds its exact {base}..{allocation_end} suballocation" + ))); + } + if end > buffer_len { + return Err(interop(format!( + "CubeCL tensor tracker range {base}..{end} exceeds its {buffer_len}-byte wgpu buffer" + ))); + } + Ok( + u64::try_from(base).map_err(|_| BurnDecodeError::SizeOverflow)? + ..u64::try_from(end).map_err(|_| BurnDecodeError::SizeOverflow)?, + ) +} + +fn pixel_format(info: &BatchGroupInfo) -> Result { + info.native_pixel_format() + .ok_or(BurnDecodeError::UnsupportedCodecContract) +} + +pub(super) fn register_int_tensor( + cube: burn_cubecl::tensor::CubeTensor, + shape: Shape, + dtype: DType, + device: &WgpuDevice, +) -> Tensor { + let fusion = get_client::(device); + let handle = ::int_tensor_handle(cube); + let desc = InitOperationIr::create(shape, dtype, || fusion.register_tensor_handle(handle)); + let primitive = fusion + .register( + OperationStreams::default(), + OperationIr::Init(desc), + NoOp::::new(), + ) + .remove(0); + Tensor::::from_primitive(primitive) +} + +fn interop(message: impl Into) -> BurnDecodeError { + BurnDecodeError::AcceleratorInterop { + backend: "Metal", + message: message.into(), + } +} + +#[cfg(test)] +mod tests { + use super::tracked_external_write_range; + + #[cfg(target_os = "macos")] + #[test] + fn paired_runtime_uses_burns_exact_command_queue() { + let (codec, _device, burn_queue) = + super::paired_metal_runtime().expect("paired Metal runtime"); + assert!( + codec + .uses_command_queue(&burn_queue) + .expect("initialized codec queue identity"), + "the codec must submit on Burn's exact queue so prior and future Burn work is ordered" + ); + } + + #[test] + fn external_write_tracker_range_covers_odd_tensor_tails_without_crossing_buffer() { + assert_eq!(tracked_external_write_range(4, 5, 8, 12).unwrap(), 4..12); + assert_eq!( + tracked_external_write_range(32, 30, 32, 64).unwrap(), + 32..64 + ); + assert_eq!( + tracked_external_write_range(256, 33, 36, 292).unwrap(), + 256..292 + ); + } + + #[test] + fn external_write_tracker_range_rejects_invalid_suballocations_before_submission() { + assert!(tracked_external_write_range(2, 5, 8, 16).is_err()); + assert!(tracked_external_write_range(8, 5, 8, 15).is_err()); + assert!(tracked_external_write_range(usize::MAX - 3, 5, 8, usize::MAX).is_err()); + assert!(tracked_external_write_range(8, 5, 5, 64).is_err()); + } +} diff --git a/crates/j2k-ml/tests/batch_sessions.rs b/crates/j2k-ml/tests/batch_sessions.rs new file mode 100644 index 00000000..598873dc --- /dev/null +++ b/crates/j2k-ml/tests/batch_sessions.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#![cfg(feature = "cpu")] + +use std::sync::Arc; + +use burn_core::tensor::DType; +#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))] +use burn_flex::{Flex, FlexDevice}; +#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +use burn_ndarray::NdArrayDevice::Cpu as FlexDevice; +use j2k::{ + encode_j2k_lossless, BatchDecodeOptions, BatchLayout, EncodedImage, J2kBlockCodingMode, + J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, +}; +use j2k_ml::{BurnBatchTensor, CpuBurnDecoder}; +use j2k_test_support::{ + htj2k_gray8_fixture, openhtj2k_refinement_fixture, openhtj2k_refinement_odd_fixture, + openhtj2k_refinement_odd_pixels, openhtj2k_refinement_pixels, +}; + +#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +type Flex = burn_ndarray::NdArray; + +#[test] +fn persistent_cpu_burn_decoder_reuses_prepared_integer_batch() { + let encoded = Arc::<[u8]>::from(htj2k_gray8_fixture(4, 3)); + let options = BatchDecodeOptions { + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBurnDecoder::::new(FlexDevice, options); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(Arc::clone(&encoded)), + EncodedImage::full(encoded), + ]) + .expect("prepare reusable Burn batch"); + + let first = decoder + .decode_prepared(&prepared) + .expect("first prepared Burn decode"); + let second = decoder + .decode_prepared(&prepared) + .expect("second prepared Burn decode"); + + assert!(first.errors.is_empty()); + assert!(second.errors.is_empty()); + assert!(first.group_errors.is_empty()); + assert!(second.group_errors.is_empty()); + assert_eq!(first.groups.len(), 1); + assert_eq!(second.groups.len(), 1); + assert_eq!(first.groups[0].source_indices, [0, 1]); + assert_eq!(second.groups[0].source_indices, [0, 1]); + + let BurnBatchTensor::U8(first_tensor) = first.groups.into_iter().next().unwrap().tensor else { + panic!("expected U8 Burn tensor group") + }; + let BurnBatchTensor::U8(second_tensor) = second.groups.into_iter().next().unwrap().tensor + else { + panic!("expected U8 Burn tensor group") + }; + assert_eq!(first_tensor.dims(), [2, 1, 3, 4]); + assert_eq!(first_tensor.dtype(), DType::U8); + assert_eq!(first_tensor.into_data(), second_tensor.into_data()); +} + +#[test] +fn cpu_burn_decoder_accepts_caller_supplied_prepared_images() { + let encoded = Arc::<[u8]>::from(htj2k_gray8_fixture(4, 3)); + let options = BatchDecodeOptions { + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBurnDecoder::::new(FlexDevice, options); + let original = decoder + .prepare(vec![EncodedImage::full(encoded)]) + .expect("prepare source image"); + let image = original.groups()[0].images()[0].clone(); + + let regrouped = decoder + .prepare_prepared_images(vec![image.clone(), image.clone()]) + .expect("regroup prepared Burn inputs"); + assert_eq!(regrouped.groups()[0].source_indices(), [0, 1]); + let first = decoder + .decode_prepared(®rouped) + .expect("decode explicitly regrouped Burn inputs"); + let second = decoder + .decode_prepared_images(vec![image.clone(), image]) + .expect("regroup and decode prepared Burn inputs"); + + assert!(first.errors.is_empty()); + assert!(second.errors.is_empty()); + assert_eq!(first.groups[0].source_indices, [0, 1]); + assert_eq!(second.groups[0].source_indices, [0, 1]); + let BurnBatchTensor::U8(first_tensor) = first.groups.into_iter().next().unwrap().tensor else { + panic!("expected U8 Burn tensor group") + }; + let BurnBatchTensor::U8(second_tensor) = second.groups.into_iter().next().unwrap().tensor + else { + panic!("expected U8 Burn tensor group") + }; + assert_eq!(first_tensor.dims(), [2, 1, 3, 4]); + assert_eq!(first_tensor.into_data(), second_tensor.into_data()); +} + +#[test] +fn burn_adapter_preserves_heterogeneous_groups_and_indexed_preflight_errors() { + let first = Arc::<[u8]>::from(htj2k_gray8_fixture(4, 3)); + let second = Arc::<[u8]>::from(htj2k_gray8_fixture(2, 2)); + let inputs = vec![ + EncodedImage::full(first), + EncodedImage::full(Arc::<[u8]>::from([0_u8, 1, 2, 3])), + EncodedImage::full(second), + ]; + let mut decoder = CpuBurnDecoder::::new(FlexDevice, BatchDecodeOptions::default()); + let output = decoder.decode(inputs).expect("decode valid groups"); + + assert_eq!(output.errors.len(), 1); + assert_eq!(output.errors[0].index, 1); + assert_eq!(output.groups.len(), 2); + assert_eq!(output.groups[0].source_indices, [0]); + assert_eq!(output.groups[1].source_indices, [2]); +} + +#[test] +fn signed_i16_codec_group_stays_i16_in_burn() { + let samples = [-300_i16, -1, 0, 300]; + let encoded = signed_gray16_fixture(&samples, 2, 2); + let mut decoder = CpuBurnDecoder::::new(FlexDevice, BatchDecodeOptions::default()); + let output = decoder + .decode(vec![EncodedImage::full(Arc::<[u8]>::from(encoded))]) + .expect("decode signed Burn batch"); + + assert!(output.errors.is_empty()); + let BurnBatchTensor::I16(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected I16 Burn tensor group") + }; + assert_eq!(tensor.dtype(), DType::I16); + assert_eq!( + tensor.into_data().into_vec::().expect("i16 data"), + samples + ); +} + +#[test] +fn independent_openhtj2k_cleanup_and_refinement_samples_are_exact() { + for (encoded, expected) in [ + ( + openhtj2k_refinement_fixture(), + openhtj2k_refinement_pixels(), + ), + ( + openhtj2k_refinement_odd_fixture(), + openhtj2k_refinement_odd_pixels(), + ), + ] { + let mut decoder = CpuBurnDecoder::::new(FlexDevice, BatchDecodeOptions::default()); + let output = decoder + .decode(vec![EncodedImage::full(Arc::<[u8]>::from(encoded))]) + .expect("decode independent OpenHTJ2K fixture"); + assert!(output.errors.is_empty()); + let BurnBatchTensor::U8(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected U8 OpenHTJ2K tensor") + }; + assert_eq!( + tensor.into_data().into_vec::().expect("u8 data"), + expected, + ); + } +} + +fn signed_gray16_fixture(samples: &[i16], width: u32, height: u32) -> Vec { + let bytes = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + let samples = + J2kLosslessSamples::new(&bytes, width, height, 1, 16, true).expect("signed gray16 samples"); + encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External), + ) + .expect("encode signed gray16") + .codestream +} diff --git a/crates/j2k-ml/tests/benchmark_inputs.rs b/crates/j2k-ml/tests/benchmark_inputs.rs new file mode 100644 index 00000000..4c5519c7 --- /dev/null +++ b/crates/j2k-ml/tests/benchmark_inputs.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[path = "../benches/support/fixture.rs"] +mod fixture; +#[path = "../benches/support/input_selection.rs"] +mod input_selection; +#[path = "../benches/support/workload.rs"] +mod workload; + +use std::{collections::HashSet, sync::Arc}; + +use j2k::{BatchDecodeOptions, CpuBatchDecoder, DecodeRequest}; +use input_selection::InputMode; +use workload::{materialize_workload, workload_specs, WorkloadSpec}; + +const GENERATED_BATCH_SIZE: usize = 64; + +fn tiny_gray12() -> WorkloadSpec { + WorkloadSpec::new("tiny_gray12", 16, 16, 1, 12, false) +} + +#[test] +fn distinct_inputs_have_unique_owners_and_payloads() { + let workload = materialize_workload(tiny_gray12(), InputMode::Distinct); + assert_eq!(workload.name, "tiny_gray12"); + assert_eq!(workload.dimensions, (16, 16)); + let inputs = workload.inputs(DecodeRequest::Full, GENERATED_BATCH_SIZE); + + assert_eq!(inputs.len(), GENERATED_BATCH_SIZE); + for (index, input) in inputs.iter().enumerate() { + assert!(inputs[..index] + .iter() + .all(|previous| !Arc::ptr_eq(&previous.bytes, &input.bytes))); + } + let payloads = inputs + .iter() + .map(|input| input.bytes.as_ref()) + .collect::>(); + assert_eq!(payloads.len(), GENERATED_BATCH_SIZE); +} + +#[test] +fn repeated_inputs_share_one_owner() { + let workload = materialize_workload(tiny_gray12(), InputMode::Repeated); + let inputs = workload.inputs(DecodeRequest::Full, GENERATED_BATCH_SIZE); + + assert!(inputs + .iter() + .all(|input| Arc::ptr_eq(&inputs[0].bytes, &input.bytes))); +} + +#[test] +fn distinct_inputs_prepare_as_one_homogeneous_group() { + let workload = materialize_workload(tiny_gray12(), InputMode::Distinct); + let inputs = workload.inputs(DecodeRequest::Full, 8); + let decoder = CpuBatchDecoder::new(BatchDecodeOptions::default()); + + let prepared = decoder.prepare(inputs).expect("prepare benchmark inputs"); + + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + assert_eq!( + prepared.groups()[0].source_indices(), + &[0, 1, 2, 3, 4, 5, 6, 7] + ); +} + +#[test] +fn input_mode_defaults_to_distinct_and_rejects_unknown_values() { + assert_eq!(InputMode::parse(None).unwrap(), InputMode::Distinct); + assert_eq!( + InputMode::parse(Some("distinct")).unwrap(), + InputMode::Distinct + ); + assert_eq!( + InputMode::parse(Some("repeated")).unwrap(), + InputMode::Repeated + ); + + let error = InputMode::parse(Some("broadcast")).unwrap_err(); + assert!(error.contains("J2K_ML_BATCH_INPUT_MODE")); + assert!(error.contains("broadcast")); + + assert_eq!(InputMode::Distinct.label(), "distinct"); + assert_eq!(InputMode::Repeated.label(), "repeated"); + if let Err(error) = InputMode::from_env() { + assert!(error.contains("J2K_ML_BATCH_INPUT_MODE")); + } +} + +#[test] +fn benchmark_workload_catalog_names_are_unique() { + let workloads = workload_specs(); + assert_eq!(workloads.len(), 16); + let names = workloads + .iter() + .map(|workload| workload.name) + .collect::>(); + assert_eq!(names.len(), workloads.len()); +} diff --git a/crates/j2k-ml/tests/cpu.rs b/crates/j2k-ml/tests/cpu.rs index 4e1c0a68..ba1f519d 100644 --- a/crates/j2k-ml/tests/cpu.rs +++ b/crates/j2k-ml/tests/cpu.rs @@ -2,23 +2,19 @@ #![cfg(feature = "cpu")] -use burn_core::{ - data::dataloader::batcher::Batcher, - tensor::{backend::Backend, DType}, -}; +use std::sync::Arc; + +use burn_core::tensor::{backend::Backend, DType}; #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))] use burn_flex::{Flex, FlexDevice}; #[cfg(all(target_arch = "aarch64", target_os = "linux"))] use burn_ndarray::NdArrayDevice::Cpu as FlexDevice; use j2k::{ - encode_j2k_lossless, wrap_j2k_codestream, DeviceDecodeRequest, Downscale, J2kBlockCodingMode, - J2kEncodeValidation, J2kFileWrapOptions, J2kLosslessEncodeOptions, J2kLosslessSamples, Rect, - ReversibleTransform, -}; -use j2k_ml::{ - cpu, ChannelSelection, FloatNormalization, PanicOnDecodeError, TensorDecodeError, - TensorDecodeOptions, TensorInput, TensorLayout, + encode_j2k_lossless, wrap_j2k_codestream, BatchDecodeOptions, BatchLayout, DecodeRequest, + Downscale, EncodedImage, J2kBlockCodingMode, J2kEncodeValidation, J2kFileWrapOptions, + J2kLosslessEncodeOptions, J2kLosslessSamples, Rect, }; +use j2k_ml::{BurnBatchTensor, CpuBurnDecoder}; use j2k_test_support::{ classic_j2k_gray8_fixture, htj2k_gray8_fixture, htj2k_gray8_large_fixture, htj2k_rgb8_fixture_with_pixels, @@ -28,492 +24,202 @@ use j2k_test_support::{ type Flex = burn_ndarray::NdArray; #[test] -fn decodes_gray_u8_to_default_chw_tensor() { - let encoded = classic_j2k_gray8_fixture(4, 3); - let decoded = cpu::decode_u8::( - TensorInput::full(&encoded), - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect("decode tensor"); - - assert_eq!(decoded.tensor.dims(), [1, 3, 4]); - let data = decoded.tensor.into_data(); - assert_eq!(data.dtype, DType::U8); - assert_eq!( - data.into_vec::().expect("u8 data"), - (0..12).collect::>() - ); -} - -#[test] -fn decodes_raw_j2k_jp2_raw_htj2k_and_jph_with_exact_values() { +fn raw_j2k_jp2_raw_htj2k_and_jph_stay_exact_u8_batches() { let classic = classic_j2k_gray8_fixture(4, 3); let ht = htj2k_gray8_fixture(4, 3); let jp2 = wrap_j2k_codestream(&classic, J2kFileWrapOptions::jp2()).expect("wrap JP2"); let jph = wrap_j2k_codestream(&ht, J2kFileWrapOptions::jph()).expect("wrap JPH"); + let inputs = [classic, jp2, ht, jph] + .into_iter() + .map(|bytes| EncodedImage::full(Arc::from(bytes))) + .collect(); + let mut decoder = CpuBurnDecoder::::new(FlexDevice, BatchDecodeOptions::default()); + let output = decoder.decode(inputs).expect("decode wrapper matrix"); - for (name, encoded) in [ - ("raw J2K", classic.as_slice()), - ("JP2", jp2.as_slice()), - ("raw HTJ2K", ht.as_slice()), - ("JPH", jph.as_slice()), - ] { - let data = cpu::decode_u8::( - TensorInput::full(encoded), - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .unwrap_or_else(|error| panic!("decode {name}: {error}")) - .tensor - .into_data(); - assert_eq!(data.dtype, DType::U8, "{name}"); + assert!(output.errors.is_empty()); + assert_eq!( + output.groups.len(), + 4, + "transfer syntax remains grouping metadata" + ); + for group in output.groups { + let BurnBatchTensor::U8(tensor) = group.tensor else { + panic!("expected U8 group") + }; + assert_eq!(tensor.dims(), [1, 1, 3, 4]); assert_eq!( - data.into_vec::().expect("u8 data"), - (0..12).collect::>(), - "{name}" + tensor.into_data().into_vec::().expect("u8 data"), + (0..12).collect::>() ); } } #[test] -fn decodes_rgb_float_with_layout_and_unit_normalization() { - let (encoded, expected) = htj2k_rgb8_fixture_with_pixels(3, 2); - let options = TensorDecodeOptions { - layout: TensorLayout::ChannelsLast, - channels: ChannelSelection::Rgb, - normalization: FloatNormalization::Unit, - }; - let decoded = cpu::decode_float::(TensorInput::full(&encoded), &options, &FlexDevice) - .expect("decode tensor"); - - assert_eq!(decoded.tensor.dims(), [2, 3, 3]); - let actual = decoded - .tensor - .into_data() - .into_vec::() - .expect("f32 data"); - let expected = expected - .into_iter() - .map(|sample| f32::from(sample) / 255.0) - .collect::>(); - assert_eq!(actual, expected); -} - -#[test] -fn rejects_invalid_normalization_before_decoding_corrupt_input() { - let options = TensorDecodeOptions { - normalization: FloatNormalization::MeanStd { - mean: vec![0.0], - std: vec![0.0], - }, - ..TensorDecodeOptions::default() - }; - let error = cpu::decode_float::(TensorInput::full(b"corrupt"), &options, &FlexDevice) - .expect_err("zero std must fail first"); - assert!(matches!( - error, - TensorDecodeError::InvalidNormalization { .. } - )); -} - -#[test] -fn batch_preserves_order_and_rejects_shape_mismatch_with_index() { - let first = classic_j2k_gray8_fixture(2, 2); - let second = classic_j2k_gray8_fixture(3, 2); - let error = cpu::decode_u8_batch::( - &[ - TensorInput { - encoded: &first, - request: DeviceDecodeRequest::Full, - }, - TensorInput::full(&second), - ], - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect_err("shape mismatch"); - assert!(matches!( - error, - TensorDecodeError::BatchShapeMismatch { index: 1, .. } - )); -} - -#[test] -fn flex_supports_the_integer_dtypes_used_by_the_contract() { - assert!(Flex::supports_dtype(&FlexDevice, DType::U8)); - assert!(Flex::supports_dtype(&FlexDevice, DType::U16)); -} - -#[test] -fn decodes_u16_values_and_preserves_u16_dtype() { - let samples = [0u16, 1, 0x1234, u16::MAX, 4096, 32_768]; - let encoded = encode_gray16(&samples, 3, 2); - let decoded = cpu::decode_u16::( - TensorInput::full(&encoded), - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect("decode u16 tensor"); - - assert_eq!(decoded.tensor.dims(), [1, 2, 3]); - let data = decoded.tensor.into_data(); - assert_eq!(data.dtype, DType::U16); - assert_eq!(data.into_vec::().expect("u16 data"), samples); -} - -#[test] -fn decodes_u16_classic_htj2k_jp2_and_jph_with_exact_values() { - let samples = [0u16, 1, 0x1234, u16::MAX, 4096, 32_768]; - let classic = encode_gray16_mode(&samples, 3, 2, J2kBlockCodingMode::Classic); - let ht = encode_gray16_mode(&samples, 3, 2, J2kBlockCodingMode::HighThroughput); - let jp2 = wrap_j2k_codestream(&classic, J2kFileWrapOptions::jp2()).expect("wrap JP2"); - let jph = wrap_j2k_codestream(&ht, J2kFileWrapOptions::jph()).expect("wrap JPH"); - - for (name, encoded) in [ - ("raw J2K", classic.as_slice()), - ("JP2", jp2.as_slice()), - ("raw HTJ2K", ht.as_slice()), - ("JPH", jph.as_slice()), - ] { - let data = cpu::decode_u16::( - TensorInput::full(encoded), - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .unwrap_or_else(|error| panic!("decode {name}: {error}")) - .tensor - .into_data(); - assert_eq!(data.dtype, DType::U16, "{name}"); - assert_eq!(data.into_vec::().expect("u16 data"), samples, "{name}"); +fn classic_and_ht_wrappers_preserve_u16_values_and_dtype() { + let samples = [0_u16, 1, 0x1234, u16::MAX, 4096, 32_768]; + let classic = encode_gray16(&samples, 3, 2, J2kBlockCodingMode::Classic); + let ht = encode_gray16(&samples, 3, 2, J2kBlockCodingMode::HighThroughput); + let inputs = [ + classic.clone(), + wrap_j2k_codestream(&classic, J2kFileWrapOptions::jp2()).expect("wrap JP2"), + ht.clone(), + wrap_j2k_codestream(&ht, J2kFileWrapOptions::jph()).expect("wrap JPH"), + ] + .into_iter() + .map(|bytes| EncodedImage::full(Arc::from(bytes))) + .collect(); + let mut decoder = CpuBurnDecoder::::new(FlexDevice, BatchDecodeOptions::default()); + let output = decoder.decode(inputs).expect("decode U16 wrapper matrix"); + + assert!(output.errors.is_empty()); + for group in output.groups { + let BurnBatchTensor::U16(tensor) = group.tensor else { + panic!("expected U16 group") + }; + assert_eq!(tensor.dtype(), DType::U16); + assert_eq!( + tensor.into_data().into_vec::().expect("u16 data"), + samples + ); } } #[test] -fn channel_selection_and_nhwc_batch_shapes_follow_the_public_contract() { - let gray = classic_j2k_gray8_fixture(2, 1); - let (rgb, _) = htj2k_rgb8_fixture_with_pixels(2, 1); - let rgba = encode_rgba8(&[1, 2, 3, 4, 5, 6, 7, 8], 2, 1); - for (encoded, selection, channels) in [ - (gray.as_slice(), ChannelSelection::Auto, 1), - (gray.as_slice(), ChannelSelection::Gray, 1), - (rgb.as_slice(), ChannelSelection::Auto, 3), - (rgb.as_slice(), ChannelSelection::Rgb, 3), - (rgba.as_slice(), ChannelSelection::Auto, 3), - (rgba.as_slice(), ChannelSelection::Rgba, 4), +fn codec_layout_controls_nchw_and_nhwc_without_adapter_repacking() { + let (encoded, expected_nhwc) = htj2k_rgb8_fixture_with_pixels(3, 2); + for (layout, expected_shape) in [ + (BatchLayout::Nchw, [2, 3, 2, 3]), + (BatchLayout::Nhwc, [2, 2, 3, 3]), ] { - let options = TensorDecodeOptions { - layout: TensorLayout::ChannelsLast, - channels: selection, - ..TensorDecodeOptions::default() + let mut decoder = CpuBurnDecoder::::new( + FlexDevice, + BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }, + ); + let output = decoder + .decode(vec![ + EncodedImage::full(Arc::from(encoded.clone())), + EncodedImage::full(Arc::from(encoded.clone())), + ]) + .expect("decode RGB batch"); + let BurnBatchTensor::U8(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected RGB U8 group") }; - let single = cpu::decode_u8::(TensorInput::full(encoded), &options, &FlexDevice) - .expect("channel-selected single decode"); - assert_eq!(single.tensor.dims(), [1, 2, channels]); - - let batch = cpu::decode_u8_batch::( - &[TensorInput::full(encoded), TensorInput::full(encoded)], - &options, - &FlexDevice, - ) - .expect("channel-selected batch decode"); - assert_eq!(batch.tensor.dims(), [2, 1, 2, channels]); - } -} - -#[test] -fn u16_unit_float_uses_the_full_canonical_denominator() { - let samples = [0u16, 32_768, u16::MAX]; - let encoded = encode_gray16(&samples, 3, 1); - let actual = cpu::decode_float::( - TensorInput::full(&encoded), - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect("u16 unit float decode") - .tensor - .into_data() - .into_vec::() - .expect("f32 data"); - let expected = samples.map(|value| f32::from(value) / 65_535.0); - for (actual, expected) in actual.iter().zip(expected) { - assert!((actual - expected).abs() <= f32::EPSILON); + assert_eq!(tensor.dims(), expected_shape); + let actual = tensor.into_data().into_vec::().expect("RGB bytes"); + let expected_image = match layout { + BatchLayout::Nhwc => expected_nhwc.clone(), + BatchLayout::Nchw => (0..3) + .flat_map(|channel| expected_nhwc.iter().skip(channel).step_by(3).copied()) + .collect(), + _ => unreachable!("known layouts"), + }; + assert_eq!( + actual, + expected_image + .iter() + .chain(&expected_image) + .copied() + .collect::>() + ); } } #[test] -fn roi_scaled_decode_reports_shape_and_rectangle() { - let encoded = classic_j2k_gray8_fixture(8, 8); +fn roi_reduction_and_every_supported_reduction_report_dense_shapes() { + let encoded = Arc::<[u8]>::from(htj2k_gray8_large_fixture(64, 64)); let roi = Rect { - x: 2, - y: 2, - w: 4, - h: 4, + x: 8, + y: 12, + w: 32, + h: 24, }; - let decoded = cpu::decode_u8::( - TensorInput { - encoded: &encoded, - request: DeviceDecodeRequest::RegionScaled { - roi, + let requests = [ + (DecodeRequest::Full, [1, 1, 64, 64]), + (DecodeRequest::Region { roi }, [1, 1, 24, 32]), + ( + DecodeRequest::Reduced { scale: Downscale::Half, }, - }, - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect("decode ROI tensor"); - - assert_eq!(decoded.tensor.dims(), [1, 2, 2]); - assert_eq!(decoded.decoded, roi.scaled_covering(Downscale::Half)); -} - -#[test] -fn every_supported_power_of_two_scale_has_the_expected_shape() { - let encoded = htj2k_gray8_large_fixture(64, 64); - for (scale, side) in [ - (Downscale::None, 64), - (Downscale::Half, 32), - (Downscale::Quarter, 16), - (Downscale::Eighth, 8), - ] { - let decoded = cpu::decode_u8::( - TensorInput { - encoded: &encoded, - request: DeviceDecodeRequest::Scaled { scale }, + [1, 1, 32, 32], + ), + ( + DecodeRequest::Reduced { + scale: Downscale::Quarter, }, - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect("scaled decode"); - assert_eq!(decoded.tensor.dims(), [1, side, side], "{scale:?}"); - } -} - -#[test] -fn raw_float_casts_without_scaling() { - let encoded = classic_j2k_gray8_fixture(3, 2); - let options = TensorDecodeOptions { - normalization: FloatNormalization::Raw, - ..TensorDecodeOptions::default() - }; - let actual = cpu::decode_float::(TensorInput::full(&encoded), &options, &FlexDevice) - .expect("raw float decode") - .tensor - .into_data() - .into_vec::() - .expect("f32 data"); - assert_eq!(actual, vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0]); -} - -#[test] -fn mean_std_is_unit_scaled_and_broadcast_per_channel() { - let (encoded, pixels) = htj2k_rgb8_fixture_with_pixels(2, 1); - let options = TensorDecodeOptions { - layout: TensorLayout::ChannelsLast, - channels: ChannelSelection::Rgb, - normalization: FloatNormalization::MeanStd { - mean: vec![0.1, 0.2, 0.3], - std: vec![0.5, 0.25, 2.0], - }, - }; - let actual = cpu::decode_float::(TensorInput::full(&encoded), &options, &FlexDevice) - .expect("normalized tensor") - .tensor - .into_data() - .into_vec::() - .expect("f32 data"); - - for (index, (actual, pixel)) in actual.iter().zip(pixels).enumerate() { - let channel = index % 3; - let mean = [0.1, 0.2, 0.3][channel]; - let std = [0.5, 0.25, 2.0][channel]; - let expected = (f32::from(pixel) / 255.0 - mean) / std; - assert!((actual - expected).abs() <= 1.0e-6); - } -} - -#[test] -fn batch_uses_one_rank_four_tensor_and_preserves_input_order() { - let first = classic_j2k_gray8_fixture(2, 2); - let second = classic_j2k_gray8_fixture(2, 2); - let batch = cpu::decode_u8_batch::( - &[TensorInput::full(&first), TensorInput::full(&second)], - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect("decode batch"); - - assert_eq!(batch.tensor.dims(), [2, 1, 2, 2]); - assert_eq!( - batch.tensor.into_data().into_vec::().expect("u8 data"), - vec![0, 1, 2, 3, 0, 1, 2, 3] - ); - assert_eq!(batch.decoded.len(), 2); - assert_eq!(batch.warnings.len(), 2); -} - -#[test] -fn u16_batch_preserves_values_dtype_and_order() { - let first_samples = [0u16, 1, 4096, u16::MAX]; - let second_samples = [32_768u16, 17, 255, 1024]; - let first = encode_gray16(&first_samples, 2, 2); - let second = encode_gray16(&second_samples, 2, 2); - let batch = cpu::decode_u16_batch::( - &[TensorInput::full(&first), TensorInput::full(&second)], - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect("decode u16 batch"); - - assert_eq!(batch.tensor.dims(), [2, 1, 2, 2]); - let data = batch.tensor.into_data(); - assert_eq!(data.dtype, DType::U16); - assert_eq!( - data.into_vec::().expect("u16 data"), - first_samples - .into_iter() - .chain(second_samples) - .collect::>() - ); -} - -#[test] -fn float_batch_mean_std_broadcasts_in_both_layouts() { - let (encoded, pixels) = htj2k_rgb8_fixture_with_pixels(2, 1); - let mean = [0.1f32, 0.2, 0.3]; - let std = [0.5f32, 0.25, 2.0]; - - for layout in [TensorLayout::ChannelsFirst, TensorLayout::ChannelsLast] { - let options = TensorDecodeOptions { - layout, - channels: ChannelSelection::Rgb, - normalization: FloatNormalization::MeanStd { - mean: mean.to_vec(), - std: std.to_vec(), + [1, 1, 16, 16], + ), + ( + DecodeRequest::Reduced { + scale: Downscale::Eighth, }, - }; - let batch = cpu::decode_float_batch::( - &[TensorInput::full(&encoded), TensorInput::full(&encoded)], - &options, - &FlexDevice, - ) - .expect("decode normalized float batch"); - - assert_eq!( - batch.tensor.dims(), - match layout { - TensorLayout::ChannelsFirst => [2, 3, 1, 2], - TensorLayout::ChannelsLast => [2, 1, 2, 3], - } - ); - let actual = batch + [1, 1, 8, 8], + ), + ( + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + [1, 1, 12, 16], + ), + ]; + for (request, expected_shape) in requests { + let mut decoder = CpuBurnDecoder::::new(FlexDevice, BatchDecodeOptions::default()); + let output = decoder + .decode(vec![EncodedImage::new(Arc::clone(&encoded), request)]) + .expect("decode requested geometry"); + let tensor = output + .groups + .into_iter() + .next() + .unwrap() .tensor - .into_data() - .into_vec::() - .expect("f32 data"); - let expected_image = match layout { - TensorLayout::ChannelsFirst => (0..3) - .flat_map(|channel| { - pixels.iter().skip(channel).step_by(3).map(move |pixel| { - (f32::from(*pixel) / 255.0 - mean[channel]) / std[channel] - }) - }) - .collect::>(), - TensorLayout::ChannelsLast => pixels - .iter() - .enumerate() - .map(|(index, pixel)| { - let channel = index % 3; - (f32::from(*pixel) / 255.0 - mean[channel]) / std[channel] - }) - .collect::>(), - }; - let expected = expected_image - .iter() - .copied() - .chain(expected_image.iter().copied()) - .collect::>(); - assert_eq!(actual.len(), expected.len()); - for (actual, expected) in actual.iter().zip(expected) { - assert!((actual - expected).abs() <= 1.0e-6); - } + .into_tensor(); + assert_eq!(tensor.dims(), expected_shape, "{request:?}"); } } #[test] -fn empty_and_corrupt_batches_are_explicit_and_indexed() { - let empty = cpu::decode_u8_batch::(&[], &TensorDecodeOptions::default(), &FlexDevice) - .expect_err("empty batch"); - assert!(matches!(empty, TensorDecodeError::EmptyBatch)); +fn empty_batch_is_a_success_and_corrupt_inputs_are_indexed() { + let mut decoder = CpuBurnDecoder::::new(FlexDevice, BatchDecodeOptions::default()); + let empty = decoder.decode(Vec::new()).expect("empty batch"); + assert!(empty.groups.is_empty()); + assert!(empty.errors.is_empty()); - let valid = classic_j2k_gray8_fixture(2, 2); - let corrupt = cpu::decode_u8_batch::( - &[TensorInput::full(&valid), TensorInput::full(b"corrupt")], - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect_err("corrupt batch item"); - assert!(matches!( - corrupt, - TensorDecodeError::BatchItem { index: 1, .. } - )); + let valid = Arc::<[u8]>::from(classic_j2k_gray8_fixture(2, 2)); + let output = decoder + .decode(vec![ + EncodedImage::full(valid), + EncodedImage::full(Arc::from(*b"corrupt")), + ]) + .expect("valid groups survive indexed preflight errors"); + assert_eq!(output.groups.len(), 1); + assert_eq!(output.errors.len(), 1); + assert_eq!(output.errors[0].index, 1); } #[test] -fn panic_batcher_includes_actionable_item_context() { - let valid = classic_j2k_gray8_fixture(2, 2); - let batcher = PanicOnDecodeError::::new(TensorDecodeOptions::default()); - let panic = std::panic::catch_unwind(|| { - let _ = batcher.batch( - vec![TensorInput::full(&valid), TensorInput::full(b"corrupt")], - &FlexDevice, - ); - }) - .expect_err("adapter must panic"); - let message = panic - .downcast_ref::() - .map(String::as_str) - .or_else(|| panic.downcast_ref::<&str>().copied()) - .expect("string panic"); - assert!(message.contains("batch item 1"), "{message}"); - assert!(message.contains("decode failed"), "{message}"); -} - -fn encode_gray16(samples: &[u16], width: u32, height: u32) -> Vec { - encode_gray16_mode(samples, width, height, J2kBlockCodingMode::Classic) +fn portable_test_backend_supports_all_fast_batch_integer_dtypes() { + assert!(Flex::supports_dtype(&FlexDevice, DType::U8)); + assert!(Flex::supports_dtype(&FlexDevice, DType::U16)); + assert!(Flex::supports_dtype(&FlexDevice, DType::I16)); } -fn encode_gray16_mode( - samples: &[u16], - width: u32, - height: u32, - mode: J2kBlockCodingMode, -) -> Vec { - let mut bytes = Vec::with_capacity(samples.len() * 2); - for sample in samples { - bytes.extend_from_slice(&sample.to_le_bytes()); - } +fn encode_gray16(samples: &[u16], width: u32, height: u32, mode: J2kBlockCodingMode) -> Vec { + let bytes = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); let samples = - J2kLosslessSamples::new(&bytes, width, height, 1, 16, false).expect("valid gray16 samples"); + J2kLosslessSamples::new(&bytes, width, height, 1, 16, false).expect("valid Gray16"); encode_j2k_lossless( samples, &J2kLosslessEncodeOptions::default() .with_block_coding_mode(mode) .with_validation(J2kEncodeValidation::External), ) - .expect("encode gray16") - .codestream -} - -fn encode_rgba8(samples: &[u8], width: u32, height: u32) -> Vec { - let samples = - J2kLosslessSamples::new(samples, width, height, 4, 8, false).expect("valid rgba8 samples"); - encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions::default() - .with_reversible_transform(ReversibleTransform::None53) - .with_validation(J2kEncodeValidation::External), - ) - .expect("encode rgba8") + .expect("encode Gray16") .codestream } diff --git a/crates/j2k-ml/tests/cuda.rs b/crates/j2k-ml/tests/cuda.rs index bc8b25fa..ab68471d 100644 --- a/crates/j2k-ml/tests/cuda.rs +++ b/crates/j2k-ml/tests/cuda.rs @@ -2,263 +2,366 @@ #![cfg(all(feature = "cuda", not(target_os = "macos")))] -use burn_autodiff::Autodiff; -use burn_core::tensor::{DType, Shape, Tensor}; -use burn_cubecl::{ - cubecl::{cuda::CudaRuntime, Runtime}, - ops::numeric::empty_device_contiguous_dtype, -}; -use burn_cuda::{Cuda, CudaDevice}; -#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))] -use burn_flex::{Flex, FlexDevice}; -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] -use burn_ndarray::NdArrayDevice::Cpu as FlexDevice; -use j2k::{DeviceDecodeRequest, Downscale, Rect}; -use j2k_cuda_runtime::{CudaContext, CudaExternalDeviceBufferViewMut}; -use j2k_ml::{ - cpu, cuda, FloatNormalization, TensorDecodeError, TensorDecodeOptions, TensorInput, - TensorLayout, TensorRoute, +use std::sync::Arc; + +use burn_core::tensor::DType; +use burn_cuda::CudaDevice; +use j2k::{ + encode_j2k_lossless, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, + DecodeRequest, Downscale, EncodedImage, J2kBlockCodingMode, J2kEncodeValidation, + J2kLosslessEncodeOptions, J2kLosslessSamples, Rect, }; +use j2k_ml::{BurnBatchTensor, CudaBurnDecoder}; use j2k_test_support::{ - cuda_runtime_and_strict_oxide_gate, htj2k_gray8_fixture, htj2k_gray8_large_fixture, - openhtj2k_refinement_fixture, + cuda_runtime_and_strict_oxide_gate, htj2k_gray8_large_fixture, OpenJphBatchFixture, }; -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] -type Flex = burn_ndarray::NdArray; - #[test] -fn direct_cuda_decode_reports_route_and_exact_pixels() { - if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA direct") { +fn direct_cuda_batch_writes_exact_u8_pixels_and_reuses_the_session() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA direct batch") { return; } - let encoded = htj2k_gray8_fixture(4, 3); - let decoded = cuda::decode_u8( - TensorInput::full(&encoded), - &TensorDecodeOptions::default(), - &CudaDevice::default(), - ) - .expect("CUDA direct tensor decode"); - assert_eq!(decoded.route, TensorRoute::CudaDirect); - assert_eq!(decoded.tensor.dims(), [1, 3, 4]); - assert_eq!( - decoded - .tensor - .into_data() - .into_vec::() - .expect("u8 data"), - (0..12).collect::>() - ); + let encoded = Arc::<[u8]>::from(htj2k_gray8_large_fixture(8, 8)); + let mut decoder = CudaBurnDecoder::new(CudaDevice::default(), BatchDecodeOptions::default()) + .expect("create persistent CUDA adapter"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(Arc::clone(&encoded)), + EncodedImage::full(encoded), + ]) + .expect("prepare CUDA batch"); + + for _ in 0..2 { + let burn_batch = decoder + .decode_prepared(&prepared) + .expect("submit prepared CUDA batch"); + assert!(burn_batch.errors.is_empty()); + let BurnBatchTensor::U8(tensor) = burn_batch.groups.into_iter().next().unwrap().tensor + else { + panic!("expected U8 tensor") + }; + assert_eq!(tensor.dims(), [2, 1, 8, 8]); + let values = tensor.into_data().into_vec::().expect("CUDA U8 data"); + assert_eq!(&values[..64], &values[64..]); + } + assert!(decoder.codec().session().submissions() >= 2); } #[test] -fn cubecl_stream_ordered_allocation_is_accessible_to_j2k_primary_context() { - if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA CubeCL allocation interop") { +fn direct_cuda_preserves_native_u16_and_i16_samples() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA native integer batches") { return; } - let device = CudaDevice::default(); - let context = CudaContext::retain_primary(device.index).expect("retain primary context"); - let client = CudaRuntime::client(&device); - let cube = - empty_device_contiguous_dtype(client, device.clone(), Shape::from(vec![16]), DType::U8); - burn_cubecl::cubecl::future::block_on(cube.client.sync()) - .expect("complete CubeCL stream-ordered allocation"); - let mut resource = cube - .client - .get_resource(cube.handle.clone()) - .expect("CubeCL managed resource"); - let raw = resource.resource(); - - // SAFETY: `resource` is CubeCL's exclusive managed-resource guard for the - // live allocation, whose stream-ordered creation completed above. - let view = unsafe { - CudaExternalDeviceBufferViewMut::from_raw_parts( - &context, - raw.ptr, - 16, - DType::U8.size(), - &mut resource, - ) + let unsigned = [0_u16, 1, 2048, 4095]; + let signed = [-2048_i16, -1, 0, 2047]; + let cases = [ + ( + encode_gray(&unsigned, 12, false), + DType::U16, + unsigned + .iter() + .map(|value| i32::from(*value)) + .collect::>(), + ), + ( + encode_gray(&signed, 12, true), + DType::I16, + signed + .iter() + .map(|value| i32::from(*value)) + .collect::>(), + ), + ]; + for (encoded, dtype, expected) in cases { + let mut decoder = + CudaBurnDecoder::new(CudaDevice::default(), BatchDecodeOptions::default()) + .expect("create CUDA adapter"); + let burn_batch = decoder + .decode(vec![EncodedImage::full(Arc::from(encoded))]) + .expect("decode native CUDA type"); + let tensor = burn_batch + .groups + .into_iter() + .next() + .unwrap() + .tensor + .into_tensor(); + assert_eq!(tensor.dtype(), dtype); + let data = tensor.into_data(); + let actual = match dtype { + DType::U16 => data + .into_vec::() + .expect("U16 data") + .into_iter() + .map(i32::from) + .collect::>(), + DType::I16 => data + .into_vec::() + .expect("I16 data") + .into_iter() + .map(i32::from) + .collect::>(), + _ => unreachable!("test only covers U16/I16"), + }; + assert_eq!(actual, expected); } - .expect("adopt CubeCL stream-ordered allocation"); - assert_eq!(view.byte_len(), 16); - assert_eq!(view.context().device_ordinal(), device.index); } #[test] -fn direct_cuda_u16_matches_portable_and_batches() { - if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA u16 tensor parity") { +fn direct_cuda_supports_roi_and_reduction_without_host_staging() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA ROI reduction") { return; } - let device = CudaDevice::default(); - - let encoded_u16 = openhtj2k_refinement_fixture(); - let expected_u16 = cpu::decode_u16::( - TensorInput::full(encoded_u16), - &TensorDecodeOptions::default(), - &FlexDevice, - ) - .expect("portable u16 decode") - .tensor - .into_data() - .into_vec::() - .expect("portable u16 data"); - let actual_u16 = cuda::decode_u16( - TensorInput::full(encoded_u16), - &TensorDecodeOptions::default(), - &device, - ) - .expect("direct u16 decode") - .tensor - .into_data() - .into_vec::() - .expect("direct u16 data"); - assert_eq!(actual_u16, expected_u16); - - let u16_batch = cuda::decode_u16_batch( - &[ - TensorInput::full(encoded_u16), - TensorInput::full(encoded_u16), - ], - &TensorDecodeOptions::default(), - &device, - ) - .expect("direct u16 batch"); - assert_eq!(u16_batch.tensor.dims()[0], 2); - let expected_u16_batch = expected_u16 - .iter() - .chain(&expected_u16) - .copied() - .collect::>(); - assert_eq!( - u16_batch - .tensor - .into_data() - .into_vec::() - .expect("direct u16 batch data"), - expected_u16_batch - ); + let encoded = Arc::<[u8]>::from(htj2k_gray8_large_fixture(64, 64)); + let roi = Rect { + x: 8, + y: 12, + w: 32, + h: 24, + }; + let mut decoder = CudaBurnDecoder::new(CudaDevice::default(), BatchDecodeOptions::default()) + .expect("create CUDA adapter"); + let burn_batch = decoder + .decode(vec![EncodedImage::new( + encoded, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + )]) + .expect("decode CUDA ROI reduction"); + let tensor = burn_batch + .groups + .into_iter() + .next() + .unwrap() + .tensor + .into_tensor(); + assert_eq!(tensor.dims(), [1, 1, 12, 16]); } #[test] -fn direct_cuda_float_matches_portable_full_batch_and_roi() { - if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA float tensor parity") { +fn direct_cuda_rgb_preserves_subnative_codes_and_burn_layout() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA exact RGB batches") { return; } - let device = CudaDevice::default(); - let encoded = htj2k_gray8_fixture(4, 3); - for layout in [TensorLayout::ChannelsFirst, TensorLayout::ChannelsLast] { - for normalization in [ - FloatNormalization::Raw, - FloatNormalization::Unit, - FloatNormalization::MeanStd { - mean: vec![0.25], - std: vec![0.5], - }, - ] { - let options = TensorDecodeOptions { - layout, - normalization, - ..TensorDecodeOptions::default() + let rgb7 = (0_u16..4 * 4 * 3) + .map(|value| ((value * 29 + 7) & 0x7f) as u8) + .collect::>(); + let rgb12 = (0_u32..4 * 4 * 3) + .map(|value| ((value * 977 + 31) & 0x0fff) as u16) + .collect::>(); + let cases = [ + (encode_rgb(&rgb7, 7), DType::U8), + (encode_rgb(&rgb12, 12), DType::U16), + ]; + + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let mut decoder = CudaBurnDecoder::new(CudaDevice::default(), options) + .expect("create exact RGB CUDA adapter"); + let mut cpu = CpuBatchDecoder::new(options); + for (encoded, expected_dtype) in &cases { + let prepared = decoder + .prepare(vec![EncodedImage::full(Arc::from(encoded.clone()))]) + .expect("prepare exact RGB Burn batch"); + let oracle = cpu + .decode_prepared(&prepared) + .expect("decode exact RGB CPU oracle"); + let expected = match oracle.groups()[0].samples() { + CpuBatchSamples::U8(samples) => { + samples.iter().copied().map(u16::from).collect::>() + } + CpuBatchSamples::U16(samples) => samples.clone(), + other => panic!("unexpected exact RGB oracle type: {other:?}"), }; - let expected = - cpu::decode_float::(TensorInput::full(&encoded), &options, &FlexDevice) - .expect("portable float decode") - .tensor - .into_data() - .into_vec::() - .expect("portable float data"); - let actual = cuda::decode_float(TensorInput::full(&encoded), &options, &device) - .expect("direct float decode") - .tensor - .into_data() - .into_vec::() - .expect("direct float data"); - for (actual, expected) in actual.iter().zip(expected) { - assert!((actual - expected).abs() <= 1.0e-6); - } + + let burn_batch = decoder + .decode_prepared(&prepared) + .expect("decode exact RGB directly into Burn storage"); + let group = burn_batch + .groups + .into_iter() + .next() + .expect("Burn RGB group"); + let tensor = group.tensor.into_tensor(); + assert_eq!(tensor.dtype(), *expected_dtype); + assert_eq!( + tensor.dims(), + match layout { + BatchLayout::Nhwc => [1, 4, 4, 3], + BatchLayout::Nchw => [1, 3, 4, 4], + _ => unreachable!("test only covers public dense layouts"), + } + ); + let data = tensor.into_data(); + let actual = match expected_dtype { + DType::U8 => data + .into_vec::() + .expect("exact RGB U8 tensor data") + .into_iter() + .map(u16::from) + .collect::>(), + DType::U16 => data.into_vec::().expect("exact RGB U16 tensor data"), + _ => unreachable!("test only covers unsigned exact RGB tensors"), + }; + assert_eq!(actual, expected, "{layout:?} {expected_dtype:?}"); } } +} - let options = TensorDecodeOptions { - layout: TensorLayout::ChannelsLast, - normalization: FloatNormalization::MeanStd { - mean: vec![0.25], - std: vec![0.5], +#[test] +fn direct_cuda_signed_rgb_matches_cpu_for_geometry_and_burn_layout() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA signed RGB batches") { + return; + } + let fixtures = j2k_test_support::openjph_batch_fixtures() + .iter() + .filter(|fixture| { + matches!( + fixture.name, + "openjph-rgb-s8-53-single-raw" + | "openjph-rgb-s12-53-single-raw" + | "openjph-rgb-s16-53-single-raw" + ) + }) + .collect::>(); + assert_eq!(fixtures.len(), 3); + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { + roi: Rect { + x: 2, + y: 3, + w: 9, + h: 7, + }, }, - ..TensorDecodeOptions::default() - }; - let batch = cuda::decode_float_batch( - &[TensorInput::full(&encoded), TensorInput::full(&encoded)], - &options, - &device, - ) - .expect("direct float batch"); - assert_eq!(batch.tensor.dims(), [2, 3, 4, 1]); - - let large = htj2k_gray8_large_fixture(64, 64); - let roi_input = TensorInput { - encoded: &large, - request: DeviceDecodeRequest::RegionScaled { + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { roi: Rect { - x: 8, - y: 8, - w: 32, - h: 32, + x: 2, + y: 4, + w: 10, + h: 8, }, scale: Downscale::Half, }, - }; - let expected_roi = cpu::decode_float::(roi_input, &options, &FlexDevice) - .expect("portable ROI decode") - .tensor - .into_data() - .into_vec::() - .expect("portable ROI data"); - let direct_roi = cuda::decode_float(roi_input, &options, &device).expect("direct ROI decode"); - assert_eq!(direct_roi.tensor.dims(), [16, 16, 1]); - for (actual, expected) in direct_roi - .tensor - .into_data() - .into_vec::() - .expect("direct ROI data") - .iter() - .zip(expected_roi) - { - assert!((actual - expected).abs() <= 1.0e-6); + ]; + + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let mut decoder = CudaBurnDecoder::new(CudaDevice::default(), options) + .expect("create signed RGB CUDA adapter"); + let mut cpu = CpuBatchDecoder::new(options); + for fixture in &fixtures { + let encoded = Arc::<[u8]>::from(fixture.encoded); + for request in requests { + let prepared = decoder + .prepare(vec![EncodedImage::new(Arc::clone(&encoded), request)]) + .unwrap_or_else(|error| panic!("{} prepare: {error}", fixture.name)); + let oracle = cpu + .decode_prepared(&prepared) + .unwrap_or_else(|error| panic!("{} CPU oracle: {error}", fixture.name)); + let expected = match oracle.groups()[0].samples() { + CpuBatchSamples::I16(samples) => samples.clone(), + other => panic!( + "{}: unexpected signed RGB oracle type: {other:?}", + fixture.name + ), + }; + if layout == BatchLayout::Nhwc && request == DecodeRequest::Full { + assert_eq!( + expected, + openjph_i16_oracle(fixture), + "{} independent OpenJPH oracle", + fixture.name + ); + } + let dimensions = prepared.groups()[0].info().dimensions; + + let burn_batch = decoder + .decode_prepared(&prepared) + .unwrap_or_else(|error| panic!("{} Burn decode: {error}", fixture.name)); + let group = burn_batch + .groups + .into_iter() + .next() + .expect("Burn signed RGB group"); + let tensor = group.tensor.into_tensor(); + assert_eq!(tensor.dtype(), DType::I16); + assert_eq!( + tensor.dims(), + match layout { + BatchLayout::Nhwc => [1, dimensions.1 as usize, dimensions.0 as usize, 3,], + BatchLayout::Nchw => [1, 3, dimensions.1 as usize, dimensions.0 as usize,], + _ => unreachable!("test only covers public dense layouts"), + } + ); + let actual = tensor + .into_data() + .into_vec::() + .expect("signed RGB I16 tensor data"); + assert_eq!(actual, expected, "{} {layout:?} {request:?}", fixture.name); + } + } } } -#[test] -fn direct_cuda_reports_batch_mismatch_and_lifts_to_autodiff() { - if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA tensor contracts") { - return; +fn openjph_i16_oracle(fixture: &OpenJphBatchFixture) -> Vec { + if fixture.precision <= 8 { + fixture + .oracle + .iter() + .map(|sample| i16::from(i8::from_ne_bytes([*sample]))) + .collect() + } else { + fixture + .oracle + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]])) + .collect() } - let device = CudaDevice::default(); - let encoded = htj2k_gray8_fixture(4, 3); - let options = TensorDecodeOptions { - layout: TensorLayout::ChannelsLast, - normalization: FloatNormalization::MeanStd { - mean: vec![0.25], - std: vec![0.5], - }, - ..TensorDecodeOptions::default() - }; - let mismatch = htj2k_gray8_fixture(5, 3); - let error = cuda::decode_u8_batch( - &[TensorInput::full(&encoded), TensorInput::full(&mismatch)], - &TensorDecodeOptions::default(), - &device, +} + +fn encode_gray(samples: &[T], precision: u8, signed: bool) -> Vec { + let byte_len = std::mem::size_of_val(samples); + // SAFETY: plain integer fixtures are copied immediately into the encoder; + // their native little-endian representation is the codec's input format. + let bytes = unsafe { std::slice::from_raw_parts(samples.as_ptr().cast::(), byte_len) }; + let samples = J2kLosslessSamples::new(bytes, 2, 2, 1, precision, signed) + .expect("valid native integer samples"); + encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External), ) - .expect_err("direct batch shape mismatch"); - assert!(matches!( - error, - TensorDecodeError::BatchShapeMismatch { index: 1, .. } - )); + .expect("encode native integer fixture") + .codestream +} - let inner = cuda::decode_float(TensorInput::full(&encoded), &options, &device) - .expect("direct float for autodiff") - .tensor; - let autodiff = Tensor::, 3>::from_inner(inner).require_grad(); - assert_eq!(autodiff.dims(), [3, 4, 1]); +fn encode_rgb(samples: &[T], precision: u8) -> Vec { + let byte_len = std::mem::size_of_val(samples); + // SAFETY: plain integer fixtures are copied immediately into the encoder; + // their native little-endian representation is the codec's input format. + let bytes = unsafe { std::slice::from_raw_parts(samples.as_ptr().cast::(), byte_len) }; + let samples = J2kLosslessSamples::new(bytes, 4, 4, 3, precision, false) + .expect("valid RGB native integer samples"); + encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External), + ) + .expect("encode RGB native integer fixture") + .codestream } diff --git a/crates/j2k-ml/tests/cuda_batch_sessions.rs b/crates/j2k-ml/tests/cuda_batch_sessions.rs new file mode 100644 index 00000000..f7704c1c --- /dev/null +++ b/crates/j2k-ml/tests/cuda_batch_sessions.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#![cfg(feature = "cuda")] + +use std::sync::Arc; + +use burn_cuda::CudaDevice; +use j2k::{prepare_batch, BatchDecodeOptions, BatchItemError, DecodeSettings, EncodedImage}; +use j2k_ml::{BurnBatchTensor, BurnDecodeError, CudaBurnDecoder}; +use j2k_native::{encode, EncodeOptions}; +use j2k_test_support::{cuda_runtime_and_strict_oxide_gate, htj2k_gray8_large_fixture}; + +fn unsupported_classic_roi_rgb() -> Arc<[u8]> { + let pixels = (0..4_u8) + .flat_map(|index| [index * 17, index * 29 + 3, index * 41 + 5]) + .collect::>(); + Arc::from( + encode( + &pixels, + 2, + 2, + 3, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + roi_component_shifts: vec![3, 0, 0], + ..EncodeOptions::default() + }, + ) + .expect("encode classic RGB8 with unsupported RGN maxshift"), + ) +} + +#[test] +fn empty_cuda_batch_uses_the_persistent_shared_codec_contract_without_initializing_work() { + let mut decoder = CudaBurnDecoder::new(CudaDevice::new(0), BatchDecodeOptions::default()) + .expect("retain the Burn CUDA primary context"); + + let prepared = decoder + .prepare(Vec::::new()) + .expect("prepare empty shared batch"); + let submitted = decoder + .submit_prepared(&prepared) + .expect("submit empty shared batch"); + assert!(submitted.is_empty()); + let output = submitted.wait().expect("finish empty shared batch"); + + assert!(output.groups.is_empty()); + assert!(output.errors.is_empty()); + assert_eq!(decoder.codec().session().submissions(), 0); +} + +#[test] +fn cuda_burn_regroups_prepared_images_and_keeps_settings_failures_indexed_without_cuda() { + let lenient_options = BatchDecodeOptions { + settings: DecodeSettings::lenient(), + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from( + j2k_test_support::htj2k_gray8_large_fixture(4, 4), + ))], + lenient_options, + ) + .expect("prepare lenient CUDA Burn input"); + let image = prepared.groups()[0].images()[0].clone(); + let mut decoder = CudaBurnDecoder::new(CudaDevice::new(0), BatchDecodeOptions::default()) + .expect("create lazy CUDA Burn decoder"); + + let regrouped = decoder + .prepare_prepared_images(vec![image.clone()]) + .expect("settings mismatch remains indexed preflight data"); + assert!(regrouped.groups().is_empty()); + assert_eq!(regrouped.errors()[0].index, 0); + assert!(matches!( + regrouped.errors()[0].source, + BatchItemError::PreparedDecodeSettingsMismatch { + prepared, + requested, + } if prepared == DecodeSettings::lenient() && requested == DecodeSettings::strict() + )); + + let output = decoder + .decode_prepared_images(vec![image]) + .expect("preflight-only batch must not initialize CUDA"); + assert!(output.groups.is_empty()); + assert_eq!(output.errors[0].index, 0); + assert_eq!(decoder.codec().session().submissions(), 0); +} + +#[test] +fn dropping_submitted_burn_batch_retires_cuda_work_and_keeps_session_reusable() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA submitted-batch drop reuse") { + return; + } + let encoded = Arc::<[u8]>::from(htj2k_gray8_large_fixture(8, 8)); + let mut decoder = CudaBurnDecoder::new(CudaDevice::default(), BatchDecodeOptions::default()) + .expect("create persistent CUDA Burn decoder"); + + let submitted = decoder + .submit(vec![EncodedImage::full(Arc::clone(&encoded))]) + .expect("submit CUDA Burn batch to drop"); + assert_eq!(submitted.len(), 1); + drop(submitted); + + let output = decoder + .decode(vec![EncodedImage::full(encoded)]) + .expect("reuse CUDA Burn decoder after dropped submission"); + assert!(output.errors.is_empty()); + let [group] = output.groups.as_slice() else { + panic!("expected one decoded CUDA Burn group") + }; + let BurnBatchTensor::U8(tensor) = &group.tensor else { + panic!("expected native U8 CUDA Burn tensor") + }; + assert_eq!(tensor.dims(), [1, 1, 8, 8]); + assert!(decoder.codec().session().submissions() >= 2); +} + +#[test] +fn burn_direct_session_reuses_events_and_codec_memory_for_one_thousand_batches() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA direct-write session soak") { + return; + } + let encoded = Arc::<[u8]>::from(htj2k_gray8_large_fixture(8, 8)); + let mut decoder = CudaBurnDecoder::new(CudaDevice::default(), BatchDecodeOptions::default()) + .expect("create persistent CUDA Burn decoder"); + let prepared = decoder + .prepare(vec![EncodedImage::full(encoded)]) + .expect("prepare reusable CUDA Burn batch"); + + for _ in 0..16 { + drop( + decoder + .decode_prepared(&prepared) + .expect("warm CUDA Burn direct-write session"), + ); + } + let warm = decoder + .codec() + .diagnostics() + .expect("warm CUDA Burn codec diagnostics"); + let warm_runtime = warm.runtime.expect("CUDA Burn runtime is initialized"); + assert!(warm.pools.retained_bytes() > 0); + + for _ in 0..1_000 { + drop( + decoder + .decode_prepared(&prepared) + .expect("reuse CUDA Burn direct-write session"), + ); + } + + let after = decoder + .codec() + .diagnostics() + .expect("post-soak CUDA Burn codec diagnostics"); + let after_runtime = after + .runtime + .expect("CUDA Burn runtime remains initialized"); + assert_eq!(after.pools.retained_bytes(), warm.pools.retained_bytes()); + assert_eq!( + after.pools.peak_retained_bytes_upper_bound(), + warm.pools.peak_retained_bytes_upper_bound() + ); + assert_eq!( + after_runtime.status_device_to_host_operations + - warm_runtime.status_device_to_host_operations, + 1_000, + "each Burn-direct group must use one status readback" + ); + assert_eq!( + after_runtime.device_to_host_operations - warm_runtime.device_to_host_operations, + 1_000, + "Burn-direct decode must not download decoded pixels" + ); + assert_eq!( + after_runtime.device_to_host_bytes - warm_runtime.device_to_host_bytes, + after_runtime.status_device_to_host_bytes - warm_runtime.status_device_to_host_bytes + ); + assert_eq!( + after_runtime.event_driver_allocations, warm_runtime.event_driver_allocations, + "Burn stream-bridge events must stabilize after warmup" + ); + assert!(after_runtime.event_reuses > warm_runtime.event_reuses); + assert_eq!( + after_runtime.event_host_synchronizations, warm_runtime.event_host_synchronizations, + "Burn-direct completion must use the group status boundary" + ); + assert_eq!( + after_runtime.context_host_synchronizations, warm_runtime.context_host_synchronizations, + "Burn-direct completion must not synchronize the whole context" + ); +} + +#[test] +fn cuda_burn_batch_continues_after_one_group_submit_failure() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA group submit continuation") { + return; + } + let valid_gray = Arc::<[u8]>::from(htj2k_gray8_large_fixture(8, 8)); + let mut decoder = CudaBurnDecoder::new(CudaDevice::default(), BatchDecodeOptions::default()) + .expect("create persistent CUDA Burn decoder"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(unsupported_classic_roi_rgb()), + EncodedImage::full(valid_gray), + ]) + .expect("prepare two homogeneous CUDA groups"); + assert_eq!(prepared.groups().len(), 2); + + let submitted = decoder + .submit_prepared(&prepared) + .expect("unsupported group must remain a result-level failure"); + assert_eq!(submitted.len(), 1); + let output = submitted.wait().expect("finish supported CUDA group"); + + assert!(output.errors.is_empty()); + assert_eq!(output.groups.len(), 1); + assert_eq!(output.groups[0].source_indices, [1]); + assert_eq!(output.group_errors.len(), 1); + assert_eq!(output.group_errors[0].source_indices(), &[0]); + assert!(matches!( + output.group_errors[0].source(), + BurnDecodeError::Cuda(j2k_cuda::CudaBatchError::GroupExecution { .. }) + )); +} diff --git a/crates/j2k-ml/tests/cuda_rgba.rs b/crates/j2k-ml/tests/cuda_rgba.rs new file mode 100644 index 00000000..3cd6afed --- /dev/null +++ b/crates/j2k-ml/tests/cuda_rgba.rs @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#![cfg(all(feature = "cuda", not(target_os = "macos")))] + +use std::sync::Arc; + +use burn_core::tensor::DType; +use burn_cuda::CudaDevice; +use j2k::{ + encode_j2k_lossless, wrap_j2k_codestream, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, + CpuBatchSamples, DecodeRequest, Downscale, EncodedImage, J2kBlockCodingMode, + J2kChannelAssociation, J2kChannelDefinition, J2kChannelType, J2kEncodeValidation, + J2kFileBoxMetadata, J2kFileColorSpec, J2kFileWrapOptions, J2kLosslessEncodeOptions, + J2kLosslessSamples, Rect, ReversibleTransform, +}; +use j2k_core::Colorspace; +use j2k_ml::{BurnBatchTensor, CudaBurnDecoder}; +use j2k_test_support::{ + cuda_runtime_and_strict_oxide_gate, generated_htj2k_rgba_fixture, Htj2kRgbaAlpha, + Htj2kRgbaFixture, Htj2kRgbaSampleProfile, Htj2kRgbaSamples, +}; + +#[test] +fn direct_cuda_burn_rgba_matches_cpu_across_codecs_types_geometry_and_layouts() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml exact RGBA CUDA batch") { + return; + } + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { + roi: Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }, + }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi: Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }, + scale: Downscale::Half, + }, + ]; + + for profile in [ + Htj2kRgbaSampleProfile::U8Rct, + Htj2kRgbaSampleProfile::U12, + Htj2kRgbaSampleProfile::I16, + ] { + let fixture = generated_htj2k_rgba_fixture(profile, Htj2kRgbaAlpha::Straight); + let encodings = [ + ("HTJ2K", Arc::<[u8]>::from(wrap_rgba_jph(&fixture))), + ( + "classic JPEG 2000", + Arc::<[u8]>::from(wrap_classic_rgba_jp2(&fixture)), + ), + ]; + for (codec, encoded) in encodings { + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + for request in requests { + assert_burn_case(codec, &encoded, layout, request); + } + } + } + } +} + +fn assert_burn_case(codec: &str, encoded: &Arc<[u8]>, layout: BatchLayout, request: DecodeRequest) { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::new(Arc::clone(encoded), request), + EncodedImage::new(Arc::clone(encoded), request), + ]; + let mut decoder = CudaBurnDecoder::new(CudaDevice::default(), options) + .unwrap_or_else(|error| panic!("create {codec} RGBA CUDA adapter: {error}")); + let prepared = decoder + .prepare(inputs.clone()) + .unwrap_or_else(|error| panic!("prepare {codec} RGBA Burn batch: {error}")); + assert!(prepared.errors().is_empty(), "{codec} preflight errors"); + let [prepared_group] = prepared.groups() else { + panic!("expected one {codec} RGBA group") + }; + if codec == "HTJ2K" { + assert!(prepared_group.images()[0].htj2k_plan().is_some()); + } else { + assert!(prepared_group.images()[0].classic_plan().is_some()); + } + + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode_prepared(&prepared) + .unwrap_or_else(|error| panic!("CPU {codec} RGBA oracle: {error}")); + let output = decoder + .decode_prepared(&prepared) + .unwrap_or_else(|error| panic!("direct {codec} RGBA Burn decode: {error}")); + assert!(output.errors.is_empty(), "{codec} {layout:?} {request:?}"); + assert!( + output.group_errors.is_empty(), + "{codec} {layout:?} {request:?}: {:?}", + output.group_errors + ); + let group = output.groups.into_iter().next().expect("RGBA Burn group"); + assert_eq!(group.source_indices, [0, 1]); + let dimensions = prepared_group.info().dimensions; + let shape = match layout { + BatchLayout::Nhwc => [2, dimensions.1 as usize, dimensions.0 as usize, 4], + BatchLayout::Nchw => [2, 4, dimensions.1 as usize, dimensions.0 as usize], + _ => unreachable!("test only covers public dense layouts"), + }; + assert_tensor_matches_cpu( + expected.groups()[0].samples(), + group.tensor, + shape, + &format!("{codec} {layout:?} {request:?}"), + ); +} + +fn assert_tensor_matches_cpu( + expected: &CpuBatchSamples, + tensor: BurnBatchTensor, + shape: [usize; 4], + case: &str, +) { + match (expected, tensor) { + (CpuBatchSamples::U8(expected), BurnBatchTensor::U8(tensor)) => { + assert_eq!(tensor.dtype(), DType::U8); + assert_eq!(tensor.dims(), shape); + assert_eq!( + tensor.into_data().into_vec::().expect("RGBA U8 data"), + *expected, + "{case}" + ); + } + (CpuBatchSamples::U16(expected), BurnBatchTensor::U16(tensor)) => { + assert_eq!(tensor.dtype(), DType::U16); + assert_eq!(tensor.dims(), shape); + assert_eq!( + tensor.into_data().into_vec::().expect("RGBA U16 data"), + *expected, + "{case}" + ); + } + (CpuBatchSamples::I16(expected), BurnBatchTensor::I16(tensor)) => { + assert_eq!(tensor.dtype(), DType::I16); + assert_eq!(tensor.dims(), shape); + assert_eq!( + tensor.into_data().into_vec::().expect("RGBA I16 data"), + *expected, + "{case}" + ); + } + (expected, actual) => { + panic!("unexpected RGBA storage for {case}: expected {expected:?}, got {actual:?}") + } + } +} + +fn wrap_rgba_jph(fixture: &Htj2kRgbaFixture) -> Vec { + wrap_rgba_container(&fixture.encoded, fixture.alpha, J2kFileWrapOptions::jph()) +} + +fn wrap_classic_rgba_jp2(fixture: &Htj2kRgbaFixture) -> Vec { + let bytes = samples_as_encoded_bytes(&fixture.samples); + let samples = J2kLosslessSamples::new( + &bytes, + fixture.width, + fixture.height, + 4, + fixture.bit_depth, + fixture.signed, + ) + .expect("valid classic RGBA samples"); + let transform = if fixture.use_mct { + ReversibleTransform::Rct53 + } else { + ReversibleTransform::None53 + }; + let codestream = encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_reversible_transform(transform) + .with_validation(J2kEncodeValidation::External), + ) + .expect("encode classic RGBA fixture") + .codestream; + wrap_rgba_container(&codestream, fixture.alpha, J2kFileWrapOptions::jp2()) +} + +fn samples_as_encoded_bytes(samples: &Htj2kRgbaSamples) -> Vec { + match samples { + Htj2kRgbaSamples::U8(samples) => samples.clone(), + Htj2kRgbaSamples::U16(samples) => samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect(), + Htj2kRgbaSamples::I16(samples) => samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect(), + } +} + +fn wrap_rgba_container( + codestream: &[u8], + alpha: Htj2kRgbaAlpha, + options: J2kFileWrapOptions<'_>, +) -> Vec { + let alpha_type = match alpha { + Htj2kRgbaAlpha::Straight => J2kChannelType::Opacity, + Htj2kRgbaAlpha::Premultiplied => J2kChannelType::PremultipliedOpacity, + }; + let definitions = [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 3, + channel_type: alpha_type, + association: J2kChannelAssociation::WholeImage, + }, + ]; + wrap_j2k_codestream( + codestream, + options + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &definitions, + }), + ) + .expect("wrap RGBA container") +} diff --git a/crates/j2k-ml/tests/metal.rs b/crates/j2k-ml/tests/metal.rs index 3d790675..3f905d6c 100644 --- a/crates/j2k-ml/tests/metal.rs +++ b/crates/j2k-ml/tests/metal.rs @@ -2,200 +2,199 @@ #![cfg(all(feature = "metal", target_os = "macos"))] -use burn_flex::{Flex, FlexDevice}; -use burn_wgpu::WgpuDevice; -use j2k::{DeviceDecodeRequest, Downscale, Rect}; -use j2k_ml::{ - cpu, metal, FloatNormalization, TensorDecodeOptions, TensorInput, TensorLayout, TensorRoute, +use std::sync::Arc; + +use burn_core::tensor::DType; +use j2k::{ + encode_j2k_lossless, prepare_batch, wrap_j2k_codestream, BatchDecodeOptions, BatchItemError, + BatchLayout, CpuBatchDecoder, CpuBatchSamples, DecodeRequest, DecodeSettings, Downscale, + EncodedImage, J2kBlockCodingMode, J2kChannelAssociation, J2kChannelDefinition, J2kChannelType, + J2kEncodeValidation, J2kFileBoxMetadata, J2kFileColorSpec, J2kFileWrapOptions, + J2kLosslessEncodeOptions, J2kLosslessSamples, Rect, }; +use j2k_core::Colorspace; +use j2k_ml::{BurnBatchTensor, BurnDecodeError, MetalBurnDecoder}; +use j2k_native::{encode, EncodeOptions}; use j2k_test_support::{ - classic_j2k_gray8_fixture, metal_runtime_gate, openhtj2k_refinement_fixture, + generated_htj2k_rgba_fixture, htj2k_rgb8_97_fixture, metal_runtime_gate, + openhtj2k_refinement_fixture, openhtj2k_refinement_odd_fixture, openhtj2k_refinement_pixels, + Htj2kRgbaAlpha, Htj2kRgbaSampleProfile, }; -#[test] -fn strict_metal_staged_decode_reports_route_and_pixels() { - if !metal_runtime_gate("j2k-ml strict Metal staged decode") { - return; - } - let encoded = classic_j2k_gray8_fixture(4, 3); - let decoded = metal::decode_u8( - TensorInput::full(&encoded), - &TensorDecodeOptions::default(), - &WgpuDevice::DefaultDevice, +fn wrap_rgba_jph(codestream: &[u8], alpha: Htj2kRgbaAlpha) -> Vec { + let alpha_type = match alpha { + Htj2kRgbaAlpha::Straight => J2kChannelType::Opacity, + Htj2kRgbaAlpha::Premultiplied => J2kChannelType::PremultipliedOpacity, + }; + let channel_definitions = [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 3, + channel_type: alpha_type, + association: J2kChannelAssociation::WholeImage, + }, + ]; + wrap_j2k_codestream( + codestream, + J2kFileWrapOptions::jph() + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &channel_definitions, + }), ) - .expect("strict Metal tensor decode"); + .expect("wrap explicit HTJ2K RGBA image") +} - assert_eq!(decoded.route, TensorRoute::MetalStaged); - assert_eq!(decoded.tensor.dims(), [1, 3, 4]); - assert_eq!( - decoded - .tensor - .into_data() - .into_vec::() - .expect("u8 data"), - (0..12).collect::>() - ); +fn unsupported_classic_roi_rgb() -> Arc<[u8]> { + let pixels = (0..4_u8) + .flat_map(|index| [index * 17, index * 29 + 3, index * 41 + 5]) + .collect::>(); + Arc::from( + encode( + &pixels, + 2, + 2, + 3, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + roi_component_shifts: vec![3, 0, 0], + ..EncodeOptions::default() + }, + ) + .expect("encode classic RGB8 with unsupported RGN maxshift"), + ) } -#[test] -fn metal_batch_returns_one_rank_four_tensor() { - if !metal_runtime_gate("j2k-ml Metal staged batch") { - return; - } - let first = classic_j2k_gray8_fixture(2, 2); - let second = classic_j2k_gray8_fixture(2, 2); - let decoded = metal::decode_u8_batch( - &[TensorInput::full(&first), TensorInput::full(&second)], - &TensorDecodeOptions::default(), - &WgpuDevice::DefaultDevice, +fn encode_gray12(samples: &[u16]) -> Vec { + let bytes = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + let samples = J2kLosslessSamples::new(&bytes, 2, 2, 1, 12, false).expect("Gray12 samples"); + encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External), ) - .expect("strict Metal batch"); - assert_eq!(decoded.route, TensorRoute::MetalStaged); - assert_eq!(decoded.tensor.dims(), [2, 1, 2, 2]); + .expect("encode Gray12") + .codestream } -#[test] -fn metal_u16_and_float_match_portable_decode() { - if !metal_runtime_gate("j2k-ml Metal u16 and float parity") { - return; - } - let encoded = openhtj2k_refinement_fixture(); - let expected_u16 = cpu::decode_u16::( - TensorInput::full(encoded), - &TensorDecodeOptions::default(), - &FlexDevice, +fn encode_signed_gray12(samples: &[i16]) -> Vec { + let bytes = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + let samples = + J2kLosslessSamples::new(&bytes, 2, 2, 1, 12, true).expect("signed Gray12 samples"); + encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External), ) - .expect("portable u16") - .tensor - .into_data() - .into_vec::() - .expect("portable u16 data"); - let actual_u16 = metal::decode_u16( - TensorInput::full(encoded), - &TensorDecodeOptions::default(), - &WgpuDevice::DefaultDevice, + .expect("encode signed Gray12") + .codestream +} + +fn encode_gray8(samples: &[u8], width: u32, height: u32) -> Vec { + let samples = + J2kLosslessSamples::new(samples, width, height, 1, 8, false).expect("Gray8 samples"); + encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External), ) - .expect("Metal u16") - .tensor - .into_data() - .into_vec::() - .expect("Metal u16 data"); - assert_eq!(actual_u16, expected_u16); + .expect("encode Gray8") + .codestream +} - for layout in [TensorLayout::ChannelsFirst, TensorLayout::ChannelsLast] { - for normalization in [ - FloatNormalization::Raw, - FloatNormalization::Unit, - FloatNormalization::MeanStd { - mean: vec![0.25], - std: vec![0.5], - }, - ] { - let options = TensorDecodeOptions { - layout, - normalization, - ..TensorDecodeOptions::default() - }; - let expected = - cpu::decode_float::(TensorInput::full(encoded), &options, &FlexDevice) - .expect("portable float") - .tensor - .into_data() - .into_vec::() - .expect("portable float data"); - let actual = metal::decode_float( - TensorInput::full(encoded), - &options, - &WgpuDevice::DefaultDevice, - ) - .expect("Metal float") - .tensor - .into_data() - .into_vec::() - .expect("Metal float data"); - for (actual, expected) in actual.iter().zip(expected) { - assert!((actual - expected).abs() <= 1.0e-6); +fn encode_rgb_u8(width: u32, height: u32, offset: u8) -> Vec { + let mut bytes = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + for pattern in [x * 3 + y * 5, x * 7 + y * 11 + 13, x * 17 + y * 19 + 29] { + let pattern = u8::try_from(pattern & 0x3f).expect("six-bit RGB pattern"); + bytes.push(offset.wrapping_add(pattern) & 0x3f); } } } + let samples = + J2kLosslessSamples::new(&bytes, width, height, 3, 6, false).expect("six-bit RGB samples"); + encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External), + ) + .expect("encode HT RGB U8") + .codestream } -#[test] -fn metal_roi_scale_and_all_batch_output_modes_report_staged_route() { - if !metal_runtime_gate("j2k-ml Metal ROI and batch modes") { - return; - } - let encoded = classic_j2k_gray8_fixture(8, 8); - let roi = Rect { - x: 2, - y: 2, - w: 4, - h: 4, - }; - let input = TensorInput { - encoded: &encoded, - request: DeviceDecodeRequest::RegionScaled { - roi, - scale: Downscale::Half, - }, - }; - let decoded = metal::decode_float( - input, - &TensorDecodeOptions::default(), - &WgpuDevice::DefaultDevice, - ) - .expect("Metal ROI-scaled float"); - assert_eq!(decoded.route, TensorRoute::MetalStaged); - assert_eq!(decoded.tensor.dims(), [1, 2, 2]); - let expected_roi = - cpu::decode_float::(input, &TensorDecodeOptions::default(), &FlexDevice) - .expect("portable ROI-scaled float") - .tensor - .into_data() - .into_vec::() - .expect("portable ROI data"); - assert_eq!( - decoded - .tensor - .into_data() - .into_vec::() - .expect("Metal ROI data"), - expected_roi - ); +fn encode_rgb_u16(width: u32, height: u32, offset: u16) -> Vec { + encode_rgb_u16_with_mode(width, height, offset, J2kBlockCodingMode::HighThroughput) +} - let encoded_u16 = openhtj2k_refinement_fixture(); - let u16_batch = metal::decode_u16_batch( - &[ - TensorInput::full(encoded_u16), - TensorInput::full(encoded_u16), - ], - &TensorDecodeOptions::default(), - &WgpuDevice::DefaultDevice, - ) - .expect("Metal u16 batch"); - assert_eq!(u16_batch.route, TensorRoute::MetalStaged); - assert_eq!(u16_batch.tensor.dims()[0], 2); - let u16_values = u16_batch - .tensor - .into_data() - .into_vec::() - .expect("Metal u16 batch data"); - let midpoint = u16_values.len() / 2; - assert_eq!(&u16_values[..midpoint], &u16_values[midpoint..]); +fn encode_classic_rgb_u16(width: u32, height: u32, offset: u16) -> Vec { + encode_rgb_u16_with_mode(width, height, offset, J2kBlockCodingMode::Classic) +} - let float_batch = metal::decode_float_batch( - &[TensorInput::full(&encoded), TensorInput::full(&encoded)], - &TensorDecodeOptions::default(), - &WgpuDevice::DefaultDevice, +fn encode_rgb_u16_with_mode( + width: u32, + height: u32, + offset: u16, + block_coding_mode: J2kBlockCodingMode, +) -> Vec { + let mut bytes = Vec::with_capacity(width as usize * height as usize * 3 * 2); + for y in 0..height { + for x in 0..width { + for pattern in [ + x * 193 + y * 257, + x * 313 + y * 97 + 31, + x * 71 + y * 401 + 63, + ] { + let pattern = u16::try_from(pattern & 0x07ff).expect("twelve-bit RGB pattern"); + let sample = offset.wrapping_add(pattern); + bytes.extend_from_slice(&(sample & 0x0fff).to_le_bytes()); + } + } + } + let samples = J2kLosslessSamples::new(&bytes, width, height, 3, 12, false) + .expect("twelve-bit RGB samples"); + encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions::default() + .with_block_coding_mode(block_coding_mode) + .with_validation(J2kEncodeValidation::External), ) - .expect("Metal float batch"); - assert_eq!(float_batch.route, TensorRoute::MetalStaged); - assert_eq!(float_batch.tensor.dims(), [2, 1, 8, 8]); - let float_values = float_batch - .tensor - .into_data() - .into_vec::() - .expect("Metal float batch data"); - let midpoint = float_values.len() / 2; - assert_eq!(&float_values[..midpoint], &float_values[midpoint..]); + .expect("encode RGB U16") + .codestream } + +#[path = "metal/native_color.rs"] +mod native_color; +#[path = "metal/requests.rs"] +mod requests; +#[path = "metal/sessions.rs"] +mod sessions; diff --git a/crates/j2k-ml/tests/metal/native_color.rs b/crates/j2k-ml/tests/metal/native_color.rs new file mode 100644 index 00000000..11b77e6c --- /dev/null +++ b/crates/j2k-ml/tests/metal/native_color.rs @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn independent_openjph_signed_rgb_is_exact_in_burn_for_all_requests_and_layouts() { + if !metal_runtime_gate("j2k-ml exact signed RGB Metal batch") { + return; + } + + let roi = Rect { + x: 3, + y: 2, + w: 9, + h: 7, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + let fixtures = j2k_test_support::openjph_batch_fixtures() + .iter() + .filter(|fixture| { + matches!( + fixture.name, + "openjph-rgb-s8-53-single-raw" + | "openjph-rgb-s12-53-single-raw" + | "openjph-rgb-s16-53-single-raw" + ) + }) + .collect::>(); + assert_eq!(fixtures.len(), 3); + + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let mut decoder = MetalBurnDecoder::system_default(options).expect("paired Metal session"); + for fixture in &fixtures { + let encoded = Arc::<[u8]>::from(fixture.encoded); + for request in requests { + let inputs = vec![ + EncodedImage::new(encoded.clone(), request), + EncodedImage::new(encoded.clone(), request), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(inputs.clone()) + .unwrap_or_else(|error| panic!("{} CPU oracle: {error}", fixture.name)); + let CpuBatchSamples::I16(expected) = expected.groups()[0].samples() else { + panic!("{} must use I16 batch storage", fixture.name) + }; + + let prepared = decoder + .prepare(inputs) + .unwrap_or_else(|error| panic!("{} prepare: {error}", fixture.name)); + let output = decoder + .decode_prepared(&prepared) + .unwrap_or_else(|error| panic!("{} decode: {error}", fixture.name)); + assert!(output.errors.is_empty(), "{}", fixture.name); + assert!(output.group_errors.is_empty(), "{}", fixture.name); + assert_eq!(output.groups.len(), 1, "{}", fixture.name); + assert_eq!(output.groups[0].source_indices, [0, 1]); + let BurnBatchTensor::I16(tensor) = output.groups.into_iter().next().unwrap().tensor + else { + panic!("{} must produce an I16 Burn tensor", fixture.name) + }; + assert_eq!(tensor.dtype(), DType::I16); + let actual = tensor.into_data().into_vec::().expect("I16 data"); + assert_eq!( + actual.as_slice(), + expected.as_slice(), + "{} {layout:?} {request:?}", + fixture.name + ); + } + } + } +} + +#[test] +fn direct_metal_burn_rgba_is_exact_for_native_types_and_layouts() { + if !metal_runtime_gate("j2k-ml exact RGBA Metal batch") { + return; + } + + for profile in [ + Htj2kRgbaSampleProfile::U8Rct, + Htj2kRgbaSampleProfile::U12, + Htj2kRgbaSampleProfile::I16, + ] { + let fixture = generated_htj2k_rgba_fixture(profile, Htj2kRgbaAlpha::Straight); + let encoded = Arc::<[u8]>::from(wrap_rgba_jph(&fixture.encoded, fixture.alpha)); + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::full(encoded.clone()), + EncodedImage::full(encoded.clone()), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU RGBA oracle"); + assert!( + expected.errors().is_empty(), + "{profile:?} {layout:?}: {:?}", + expected.errors() + ); + assert_eq!(expected.groups().len(), 1); + + let mut decoder = + MetalBurnDecoder::system_default(options).expect("paired Metal session"); + let prepared = decoder.prepare(inputs).expect("prepare RGBA Burn batch"); + let output = decoder + .decode_prepared(&prepared) + .expect("direct RGBA Burn decode"); + assert!(output.errors.is_empty(), "{profile:?} {layout:?}"); + assert!(output.group_errors.is_empty(), "{profile:?} {layout:?}"); + assert_eq!(output.groups.len(), 1, "{profile:?} {layout:?}"); + let group = output.groups.into_iter().next().expect("RGBA group"); + assert_eq!(group.source_indices, [0, 1]); + let shape = match layout { + BatchLayout::Nhwc => [ + 2, + usize::try_from(fixture.height).expect("RGBA height fits usize"), + usize::try_from(fixture.width).expect("RGBA width fits usize"), + 4, + ], + BatchLayout::Nchw => [ + 2, + 4, + usize::try_from(fixture.height).expect("RGBA height fits usize"), + usize::try_from(fixture.width).expect("RGBA width fits usize"), + ], + _ => unreachable!(), + }; + match (expected.groups()[0].samples(), group.tensor) { + (CpuBatchSamples::U8(expected), BurnBatchTensor::U8(tensor)) => { + assert_eq!(tensor.dtype(), DType::U8); + assert_eq!(tensor.dims(), shape); + assert_eq!( + tensor.into_data().into_vec::().expect("RGBA U8 data"), + *expected, + "{profile:?} {layout:?}" + ); + } + (CpuBatchSamples::U16(expected), BurnBatchTensor::U16(tensor)) => { + assert_eq!(tensor.dtype(), DType::U16); + assert_eq!(tensor.dims(), shape); + assert_eq!( + tensor + .into_data() + .into_vec::() + .expect("RGBA U16 data"), + *expected, + "{profile:?} {layout:?}" + ); + } + (CpuBatchSamples::I16(expected), BurnBatchTensor::I16(tensor)) => { + assert_eq!(tensor.dtype(), DType::I16); + assert_eq!(tensor.dims(), shape); + assert_eq!( + tensor + .into_data() + .into_vec::() + .expect("RGBA I16 data"), + *expected, + "{profile:?} {layout:?}" + ); + } + (expected, actual) => panic!( + "unexpected RGBA storage for {profile:?} {layout:?}: expected {expected:?}, got {actual:?}" + ), + } + } + } +} + +#[test] +fn direct_metal_batch_preserves_native_u16_samples() { + if !metal_runtime_gate("j2k-ml direct Metal U16 batch") { + return; + } + let samples = [0_u16, 1, 2048, 4095]; + let encoded = encode_gray12(&samples); + let mut decoder = MetalBurnDecoder::system_default(BatchDecodeOptions::default()) + .expect("paired J2K/Burn Metal session"); + let output = decoder + .decode(vec![EncodedImage::full(Arc::from(encoded))]) + .expect("direct Metal U16 decode"); + + let BurnBatchTensor::U16(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected native U16 Metal tensor") + }; + assert_eq!(tensor.dtype(), DType::U16); + assert_eq!( + tensor.into_data().into_vec::().expect("U16 data"), + samples + ); +} + +#[test] +fn direct_metal_batch_preserves_native_signed_i16_samples() { + if !metal_runtime_gate("j2k-ml direct Metal signed I16 batch") { + return; + } + let samples = [-2048_i16, -1, 0, 2047]; + let encoded = Arc::<[u8]>::from(encode_signed_gray12(&samples)); + let options = BatchDecodeOptions::default(); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(vec![EncodedImage::full(Arc::clone(&encoded))]) + .expect("CPU signed Gray12 oracle"); + let CpuBatchSamples::I16(expected) = expected.groups()[0].samples() else { + panic!("signed Gray12 CPU oracle must be I16") + }; + + let mut decoder = MetalBurnDecoder::system_default(options).expect("paired Metal session"); + let output = decoder + .decode(vec![EncodedImage::full(encoded)]) + .expect("direct Metal signed Gray12 decode"); + let BurnBatchTensor::I16(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected native I16 Metal tensor") + }; + assert_eq!(tensor.dtype(), DType::I16); + assert_eq!( + tensor.into_data().into_vec::().expect("I16 data"), + *expected + ); +} diff --git a/crates/j2k-ml/tests/metal/requests.rs b/crates/j2k-ml/tests/metal/requests.rs new file mode 100644 index 00000000..a8c8912b --- /dev/null +++ b/crates/j2k-ml/tests/metal/requests.rs @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn direct_metal_burn_region_reduced_matches_cpu_oracle() { + if !metal_runtime_gate("j2k-ml direct Metal ROI-reduced batch") { + return; + } + let encoded = Arc::<[u8]>::from(openhtj2k_refinement_odd_fixture()); + let request = DecodeRequest::RegionReduced { + roi: Rect { + x: 3, + y: 5, + w: 9, + h: 21, + }, + scale: Downscale::Half, + }; + let options = BatchDecodeOptions::default(); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(vec![EncodedImage::new(encoded.clone(), request)]) + .expect("CPU ROI-reduced oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("odd OpenHT fixture must decode to U8") + }; + + let mut decoder = MetalBurnDecoder::system_default(options).expect("paired Metal session"); + let output = decoder + .decode(vec![EncodedImage::new(encoded, request)]) + .expect("direct Metal ROI-reduced decode"); + let BurnBatchTensor::U8(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected native U8 Metal tensor") + }; + let actual = tensor.into_data().into_vec::().expect("U8 data"); + assert_eq!(actual.as_slice(), expected.as_slice()); +} + +#[test] +fn direct_metal_odd_u8_tensor_tails_are_not_zero_initialized() { + if !metal_runtime_gate("j2k-ml odd direct Metal tensor tails") { + return; + } + let mut decoder = MetalBurnDecoder::system_default(BatchDecodeOptions::default()) + .expect("paired J2K/Burn Metal session"); + for (width, height) in [(5_u32, 1_u32), (11, 3)] { + let pixels = (0..width * height) + .map(|index| index.to_le_bytes()[0].wrapping_mul(17).wrapping_add(1)) + .collect::>(); + let encoded = encode_gray8(&pixels, width, height); + let output = decoder + .decode(vec![EncodedImage::full(Arc::from(encoded))]) + .expect("direct odd-size Metal decode"); + let BurnBatchTensor::U8(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected native U8 Metal tensor") + }; + assert_eq!( + tensor.into_data().into_vec::().expect("U8 data"), + pixels + ); + } +} + +#[test] +fn direct_metal_burn_rgb_u8_region_reduced_is_exact_for_nhwc_and_nchw() { + if !metal_runtime_gate("j2k-ml exact RGB U8 Metal batch") { + return; + } + let first = Arc::<[u8]>::from(encode_rgb_u8(8, 8, 0)); + let second = Arc::<[u8]>::from(encode_rgb_u8(8, 8, 17)); + let request = DecodeRequest::RegionReduced { + roi: Rect { + x: 2, + y: 2, + w: 4, + h: 4, + }, + scale: Downscale::Half, + }; + + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::new(first.clone(), request), + EncodedImage::new(second.clone(), request), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU RGB U8 oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("six-bit RGB must use U8 batch storage") + }; + + let mut decoder = MetalBurnDecoder::system_default(options).expect("paired Metal session"); + let prepared = decoder + .prepare(inputs) + .expect("prepare reusable RGB U8 batch"); + for _ in 0..2 { + let output = decoder + .decode_prepared(&prepared) + .expect("direct exact RGB U8 Burn decode"); + let group = output.groups.into_iter().next().expect("RGB U8 group"); + let BurnBatchTensor::U8(tensor) = group.tensor else { + panic!("expected native RGB U8 Metal tensor") + }; + assert_eq!(tensor.dtype(), DType::U8); + assert_eq!( + tensor.dims(), + match layout { + BatchLayout::Nhwc => [2, 2, 2, 3], + BatchLayout::Nchw => [2, 3, 2, 2], + _ => unreachable!(), + } + ); + let actual = tensor.into_data().into_vec::().expect("RGB U8 data"); + assert_eq!(actual.as_slice(), expected.as_slice(), "{layout:?}"); + assert!(actual.iter().all(|sample| *sample <= 0x3f)); + } + } +} + +#[test] +fn direct_metal_burn_rgb_u16_is_exact_and_drop_safe() { + if !metal_runtime_gate("j2k-ml exact RGB U16 Metal batch") { + return; + } + let encoded = Arc::<[u8]>::from(encode_rgb_u16(8, 8, 257)); + let options = BatchDecodeOptions { + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let inputs = vec![EncodedImage::full(encoded.clone())]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU RGB U16 oracle"); + let CpuBatchSamples::U16(expected) = expected.groups()[0].samples() else { + panic!("twelve-bit RGB must use U16 batch storage") + }; + + let mut decoder = MetalBurnDecoder::system_default(options).expect("paired Metal session"); + let prepared = decoder + .prepare(inputs) + .expect("prepare reusable RGB U16 batch"); + drop( + decoder + .submit_prepared(&prepared) + .expect("submit disposable RGB U16 Burn batch"), + ); + let output = decoder + .decode_prepared(&prepared) + .expect("reuse RGB U16 Burn decoder after pending drop"); + let BurnBatchTensor::U16(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected native RGB U16 Metal tensor") + }; + assert_eq!(tensor.dtype(), DType::U16); + let actual = tensor.into_data().into_vec::().expect("RGB U16 data"); + assert_eq!(actual.as_slice(), expected.as_slice()); + assert!(actual.iter().all(|sample| *sample <= 0x0fff)); +} + +#[test] +fn direct_metal_burn_classic_rgb_u16_is_exact_for_both_layouts() { + if !metal_runtime_gate("j2k-ml exact classic RGB U16 Metal batch") { + return; + } + let encoded = Arc::<[u8]>::from(encode_classic_rgb_u16(8, 8, 257)); + for layout in [BatchLayout::Nchw, BatchLayout::Nhwc] { + let options = BatchDecodeOptions { + layout, + ..BatchDecodeOptions::default() + }; + let inputs = vec![ + EncodedImage::full(encoded.clone()), + EncodedImage::full(encoded.clone()), + ]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu.decode(inputs.clone()).expect("CPU classic RGB oracle"); + let CpuBatchSamples::U16(expected) = expected.groups()[0].samples() else { + panic!("classic twelve-bit RGB must use U16 batch storage") + }; + + let mut decoder = MetalBurnDecoder::system_default(options).expect("paired Metal session"); + let prepared = decoder + .prepare(inputs) + .expect("prepare classic RGB Burn batch"); + let output = decoder + .decode_prepared(&prepared) + .expect("direct classic RGB Burn decode"); + assert!(output.errors.is_empty(), "{layout:?}"); + assert!(output.group_errors.is_empty(), "{layout:?}"); + let group = output.groups.into_iter().next().expect("classic RGB group"); + assert_eq!(group.source_indices, [0, 1]); + let BurnBatchTensor::U16(tensor) = group.tensor else { + panic!("expected native classic RGB U16 Metal tensor") + }; + assert_eq!(tensor.dtype(), DType::U16); + let actual = tensor.into_data().into_vec::().expect("RGB U16 data"); + assert_eq!(actual.as_slice(), expected.as_slice(), "{layout:?}"); + } +} + +#[test] +fn direct_metal_burn_irreversible_rgb_u8_is_within_one_lsb_of_cpu() { + if !metal_runtime_gate("j2k-ml irreversible RGB U8 Metal batch") { + return; + } + let encoded = Arc::<[u8]>::from(htj2k_rgb8_97_fixture(8, 8)); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let inputs = vec![EncodedImage::full(encoded)]; + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode(inputs.clone()) + .expect("CPU irreversible RGB oracle"); + let CpuBatchSamples::U8(expected) = expected.groups()[0].samples() else { + panic!("irreversible RGB8 must use U8 batch storage") + }; + + let mut decoder = MetalBurnDecoder::system_default(options).expect("paired Metal session"); + let output = decoder + .decode(inputs) + .expect("direct irreversible RGB Burn decode"); + let BurnBatchTensor::U8(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected native irreversible RGB U8 Metal tensor") + }; + let actual = tensor.into_data().into_vec::().expect("RGB U8 data"); + assert_eq!(actual.len(), expected.len()); + assert!( + actual + .iter() + .zip(expected.iter()) + .all(|(metal, cpu)| metal.abs_diff(*cpu) <= 1), + "irreversible 9/7 Metal reconstruction must stay within one integer LSB of CPU" + ); +} diff --git a/crates/j2k-ml/tests/metal/sessions.rs b/crates/j2k-ml/tests/metal/sessions.rs new file mode 100644 index 00000000..d01ad501 --- /dev/null +++ b/crates/j2k-ml/tests/metal/sessions.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn repeated_default_decoders_reuse_one_initialized_burn_device() { + if !metal_runtime_gate("j2k-ml repeated paired Metal device") { + return; + } + + let first = MetalBurnDecoder::system_default(BatchDecodeOptions::default()) + .expect("first paired J2K/Burn Metal session"); + let second = MetalBurnDecoder::system_default(BatchDecodeOptions::default()) + .expect("second paired J2K/Burn Metal session"); + + assert_eq!( + first.device(), + second.device(), + "one cached wgpu setup must map to one initialized CubeCL device identity" + ); +} + +#[test] +fn persistent_metal_burn_decoder_writes_independent_ht_directly() { + if !metal_runtime_gate("j2k-ml direct Metal Burn batch") { + return; + } + let mut decoder = MetalBurnDecoder::system_default(BatchDecodeOptions::default()) + .expect("paired J2K/Burn Metal session"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(Arc::from(openhtj2k_refinement_fixture())), + EncodedImage::full(Arc::from(openhtj2k_refinement_fixture())), + ]) + .expect("prepare independent HTJ2K fixture"); + + for _ in 0..2 { + let output = decoder + .decode_prepared(&prepared) + .expect("direct Metal Burn decode"); + assert!(output.errors.is_empty()); + let BurnBatchTensor::U8(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected native U8 Metal tensor") + }; + assert_eq!(tensor.dims()[0], 2); + let actual = tensor.into_data().into_vec::().expect("U8 data"); + let expected = openhtj2k_refinement_pixels(); + assert_eq!( + actual, + expected.iter().chain(expected).copied().collect::>() + ); + } + assert!(decoder.codec().submissions().expect("submission count") >= 2); +} + +#[test] +fn metal_burn_regroups_prepared_images_with_submission_indices_and_settings_errors() { + if !metal_runtime_gate("j2k-ml Metal prepared-image regrouping") { + return; + } + + let encoded = Arc::<[u8]>::from(openhtj2k_refinement_fixture()); + let strict = prepare_batch( + vec![EncodedImage::full(encoded.clone())], + BatchDecodeOptions::default(), + ) + .expect("strict preparation") + .groups()[0] + .images()[0] + .clone(); + let lenient_options = BatchDecodeOptions { + settings: DecodeSettings::lenient(), + ..BatchDecodeOptions::default() + }; + let lenient = prepare_batch(vec![EncodedImage::full(encoded)], lenient_options) + .expect("lenient preparation") + .groups()[0] + .images()[0] + .clone(); + let mut decoder = MetalBurnDecoder::system_default(BatchDecodeOptions::default()) + .expect("paired J2K/Burn Metal session"); + + let regrouped = decoder + .prepare_prepared_images(vec![lenient, strict.clone(), strict.clone()]) + .expect("regroup prepared images"); + assert_eq!(regrouped.errors().len(), 1); + assert_eq!(regrouped.errors()[0].index, 0); + assert!(matches!( + regrouped.errors()[0].source, + BatchItemError::PreparedDecodeSettingsMismatch { + prepared, + requested, + } if prepared == DecodeSettings::lenient() && requested == DecodeSettings::strict() + )); + assert_eq!(regrouped.groups().len(), 1); + assert_eq!(regrouped.groups()[0].source_indices(), [1, 2]); + + let output = decoder + .decode_prepared_images(vec![strict.clone(), strict]) + .expect("decode regrouped prepared images"); + assert!(output.errors.is_empty()); + assert!(output.group_errors.is_empty()); + assert_eq!(output.groups.len(), 1); + assert_eq!(output.groups[0].source_indices, [0, 1]); + assert_eq!(output.groups[0].decoded_rects.len(), 2); + assert_eq!(output.groups[0].warnings, [Vec::new(), Vec::new()]); + assert_eq!(output.groups[0].tensor.tensor().dims()[0], 2); +} + +#[test] +fn dropped_pending_metal_burn_batch_retires_storage_and_decoder_reuses() { + if !metal_runtime_gate("j2k-ml dropped direct Metal Burn batch") { + return; + } + let mut decoder = MetalBurnDecoder::system_default(BatchDecodeOptions::default()) + .expect("paired J2K/Burn Metal session"); + let prepared = decoder + .prepare(vec![EncodedImage::full(Arc::from( + openhtj2k_refinement_fixture(), + ))]) + .expect("prepare HTJ2K fixture"); + + let pending = decoder + .submit_prepared(&prepared) + .expect("submit disposable Burn batch"); + drop(pending); + + let output = decoder + .decode_prepared(&prepared) + .expect("reuse decoder after dropped pending batch"); + let BurnBatchTensor::U8(tensor) = output.groups.into_iter().next().unwrap().tensor else { + panic!("expected native U8 Metal tensor") + }; + assert_eq!( + tensor.into_data().into_vec::().expect("U8 data"), + openhtj2k_refinement_pixels() + ); +} + +#[test] +fn metal_burn_batch_continues_after_one_group_submit_failure() { + if !metal_runtime_gate("j2k-ml Metal group submit continuation") { + return; + } + let valid_gray = Arc::<[u8]>::from(openhtj2k_refinement_fixture()); + let mut decoder = MetalBurnDecoder::system_default(BatchDecodeOptions::default()) + .expect("paired J2K/Burn Metal session"); + let prepared = decoder + .prepare(vec![ + EncodedImage::full(unsupported_classic_roi_rgb()), + EncodedImage::full(valid_gray), + ]) + .expect("prepare two homogeneous Metal groups"); + assert_eq!(prepared.groups().len(), 2); + + let submitted = decoder + .submit_prepared(&prepared) + .expect("unsupported group must remain a result-level failure"); + assert_eq!(submitted.len(), 1); + let output = submitted.wait().expect("finish supported Metal group"); + + assert!(output.errors.is_empty()); + assert_eq!(output.groups.len(), 1); + assert_eq!(output.groups[0].source_indices, [1]); + assert_eq!(output.group_errors.len(), 1); + assert_eq!(output.group_errors[0].source_indices(), &[0]); + assert!(matches!( + output.group_errors[0].source(), + BurnDecodeError::Metal(j2k_metal::Error::UnsupportedMetalRequest { .. }) + )); +} diff --git a/crates/j2k-native/fixtures/htj2k/LICENSE.OpenJPH b/crates/j2k-native/fixtures/htj2k/LICENSE.OpenJPH new file mode 100644 index 00000000..d403b9c7 --- /dev/null +++ b/crates/j2k-native/fixtures/htj2k/LICENSE.OpenJPH @@ -0,0 +1,27 @@ +BSD 2-Clause License + +Copyright (c) 2019, Aous Naman +Copyright (c) 2019, Kakadu Software Pty Ltd, Australia +Copyright (c) 2019, The University of New South Wales, Sydney, Australia +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crates/j2k-native/fixtures/htj2k/README.md b/crates/j2k-native/fixtures/htj2k/README.md index 1c5d7e6f..1c0a9474 100644 --- a/crates/j2k-native/fixtures/htj2k/README.md +++ b/crates/j2k-native/fixtures/htj2k/README.md @@ -1,9 +1,8 @@ HTJ2K fixtures for native decoder coverage. -These fixtures are from the OpenHTJ2K conformance data, copied from OpenHTJ2K -commit `ffe5acf9f1eedb87c36c3fd2134fdc1ddea5e75f`. They are tiny HTONLY -codestreams derived from the JPEG 2000 Part 4 / ITU-T T.803 HTJ2K conformance -set. +The OpenHTJ2K fixtures are copied from OpenHTJ2K commit +`ffe5acf9f1eedb87c36c3fd2134fdc1ddea5e75f`. They are tiny HTONLY codestreams +derived from the JPEG 2000 Part 4 / ITU-T T.803 HTJ2K conformance set. `openhtj2k_ds0_ht_12_b11.j2k` is copied from `ds0_ht_12_b11.j2k`, blob `cf3fb0bc7e55898b4e6977f38ba0d38d91c359bf`. The native decoder sees 8 HT code @@ -20,3 +19,28 @@ The paired `.gray` file contains the expected 8-bit grayscale samples in row-major order from decoding the checked-in codestream with OpenJPH. The OpenHTJ2K source license is retained in `LICENSE.OpenHTJ2K`. + +The `gray_u12_53` and `rgb_u12_53` pairs are package-local copies of the +independent OpenJPH batch fixtures used by the native multi-tile integration +tests. OpenJPH 0.27.0 encoded them on 2026-07-18 as reversible 5/3 raw Part 15 +codestreams with a 19 x 13 image, 11 x 7 tiles, 8 x 8 code blocks, and two +wavelet decompositions. A separate OpenJPH decode produced each top-down raw +oracle; 12-bit samples use little-endian `u16` containers and RGB samples are +interleaved. The generating `ojph_compress` and `ojph_expand` executable +SHA-256 values were, respectively, +`b9846d39ca27506e0a93c66e42b287f4730ad071a26fc54bd4024aedaecf280f` and +`4b420506bd2a44439cf472d956bc1552c8be72ff6dffe2c10042e0e36b8de843`. + +The copied artifact SHA-256 values are: + +```text +f2735a7b4911f82ce53f4e0da9c155900f0974be43a9f813612ee5215c7492aa gray_u12_53.j2c +8962a79ec54f5bb1c244bfce724b77c6e3d16367b8a5cec10a4b7809b7bc2b85 gray_u12_53.oracle.raw +ae47e7be0f46df81c1da8a59e06ef76ec310992254344ffc27c62fc58d800b16 rgb_u12_53.j2c +88653085de44d28bd02f19e8a245d9d7c000be33ce33d6cce5087818933ac15b rgb_u12_53.oracle.raw +``` + +The canonical source/oracle generator and full reproduction commands remain in +`crates/j2k-test-support/fixtures/htj2k/openjph_batch/` in the repository. The +OpenJPH BSD 2-Clause license is retained here in `LICENSE.OpenJPH` so packaged +native tests keep their fixture provenance and license material together. diff --git a/crates/j2k-native/fixtures/htj2k/gray_u12_53.j2c b/crates/j2k-native/fixtures/htj2k/gray_u12_53.j2c new file mode 100644 index 0000000000000000000000000000000000000000..592a94641c1c0d03c9ea93de77f13b4d1cb4adc5 GIT binary patch literal 404 zcmezG|38pH(}4j9gn^hB#DD^BAk7ZrFfej6GX4)>;9y_^3bFkUV&DO?7#Wxt866n^ z$1rdyWE2z>WMpLgPhk*eVDv9Y&GQQIPzXycQZUdnGS@TE`#*t!3v8M#1LOb62blK! z&%btk-rggAFBn=Fer{tpz-YoCYQW$%=|m%U34_v70R|3-@-T*P4d+W37{Hntf!duQ znpaG=>6tzEH)FdAP+uSe%LN86jy9H`3{6`YHe6q-U(&p4Rj1H%f2lUh0Xhqy~2 zCV<=?1hu~N_QXp*zdtDaV@P56Zw55RhC$eYQQ_glKWw%s3==+ZeKTSB!>-J5fGL6@ zoZ(Las|v%!xgokY{#Y;qLn+GwVj?rp#Bhj-OTOmrv=(J_hB|2ig9o3`J0*tXDGaxa wj2ZqgJO;XFRRZH|a|TX_a@L6s9Q6XP%Nedeb#O9a`1lt@{z+&rVfcR&0Ac2N+yDRo literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/htj2k/gray_u12_53.oracle.raw b/crates/j2k-native/fixtures/htj2k/gray_u12_53.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..d181ee66a35f3aa238cd670b0a4d7cfbb6b6624c GIT binary patch literal 494 zcmWO0O-qzv6b9hu-1j-pI}=#E3}P-^SYsq^3JQ_3vapW{p&?vJlnWO&T3HB5w1_^S zMdjFJ*`gRiqDVw4Ei7W%h^v@DXfg{osZoeUv?*==z{TZUl}osjF5_Oh&yKQDn&h9yK_GZrsHI*X0IW&Mml5>ZDb!%7_$1P>m#dki{gHP|iO0+ReL|?2vZJ$Q$`4 zrPzfO`jNwTlyNsZd56<3Dz(xkHzg;FQh|d-;UT6#Zst*5<2V;srg3SOD>5eYQjP@9 z;R$B2nmc)lS-vNAi?(UMc1l*JB#L@;VGsqBuz}rt#2JR#s69HQvs!AS@rdTkXKmb;t-)XN6Mj|D6!Zq`+0J49=Cdg pVlWx3iFEKem$^@SwV>-PWzX!N?e#bO2VWT+51s~pgT~19$bTz$dq4mH literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/htj2k/rgb_u12_53.j2c b/crates/j2k-native/fixtures/htj2k/rgb_u12_53.j2c new file mode 100644 index 0000000000000000000000000000000000000000..3b723df4532f49af6d35e8ff0ed809a8cb4afacc GIT binary patch literal 868 zcmZ`%eMl2w9Dd$Yx9LoVCjQaaa7j?KWR8?HJS<37s7qV@AtY_WkjfG2Duq^QRz$g! zgk&SiYDrL##Ds(u5?lZDfn=Fr7J(JXlqNfN-u2#Xs}Ov-kLUM1&)q%0-+N-UXvQ`c zbmh=`z?P4CKNffO|%@k6YE3Jet#;(laOM8FWj60sf`TD!yHu-omT4S68dj>hAr zno6yu@r1TeZz$Fm>ct*p$nU%fBK8KDXJYgF3Ge7w)hs%}Jq2Y7uFVK@<69ohCmH`< z?}S@0MGWX9r>KP(ky{AU_25}`3-a%3pkkdCEV^E|EYqr%>?>-g4?O=*qmkrfgo20h zWWL_AD+8Vw={C|(zLC<7eoKtFhNWWKw9qgal?!4<>Km9;7g9N#8^+_eu+||U6xGrw zR0>n6D=tl$qd31@OFX7xxW8H$K7=+OZGNF{#Y$MtDs917Vv3hOx!>)Jg|z}~5VzBe ze)5V_e&G)H!$wX@WA(TDe%xDgW6yd@v{MHuLk>%{Gj3k7(}tdK#-b6tQWuXMaOH7$ z$+u$sx5Ue41ar8|_SLX(GQ}LiLGX^Wjc|MFwtnDC2>o}ON^pzR-+wZ_utP`^0qV54 zqK-KmMng@0Pm-5=NTEH<#(Nly92+UkCA;Y7n8D^Lv+imFf4?2MtiYYVc(fi=W@XRi z!`g>QLlJm7M-rybWGLiPbnqyEGtCv;U4AjW4FYRL OicU7ItNrH{KmGxKxF`ky literal 0 HcmV?d00001 diff --git a/crates/j2k-native/fixtures/htj2k/rgb_u12_53.oracle.raw b/crates/j2k-native/fixtures/htj2k/rgb_u12_53.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..a074d69e97f8b5119db7a841f99fc654b5fd516e GIT binary patch literal 1482 zcmWlYeN2^Q7>D2c>v`T6GdmF)Dq}}MC09vFgr?yKDk$QaS#xB@963V7BUv&dqcm5| z5)@F!v}2K)5<4XV5;G(+Th5_#iHLY=eo!hN8Kbt;p7+n|pZj-T&+mRLtH8Qw#ardp z11rO7u;QT5x@4up4hvv4G+E-85I`c6`Y7WQD_mi;1jeC|G+HTjhfz~bgR`$0l(F2<-jh>10RFSL2$VfTn2PG z0bTZ?%f;w&09`(XE{DN|Fv{#dic+x+=TJW0Mn6@W`PI~lv*{$=#6r4D zBWO3p@Jwo_d0aqusEBvcDBi#!BFup9fJ1cy)z z9>NrwkDa*ESZJ!C{g^~;bRJhwn8@rO%ZYTHGI=@Oq7A&&%s=~uI40}%%}Mc$>sZ8C6)ZN{2~S5!Fmipnn^hi#TY{l z)}oK5VF)*qAEQhQHJEO;?!*%E@=yxW5}rX<>0@3;(cHoZja8Vtsg&cz!}YvW(zutm zN-@VstJJbj1|-CrB|$|=lPpt5A=m7F3#yGv9G=4uC>tY8O1WwLj_G|pjc1GM=rvBJ zYgECjO*g-B4ZXx9m*`WTB9VN8{gT1|@&PI3Nzx;Yyi$g%KHejlYMh*xO=_7uq?*+Z znWgTk2hfB?l!Eplb5e!fU?<3T_z=)qWBC~P!>CK#<%+wbM_-8Jr>Bi-#xr0>cr$&=9i3j+Y6!0@LDBp0QOjg(Vu=rJ+^vbs?N5-kY z)gD=)Cg?8tLgnhmRkzxwm#C4tN0+Nt^k};bb10EJjFF!`=FbG#XAHw!Mmd})9TYIT zdpN{F@v+yqlyQzEstyjA6pw@?sC?p8VO1tMDp7YxK>4*tg;Y@clvjs!naa_Lc83aR zzwOZ>9kefEJ#FI{`iIW(>ok>z%a`P%-FWPrN3Ri+vvtxC-eWU7-~YkC+e{buT7Nl~$SmuxgI)+6oekd9YFj7z@q z>0~>os!YnD>eg-cG(FaKoDX%Lo#mX;b#{d_%pS12oK>`p7fL;SZT`!B>fks4L0@5yb|qvxn(v-{`jee+@$)G2+;_UcEC%bPme dUf_JMzcMNR>09, component_planes: Vec, + compressed_payload: Vec, + classic_workspace: J2kCodeBlockDecodeWorkspace, + ht_workspace: HtCodeBlockDecodeWorkspace, + staged_state: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StagedDirectRoute { + Classic, + Htj2k, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct StagedDirectState { + route: StagedDirectRoute, + next_tile: usize, + active_tile: Option, + tile_count: usize, } impl J2kDirectCpuScratch { @@ -30,6 +75,10 @@ impl J2kDirectCpuScratch { Self { component_band_sets: Vec::new(), component_planes: Vec::new(), + compressed_payload: Vec::new(), + classic_workspace: J2kCodeBlockDecodeWorkspace::empty(), + ht_workspace: HtCodeBlockDecodeWorkspace::empty(), + staged_state: None, } } @@ -38,6 +87,35 @@ impl J2kDirectCpuScratch { *self = Self::new(); } + /// Bytes retained by the parse-free HT code-block workspace. + #[doc(hidden)] + #[must_use] + pub fn retained_ht_workspace_bytes(&self) -> usize { + self.ht_workspace.allocated_bytes().unwrap_or(usize::MAX) + } + + /// Bytes retained by the parse-free classic code-block workspace. + #[doc(hidden)] + #[must_use] + pub fn retained_classic_workspace_bytes(&self) -> usize { + self.classic_workspace + .allocated_bytes() + .unwrap_or(usize::MAX) + } + + /// Number of coefficient/IDWT buffer owners retained by this scratch. + /// + /// Referenced multi-tile execution retains the maximum live tile shape, + /// not the sum of every tile's bands. + #[doc(hidden)] + #[must_use] + pub fn retained_band_owner_count(&self) -> usize { + self.component_band_sets + .iter() + .map(|component| component.bands.len()) + .sum() + } + /// Prepare retained scratch for `plan` and report the actual-capacity peak /// allocation for one execution, including the retained plan and temporary /// scalar code-block workspace. @@ -90,6 +168,9 @@ impl J2kDirectCpuScratch { band_sample_capacity, component_sample_len, component_sample_capacity, + compressed_payload_capacity: self.compressed_payload.capacity(), + classic_workspace_bytes: self.retained_classic_workspace_bytes(), + ht_workspace_bytes: self.retained_ht_workspace_bytes(), } } } @@ -104,6 +185,9 @@ struct DirectScratchAllocationProfile { band_sample_capacity: usize, component_sample_len: usize, component_sample_capacity: usize, + compressed_payload_capacity: usize, + classic_workspace_bytes: usize, + ht_workspace_bytes: usize, } #[derive(Debug, Default)] @@ -169,604 +253,34 @@ struct DirectComponentPlane { samples: Vec, } -/// Execute a adapter direct RGB plan on the CPU and write an RGB8 output region. -/// -/// # Errors -/// -/// Returns an error for invalid plan geometry, output bounds, or decode-stage failure. -pub fn execute_direct_color_plan_rgb8_into( - plan: &J2kDirectColorPlan, - output_region: J2kRect, - scratch: &mut J2kDirectCpuScratch, - out: &mut [u8], - stride: usize, -) -> Result<()> { - execute_direct_color_plan_u8_into( - plan, - output_region, - scratch, - out, - stride, - DirectColorU8Output::Rgb8, - ) -} - -/// Execute a adapter direct RGB plan on the CPU and write an RGBA8 output region. -/// -/// # Errors -/// -/// Returns an error for invalid plan geometry, output bounds, or decode-stage failure. -pub fn execute_direct_color_plan_rgba8_into( - plan: &J2kDirectColorPlan, - output_region: J2kRect, - scratch: &mut J2kDirectCpuScratch, - out: &mut [u8], - stride: usize, -) -> Result<()> { - execute_direct_color_plan_u8_into( - plan, - output_region, - scratch, - out, - stride, - DirectColorU8Output::Rgba8, - ) -} - -#[derive(Clone, Copy)] -enum DirectColorU8Output { - Rgb8, - Rgba8, -} - -impl DirectColorU8Output { - const fn bytes_per_pixel(self) -> usize { - match self { - Self::Rgb8 => 3, - Self::Rgba8 => 4, - } - } -} - -fn execute_direct_color_plan_u8_into( - plan: &J2kDirectColorPlan, - output_region: J2kRect, - scratch: &mut J2kDirectCpuScratch, - out: &mut [u8], - stride: usize, - output: DirectColorU8Output, -) -> Result<()> { - if plan.component_plans.len() != 3 { - bail!(DecodingError::UnsupportedFeature( - "direct CPU color plan requires three components" - )); - } - validate_output_region(plan, output_region, out.len(), stride, output)?; - - let workspace_budget = prepare_direct_scratch(plan, scratch)?; - for (component_index, component_plan) in plan.component_plans.iter().enumerate() { - let band_scratch = &mut scratch.component_band_sets[component_index]; - let plane = &mut scratch.component_planes[component_index]; - execute_component_plan(component_plan, band_scratch, plane, workspace_budget)?; - } - - let [plane0, plane1, plane2, ..] = scratch.component_planes.as_mut_slice() else { - bail!(DecodingError::CodeBlockDecodeFailure); - }; - if plan.mct { - apply_inverse_mct(plan.transform, plan.bit_depths, plane0, plane1, plane2)?; - } - write_rgb8_region( - [plane0, plane1, plane2], - plan.bit_depths, - output_region, - out, - stride, - output, - ) -} - -fn execute_component_plan( - plan: &J2kDirectGrayscalePlan, - bands: &mut DirectComponentBandScratch, - output: &mut DirectComponentPlane, - workspace_budget: DirectWorkspaceBudget, -) -> Result<()> { - bands.reset(); - let mut output_written = false; - - for step in &plan.steps { - match step { - J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { - execute_classic_sub_band(sub_band, bands, workspace_budget)?; - } - J2kDirectGrayscaleStep::HtSubBand(sub_band) => { - execute_ht_sub_band(sub_band, bands, workspace_budget)?; - } - J2kDirectGrayscaleStep::Idwt(step) => { - execute_idwt_step(step, bands)?; - } - J2kDirectGrayscaleStep::Store(store) => { - store_component(store, bands.active(), output, &mut output_written)?; - } - } - } - - if output_written { - Ok(()) - } else { - Err(DecodingError::CodeBlockDecodeFailure.into()) - } -} - -fn execute_classic_sub_band( - plan: &J2kOwnedSubBandPlan, - bands: &mut DirectComponentBandScratch, - workspace_budget: DirectWorkspaceBudget, -) -> Result<()> { - let (output, sub_band_width) = - prepare_sub_band_output(bands, plan.band_id, plan.rect, plan.width, plan.height)?; - let mut workspace = J2kCodeBlockDecodeWorkspace::default(); - if let Some((width, height)) = max_classic_job_dimensions(plan) { - workspace.prepare(width, height)?; - workspace_budget.validate_workspace(workspace.allocated_bytes()?)?; - } - - for job in &plan.jobs { - let output_range = checked_sub_band_job_output_range(&SubBandJobOutputRange { - output_x: job.output_x, - output_y: job.output_y, - output_stride: job.output_stride, - width: job.width, - height: job.height, - sub_band_width, - plan_width: plan.width, - plan_height: plan.height, - output_len: output.len(), - })?; - - let code_block = J2kCodeBlockDecodeJob { - data: &job.data, - segments: &job.segments, - width: job.width, - height: job.height, - output_stride: job.output_stride, - missing_bit_planes: job.missing_bit_planes, - number_of_coding_passes: job.number_of_coding_passes, - total_bitplanes: job.total_bitplanes, - roi_shift: job.roi_shift, - sub_band_type: job.sub_band_type, - style: job.style, - strict: job.strict, - dequantization_step: job.dequantization_step, - }; - decode_j2k_code_block_scalar_with_workspace( - code_block, - &mut output[output_range], - &mut workspace, - )?; - } - Ok(()) -} - -fn execute_ht_sub_band( - plan: &HtOwnedSubBandPlan, - bands: &mut DirectComponentBandScratch, - workspace_budget: DirectWorkspaceBudget, -) -> Result<()> { - let (output, sub_band_width) = - prepare_sub_band_output(bands, plan.band_id, plan.rect, plan.width, plan.height)?; - let mut workspace = HtCodeBlockDecodeWorkspace::default(); - if let Some((width, height)) = max_ht_job_dimensions(plan) { - workspace.prepare(width, height)?; - workspace_budget.validate_workspace(workspace.allocated_bytes()?)?; - } - - for job in &plan.jobs { - let output_range = checked_sub_band_job_output_range(&SubBandJobOutputRange { - output_x: job.output_x, - output_y: job.output_y, - output_stride: job.output_stride, - width: job.width, - height: job.height, - sub_band_width, - plan_width: plan.width, - plan_height: plan.height, - output_len: output.len(), - })?; - - let code_block = HtCodeBlockDecodeJob { - data: &job.data, - cleanup_length: job.cleanup_length, - refinement_length: job.refinement_length, - width: job.width, - height: job.height, - output_stride: job.output_stride, - missing_bit_planes: job.missing_bit_planes, - number_of_coding_passes: job.number_of_coding_passes, - num_bitplanes: job.num_bitplanes, - roi_shift: job.roi_shift, - stripe_causal: job.stripe_causal, - strict: job.strict, - dequantization_step: job.dequantization_step, - }; - decode_ht_code_block_scalar_with_workspace( - code_block, - &mut output[output_range], - &mut workspace, - )?; - } - Ok(()) -} - -fn max_classic_job_dimensions(plan: &J2kOwnedSubBandPlan) -> Option<(u32, u32)> { - plan.jobs.iter().fold(None, |dimensions, job| { - Some( - dimensions.map_or((job.width, job.height), |(width, height)| { - (width.max(job.width), height.max(job.height)) - }), - ) - }) -} - -fn max_ht_job_dimensions(plan: &HtOwnedSubBandPlan) -> Option<(u32, u32)> { - plan.jobs.iter().fold(None, |dimensions, job| { - Some( - dimensions.map_or((job.width, job.height), |(width, height)| { - (width.max(job.width), height.max(job.height)) - }), - ) - }) -} - -fn prepare_sub_band_output( - bands: &mut DirectComponentBandScratch, - band_id: J2kDirectBandId, - rect: J2kRect, - width: u32, - height: u32, -) -> Result<(&mut [f32], usize)> { - let required_len = checked_area(width, height)?; - let band_index = bands.prepare_band(band_id, rect, required_len)?; - let output = bands.bands[band_index].coefficients.as_mut_slice(); - let sub_band_width = - usize::try_from(width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - Ok((output, sub_band_width)) -} - -fn execute_idwt_step( - step: &J2kDirectIdwtStep, - bands: &mut DirectComponentBandScratch, -) -> Result<()> { - let output_index = bands.prepare_band(step.output_band_id, step.rect, 0)?; - let (input_bands, output_bands) = bands.bands.split_at_mut(output_index); - let output = &mut output_bands[0].coefficients; - let ll = find_idwt_band(input_bands, step.ll_band_id)?; - let hl = find_idwt_band(input_bands, step.hl_band_id)?; - let lh = find_idwt_band(input_bands, step.lh_band_id)?; - let hh = find_idwt_band(input_bands, step.hh_band_id)?; - let job = J2kSingleDecompositionIdwtJob { - rect: step.rect, - transform: step.transform, - ll, - hl, - lh, - hh, - }; - idwt::apply_single_decomposition_idwt_job(job, output) -} - -fn find_idwt_band(bands: &[DirectCpuBand], band_id: J2kDirectBandId) -> Result> { - let band = find_band(bands, band_id)?; - Ok(J2kIdwtBand { - rect: band.rect, - coefficients: &band.coefficients, - }) -} - -fn store_component( - store: &J2kDirectStoreStep, - bands: &[DirectCpuBand], - plane: &mut DirectComponentPlane, - output_written: &mut bool, -) -> Result<()> { - let input = find_band(bands, store.input_band_id)?; - if !*output_written { - plane.width = store.output_width; - plane.height = store.output_height; - let required_len = checked_area(store.output_width, store.output_height)?; - resize_and_zero(&mut plane.samples, required_len)?; - *output_written = true; - } - if plane.width != store.output_width - || plane.height != store.output_height - || plane.samples.len() != checked_area(store.output_width, store.output_height)? - { - bail!(DecodingError::CodeBlockDecodeFailure); - } - - validate_store_bounds(store, input, plane)?; - let input_width = input.rect.width() as usize; - let output_width = plane.width as usize; - let copy_width = store.copy_width as usize; - for row in 0..store.copy_height as usize { - let src_start = (store.source_y as usize + row) - .checked_mul(input_width) - .and_then(|base| base.checked_add(store.source_x as usize)) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let dst_start = (store.output_y as usize + row) - .checked_mul(output_width) - .and_then(|base| base.checked_add(store.output_x as usize)) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let src = &input.coefficients[src_start..src_start + copy_width]; - let dst = &mut plane.samples[dst_start..dst_start + copy_width]; - for (src, dst) in src.iter().zip(dst.iter_mut()) { - *dst = *src + store.addend; - } - } - Ok(()) -} - -fn find_band(bands: &[DirectCpuBand], band_id: J2kDirectBandId) -> Result<&DirectCpuBand> { - bands - .iter() - .find(|band| band.band_id == band_id) - .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) -} - -fn validate_store_bounds( - store: &J2kDirectStoreStep, - input: &DirectCpuBand, - output: &DirectComponentPlane, -) -> Result<()> { - if store - .source_x - .checked_add(store.copy_width) - .is_none_or(|x| x > input.rect.width()) - || store - .source_y - .checked_add(store.copy_height) - .is_none_or(|y| y > input.rect.height()) - || store - .output_x - .checked_add(store.copy_width) - .is_none_or(|x| x > output.width) - || store - .output_y - .checked_add(store.copy_height) - .is_none_or(|y| y > output.height) - { - bail!(DecodingError::CodeBlockDecodeFailure); - } - Ok(()) -} - -fn apply_inverse_mct( - transform: J2kWaveletTransform, - bit_depths: [u8; 3], - plane0: &mut DirectComponentPlane, - plane1: &mut DirectComponentPlane, - plane2: &mut DirectComponentPlane, -) -> Result<()> { - if plane0.width != plane1.width - || plane1.width != plane2.width - || plane0.height != plane1.height - || plane1.height != plane2.height - || plane0.samples.len() != plane1.samples.len() - || plane1.samples.len() != plane2.samples.len() - { - bail!(DecodingError::CodeBlockDecodeFailure); - } - - let addend0 = sign_addend(bit_depths[0]); - let addend1 = sign_addend(bit_depths[1]); - let addend2 = sign_addend(bit_depths[2]); - for ((y0, y1), y2) in plane0 - .samples - .iter_mut() - .zip(plane1.samples.iter_mut()) - .zip(plane2.samples.iter_mut()) - { - let src0 = *y0; - let src1 = *y1; - let src2 = *y2; - let (out0, out1, out2) = match transform { - J2kWaveletTransform::Irreversible97 => ( - src0 + 1.402 * src2, - src0 - 0.34413 * src1 - 0.71414 * src2, - src0 + 1.772 * src1, - ), - J2kWaveletTransform::Reversible53 => { - let i1 = src0 - floor_f32((src2 + src1) * 0.25); - (src2 + i1, i1, src1 + i1) - } - }; - *y0 = out0 + addend0; - *y1 = out1 + addend1; - *y2 = out2 + addend2; - } - Ok(()) -} - -fn write_rgb8_region( - planes: [&DirectComponentPlane; 3], - bit_depths: [u8; 3], - output_region: J2kRect, - out: &mut [u8], - stride: usize, - output: DirectColorU8Output, -) -> Result<()> { - let width = output_region.width() as usize; - let height = output_region.height() as usize; - let bytes_per_pixel = output.bytes_per_pixel(); - let row_bytes = width - .checked_mul(bytes_per_pixel) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - for plane in planes { - if output_region.x1 > plane.width || output_region.y1 > plane.height { - bail!(DecodingError::CodeBlockDecodeFailure); - } - } - - for y in 0..height { - let src_y = output_region.y0 as usize + y; - let dst = &mut out[y * stride..y * stride + row_bytes]; - for x in 0..width { - let src_x = output_region.x0 as usize + x; - let dst = &mut dst[x * bytes_per_pixel..x * bytes_per_pixel + bytes_per_pixel]; - for channel in 0..3 { - let plane = planes[channel]; - let sample = plane.samples[src_y * plane.width as usize + src_x]; - dst[channel] = sample_as_u8(sample, bit_depths[channel]); - } - if matches!(output, DirectColorU8Output::Rgba8) { - dst[3] = u8::MAX; - } - } - } - Ok(()) -} - -fn validate_output_region( - plan: &J2kDirectColorPlan, - output_region: J2kRect, - out_len: usize, - stride: usize, - output: DirectColorU8Output, -) -> Result<()> { - if output_region.x1 > plan.dimensions.0 - || output_region.y1 > plan.dimensions.1 - || output_region.x0 > output_region.x1 - || output_region.y0 > output_region.y1 - { - bail!(DecodingError::CodeBlockDecodeFailure); - } - let bytes_per_pixel = u32::try_from(output.bytes_per_pixel()) - .map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - let row_bytes = output_region - .width() - .checked_mul(bytes_per_pixel) - .and_then(|len| usize::try_from(len).ok()) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - if stride < row_bytes { - bail!(DecodingError::CodeBlockDecodeFailure); - } - let height = usize::try_from(output_region.height()) - .map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - let required = if height == 0 { - 0 - } else { - stride - .checked_mul(height - 1) - .and_then(|prefix| prefix.checked_add(row_bytes)) - .ok_or(DecodingError::CodeBlockDecodeFailure)? - }; - if out_len < required { - bail!(DecodingError::CodeBlockDecodeFailure); - } - Ok(()) -} - -fn checked_area(width: u32, height: u32) -> Result { - let height = usize::try_from(height).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - usize::try_from(width) - .ok() - .and_then(|width| width.checked_mul(height)) - .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) -} - -fn checked_block_base(output_x: u32, output_y: u32, stride: usize) -> Result { - let output_x = usize::try_from(output_x).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - usize::try_from(output_y) - .ok() - .and_then(|y| y.checked_mul(stride)) - .and_then(|base| base.checked_add(output_x)) - .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) -} - -struct SubBandJobOutputRange { - output_x: u32, - output_y: u32, - output_stride: usize, - width: u32, - height: u32, - sub_band_width: usize, - plan_width: u32, - plan_height: u32, - output_len: usize, -} - -fn checked_sub_band_job_output_range(bounds: &SubBandJobOutputRange) -> Result> { - let base_idx = checked_block_base(bounds.output_x, bounds.output_y, bounds.sub_band_width)?; - let block_len = checked_block_output_len(bounds.output_stride, bounds.width, bounds.height)?; - let end_idx = base_idx - .checked_add(block_len) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - if end_idx > bounds.output_len - || bounds - .output_x - .checked_add(bounds.width) - .is_none_or(|x| x > bounds.plan_width) - || bounds - .output_y - .checked_add(bounds.height) - .is_none_or(|y| y > bounds.plan_height) - { - bail!(DecodingError::CodeBlockDecodeFailure); - } - Ok(base_idx..end_idx) -} - -fn checked_block_output_len(stride: usize, width: u32, height: u32) -> Result { - if height == 0 { - return Ok(0); - } - let height = usize::try_from(height).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - let width = usize::try_from(width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - stride - .checked_mul(height - 1) - .and_then(|prefix| prefix.checked_add(width)) - .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) -} - -fn resize_and_zero(buffer: &mut Vec, len: usize) -> Result<()> { - try_resize_decode_elements(buffer, len, 0.0)?; - buffer.fill(0.0); - Ok(()) -} - -#[expect( - clippy::cast_precision_loss, - reason = "direct CPU color math intentionally uses the decoder's f32 sample representation" -)] -fn sign_addend(bit_depth: u8) -> f32 { - (1_u32 << (bit_depth - 1)) as f32 -} - -#[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - reason = "samples are rounded and clamped before stable 8-bit output quantization" -)] -fn sample_as_u8(sample: f32, bit_depth: u8) -> u8 { - let rounded = round_f32(sample); - if bit_depth == 8 { - return rounded.clamp(0.0, f32::from(u8::MAX)) as u8; - } - let max_value = if bit_depth >= 16 { - f32::from(u16::MAX) - } else { - f32::from(((1_u16 << bit_depth) - 1).max(1)) - }; - round_f32((rounded.clamp(0.0, max_value) / max_value) * f32::from(u8::MAX)) as u8 -} - #[cfg(test)] mod tests { use super::*; use crate::{encode_htj2k, DecodeSettings, DecoderContext, EncodeOptions, Image}; use alloc::vec; + const EMPTY_DIRECT_CPU_SCRATCH: J2kDirectCpuScratch = J2kDirectCpuScratch::new(); + + #[test] + fn direct_cpu_scratch_constructor_remains_const() { + let scratch = EMPTY_DIRECT_CPU_SCRATCH; + assert_eq!( + scratch.allocation_profile_for_tests(), + DirectScratchAllocationProfile { + component_band_sets: 0, + component_planes: 0, + band_buffers: 0, + band_sample_len: 0, + band_sample_capacity: 0, + component_sample_len: 0, + component_sample_capacity: 0, + compressed_payload_capacity: 0, + classic_workspace_bytes: 0, + ht_workspace_bytes: 0, + } + ); + } + fn direct_htj2k_rgb_plan() -> (J2kDirectColorPlan, J2kRect) { let pixels = (0..16 * 16 * 3) .map(|idx| { @@ -851,6 +365,9 @@ mod tests { band_sample_capacity: 0, component_sample_len: 0, component_sample_capacity: 0, + compressed_payload_capacity: 0, + classic_workspace_bytes: 0, + ht_workspace_bytes: 0, } ); assert_eq!(scratch.component_band_sets.capacity(), 0); diff --git a/crates/j2k-native/src/direct_cpu/allocation.rs b/crates/j2k-native/src/direct_cpu/allocation.rs index 2198348d..707416d1 100644 --- a/crates/j2k-native/src/direct_cpu/allocation.rs +++ b/crates/j2k-native/src/direct_cpu/allocation.rs @@ -6,21 +6,60 @@ use super::{ checked_area, DirectComponentBandScratch, DirectComponentPlane, DirectCpuBand, J2kDirectCpuScratch, }; -use crate::error::{DecodeError, Result, ValidationError}; +use crate::error::{DecodeError, DecodingError, Result, ValidationError}; use crate::j2c::{bitplane, ht_block_decode}; use crate::{ try_reserve_decode_elements, J2kDirectColorPlan, J2kDirectGrayscalePlan, - J2kDirectGrayscaleStep, DEFAULT_MAX_DECODE_BYTES, + J2kDirectGrayscaleStep, J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, + DEFAULT_MAX_DECODE_BYTES, }; use alloc::vec::Vec; use core::mem::size_of; +mod referenced; +pub(super) use self::referenced::{ + max_referenced_classic_dimensions, max_referenced_ht_dimensions, + prepare_referenced_classic_scratch, prepare_referenced_classic_staged_scratch, + prepare_referenced_direct_scratch, prepare_referenced_htj2k_staged_scratch, +}; + pub(super) fn prepare_direct_scratch( plan: &J2kDirectColorPlan, scratch: &mut J2kDirectCpuScratch, ) -> Result { - normalize_retained_scratch(plan, scratch)?; - if let Err(error) = validate_aggregate_plan(plan, scratch) { + prepare_component_scratch( + &plan.component_plans, + plan.retained_allocation_bytes()?, + 0, + None, + None, + scratch, + ) +} + +fn prepare_component_scratch( + components: &[J2kDirectGrayscalePlan], + retained_plan_bytes: usize, + compressed_payload_bytes: usize, + retained_classic_workspace_dimensions: Option<(u32, u32)>, + retained_ht_workspace_dimensions: Option<(u32, u32)>, + scratch: &mut J2kDirectCpuScratch, +) -> Result { + normalize_retained_scratch( + components, + compressed_payload_bytes, + retained_classic_workspace_dimensions.is_some(), + retained_ht_workspace_dimensions.is_some(), + scratch, + )?; + if let Err(error) = validate_aggregate_plan( + components, + retained_plan_bytes, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + ) { if !matches!( error, DecodeError::Validation(ValidationError::ImageTooLarge) @@ -30,14 +69,34 @@ pub(super) fn prepare_direct_scratch( // Retention is optional. Retry from an empty owner so prior larger // plans cannot make the current logical request fail. scratch.clear(); - validate_aggregate_plan(plan, scratch)?; + validate_aggregate_plan( + components, + retained_plan_bytes, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + )?; } - if let Err(error) = reserve_scratch(plan, scratch) { + if let Err(error) = reserve_scratch( + components, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + ) { scratch.clear(); return Err(error); } - match validate_aggregate_plan(plan, scratch) { + match validate_aggregate_plan( + components, + retained_plan_bytes, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + ) { Ok(workspace_budget) => Ok(workspace_budget), Err(error) => { scratch.clear(); @@ -70,10 +129,13 @@ impl DirectWorkspaceBudget { } fn normalize_retained_scratch( - plan: &J2kDirectColorPlan, + components: &[J2kDirectGrayscalePlan], + compressed_payload_bytes: usize, + retain_classic_workspace: bool, + retain_ht_workspace: bool, scratch: &mut J2kDirectCpuScratch, ) -> Result<()> { - let component_count = plan.component_plans.len(); + let component_count = components.len(); normalize_outer_owner(&mut scratch.component_band_sets, component_count); normalize_outer_owner(&mut scratch.component_planes, component_count); @@ -88,7 +150,7 @@ fn normalize_retained_scratch( DirectComponentPlane::default, ); - for (component_idx, component_plan) in plan.component_plans.iter().enumerate() { + for (component_idx, component_plan) in components.iter().enumerate() { let band_count = component_band_count(component_plan)?; if let Some(component) = scratch.component_band_sets.get_mut(component_idx) { normalize_outer_owner(&mut component.bands, band_count); @@ -113,6 +175,16 @@ fn normalize_retained_scratch( } } } + scratch.compressed_payload.clear(); + if scratch.compressed_payload.capacity() < compressed_payload_bytes { + scratch.compressed_payload = Vec::new(); + } + if !retain_classic_workspace { + scratch.classic_workspace = crate::J2kCodeBlockDecodeWorkspace::default(); + } + if !retain_ht_workspace { + scratch.ht_workspace = crate::HtCodeBlockDecodeWorkspace::default(); + } Ok(()) } @@ -135,12 +207,24 @@ fn fill_without_allocation( } fn validate_aggregate_plan( - plan: &J2kDirectColorPlan, + components: &[J2kDirectGrayscalePlan], + retained_plan_bytes: usize, + compressed_payload_bytes: usize, + retained_classic_workspace_dimensions: Option<(u32, u32)>, + retained_ht_workspace_dimensions: Option<(u32, u32)>, scratch: &J2kDirectCpuScratch, ) -> Result { let mut budget = DirectAllocationBudget::default(); - let temporary_workspace_bytes = include_plan_allocations(&mut budget, plan)?; - include_scratch_allocations(&mut budget, plan, scratch)?; + let temporary_workspace_bytes = + include_plan_allocations(&mut budget, components, retained_plan_bytes)?; + include_scratch_allocations( + &mut budget, + components, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + )?; let base_bytes = budget.bytes; budget.include_bytes(temporary_workspace_bytes)?; Ok(DirectWorkspaceBudget { @@ -176,12 +260,13 @@ impl DirectAllocationBudget { fn include_plan_allocations( budget: &mut DirectAllocationBudget, - plan: &J2kDirectColorPlan, + components: &[J2kDirectGrayscalePlan], + retained_plan_bytes: usize, ) -> Result { - budget.include_bytes(plan.retained_allocation_bytes()?)?; + budget.include_bytes(retained_plan_bytes)?; let mut classic_dimensions = None; let mut ht_dimensions = None; - for component in &plan.component_plans { + for component in components { for step in &component.steps { match step { J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { @@ -217,10 +302,13 @@ fn observe_max_dimensions(target: &mut Option<(u32, u32)>, width: u32, height: u fn include_scratch_allocations( budget: &mut DirectAllocationBudget, - plan: &J2kDirectColorPlan, + components: &[J2kDirectGrayscalePlan], + compressed_payload_bytes: usize, + retained_classic_workspace_dimensions: Option<(u32, u32)>, + retained_ht_workspace_dimensions: Option<(u32, u32)>, scratch: &J2kDirectCpuScratch, ) -> Result<()> { - let component_count = plan.component_plans.len(); + let component_count = components.len(); budget.include_capacity::( scratch.component_band_sets.capacity().max(component_count), )?; @@ -228,7 +316,7 @@ fn include_scratch_allocations( scratch.component_planes.capacity().max(component_count), )?; - for (component_idx, component_plan) in plan.component_plans.iter().enumerate() { + for (component_idx, component_plan) in components.iter().enumerate() { let band_count = component_band_count(component_plan)?; let component = scratch.component_band_sets.get(component_idx); budget.include_capacity::( @@ -250,11 +338,35 @@ fn include_scratch_allocations( .map_or(0, |plane| plane.samples.capacity()); budget.include_capacity::(retained_capacity.max(plane_len))?; } + budget.include_capacity::( + scratch + .compressed_payload + .capacity() + .max(compressed_payload_bytes), + )?; + let retained_classic_workspace_bytes = scratch.classic_workspace.allocated_bytes()?; + let target_classic_workspace_bytes = retained_classic_workspace_dimensions + .map_or(Ok(0), |(width, height)| { + bitplane::classic_decode_workspace_bytes(width, height) + })?; + budget.include_bytes(retained_classic_workspace_bytes.max(target_classic_workspace_bytes))?; + let retained_ht_workspace_bytes = scratch.ht_workspace.allocated_bytes()?; + let target_ht_workspace_bytes = retained_ht_workspace_dimensions + .map_or(Ok(0), |(width, height)| { + ht_block_decode::ht_decode_workspace_bytes(width, height) + })?; + budget.include_bytes(retained_ht_workspace_bytes.max(target_ht_workspace_bytes))?; Ok(()) } -fn reserve_scratch(plan: &J2kDirectColorPlan, scratch: &mut J2kDirectCpuScratch) -> Result<()> { - let component_count = plan.component_plans.len(); +fn reserve_scratch( + components: &[J2kDirectGrayscalePlan], + compressed_payload_bytes: usize, + retained_classic_workspace_dimensions: Option<(u32, u32)>, + retained_ht_workspace_dimensions: Option<(u32, u32)>, + scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + let component_count = components.len(); try_reserve_decode_elements(&mut scratch.component_band_sets, component_count)?; try_reserve_decode_elements(&mut scratch.component_planes, component_count)?; while scratch.component_band_sets.len() < component_count { @@ -268,7 +380,7 @@ fn reserve_scratch(plan: &J2kDirectColorPlan, scratch: &mut J2kDirectCpuScratch) .push(DirectComponentPlane::default()); } - for (component_idx, component_plan) in plan.component_plans.iter().enumerate() { + for (component_idx, component_plan) in components.iter().enumerate() { let component = &mut scratch.component_band_sets[component_idx]; let band_count = component_band_count(component_plan)?; try_reserve_decode_elements(&mut component.bands, band_count)?; @@ -285,6 +397,13 @@ fn reserve_scratch(plan: &J2kDirectColorPlan, scratch: &mut J2kDirectCpuScratch) plane_len, )?; } + try_reserve_decode_elements(&mut scratch.compressed_payload, compressed_payload_bytes)?; + if let Some((width, height)) = retained_classic_workspace_dimensions { + scratch.classic_workspace.prepare(width, height)?; + } + if let Some((width, height)) = retained_ht_workspace_dimensions { + scratch.ht_workspace.prepare(width, height)?; + } Ok(()) } diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs new file mode 100644 index 00000000..7911aed4 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Scratch sizing and retained workspace accounting for referenced plans. + +mod budget; +mod storage; +mod view; + +pub(in super::super) use view::{max_referenced_classic_dimensions, max_referenced_ht_dimensions}; + +use super::{ + DecodeError, DirectWorkspaceBudget, J2kDirectCpuScratch, J2kReferencedClassicPlan, + J2kReferencedHtj2kPlan, Result, ValidationError, +}; +use budget::validate_referenced_aggregate_plan; +use storage::{normalize_referenced_scratch, reserve_referenced_scratch}; +use view::{ + max_referenced_classic_payload_bytes, max_referenced_payload_bytes, ReferencedPlanView, +}; + +pub(in super::super) fn prepare_referenced_direct_scratch( + plan: &J2kReferencedHtj2kPlan, + scratch: &mut J2kDirectCpuScratch, +) -> Result { + let view = ReferencedPlanView::Htj2k(plan); + prepare_referenced_component_scratch( + view, + plan.retained_allocation_bytes()?, + max_referenced_payload_bytes(plan)?, + None, + max_referenced_ht_dimensions(plan), + scratch, + ) +} + +pub(in super::super) fn prepare_referenced_classic_scratch( + plan: &J2kReferencedClassicPlan, + scratch: &mut J2kDirectCpuScratch, +) -> Result { + let view = ReferencedPlanView::Classic(plan); + prepare_referenced_component_scratch( + view, + plan.retained_allocation_bytes()?, + max_referenced_classic_payload_bytes(plan), + max_referenced_classic_dimensions(plan), + None, + scratch, + ) +} + +pub(in super::super) fn prepare_referenced_htj2k_staged_scratch( + plan: &J2kReferencedHtj2kPlan, + scratch: &mut J2kDirectCpuScratch, +) -> Result { + prepare_referenced_component_scratch( + ReferencedPlanView::Htj2k(plan), + plan.retained_allocation_bytes()?, + 0, + None, + None, + scratch, + ) +} + +pub(in super::super) fn prepare_referenced_classic_staged_scratch( + plan: &J2kReferencedClassicPlan, + scratch: &mut J2kDirectCpuScratch, +) -> Result { + prepare_referenced_component_scratch( + ReferencedPlanView::Classic(plan), + plan.retained_allocation_bytes()?, + 0, + None, + None, + scratch, + ) +} + +fn prepare_referenced_component_scratch( + plan: ReferencedPlanView<'_>, + retained_plan_bytes: usize, + compressed_payload_bytes: usize, + retained_classic_workspace_dimensions: Option<(u32, u32)>, + retained_ht_workspace_dimensions: Option<(u32, u32)>, + scratch: &mut J2kDirectCpuScratch, +) -> Result { + normalize_referenced_scratch( + plan, + compressed_payload_bytes, + retained_classic_workspace_dimensions.is_some(), + retained_ht_workspace_dimensions.is_some(), + scratch, + )?; + if let Err(error) = validate_referenced_aggregate_plan( + plan, + retained_plan_bytes, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + ) { + if !matches!( + error, + DecodeError::Validation(ValidationError::ImageTooLarge) + ) { + return Err(error); + } + scratch.clear(); + validate_referenced_aggregate_plan( + plan, + retained_plan_bytes, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + )?; + } + + if let Err(error) = reserve_referenced_scratch( + plan, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + ) { + scratch.clear(); + return Err(error); + } + match validate_referenced_aggregate_plan( + plan, + retained_plan_bytes, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + ) { + Ok(workspace_budget) => Ok(workspace_budget), + Err(error) => { + scratch.clear(); + Err(error) + } + } +} diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced/budget.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced/budget.rs new file mode 100644 index 00000000..88ba21c5 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced/budget.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::{ + bitplane, ht_block_decode, DirectAllocationBudget, DirectComponentBandScratch, + DirectComponentPlane, DirectCpuBand, DirectWorkspaceBudget, J2kDirectCpuScratch, Result, +}; +use super::view::{ + referenced_band_target, referenced_component_band_count, referenced_component_plane_len, + referenced_temporary_workspace_bytes, validate_referenced_shape, ReferencedPlanView, +}; + +pub(super) fn validate_referenced_aggregate_plan( + plan: ReferencedPlanView<'_>, + retained_plan_bytes: usize, + compressed_payload_bytes: usize, + retained_classic_workspace_dimensions: Option<(u32, u32)>, + retained_ht_workspace_dimensions: Option<(u32, u32)>, + scratch: &J2kDirectCpuScratch, +) -> Result { + validate_referenced_shape(plan)?; + let mut budget = DirectAllocationBudget::default(); + budget.include_bytes(retained_plan_bytes)?; + let temporary_workspace_bytes = referenced_temporary_workspace_bytes(plan)?; + include_referenced_scratch_allocations( + &mut budget, + plan, + compressed_payload_bytes, + retained_classic_workspace_dimensions, + retained_ht_workspace_dimensions, + scratch, + )?; + let base_bytes = budget.bytes; + budget.include_bytes(temporary_workspace_bytes)?; + Ok(DirectWorkspaceBudget { + base_bytes, + peak_bytes: budget.bytes, + }) +} + +fn include_referenced_scratch_allocations( + budget: &mut DirectAllocationBudget, + plan: ReferencedPlanView<'_>, + compressed_payload_bytes: usize, + retained_classic_workspace_dimensions: Option<(u32, u32)>, + retained_ht_workspace_dimensions: Option<(u32, u32)>, + scratch: &J2kDirectCpuScratch, +) -> Result<()> { + let component_count = plan.component_count(); + budget.include_capacity::( + scratch.component_band_sets.capacity().max(component_count), + )?; + budget.include_capacity::( + scratch.component_planes.capacity().max(component_count), + )?; + for component_index in 0..component_count { + let band_count = referenced_component_band_count(plan, component_index)?; + let component = scratch.component_band_sets.get(component_index); + budget.include_capacity::( + component + .map_or(0, |component| component.bands.capacity()) + .max(band_count), + )?; + for band_index in 0..band_count { + let target_len = referenced_band_target(plan, component_index, band_index)?; + let retained_capacity = component + .and_then(|component| component.bands.get(band_index)) + .map_or(0, |band| band.coefficients.capacity()); + budget.include_capacity::(retained_capacity.max(target_len))?; + } + let plane_len = referenced_component_plane_len(plan, component_index)?; + let retained_capacity = scratch + .component_planes + .get(component_index) + .map_or(0, |plane| plane.samples.capacity()); + budget.include_capacity::(retained_capacity.max(plane_len))?; + } + budget.include_capacity::( + scratch + .compressed_payload + .capacity() + .max(compressed_payload_bytes), + )?; + let retained_classic_workspace_bytes = scratch.classic_workspace.allocated_bytes()?; + let target_classic_workspace_bytes = retained_classic_workspace_dimensions + .map_or(Ok(0), |(width, height)| { + bitplane::classic_decode_workspace_bytes(width, height) + })?; + budget.include_bytes(retained_classic_workspace_bytes.max(target_classic_workspace_bytes))?; + let retained_ht_workspace_bytes = scratch.ht_workspace.allocated_bytes()?; + let target_ht_workspace_bytes = retained_ht_workspace_dimensions + .map_or(Ok(0), |(width, height)| { + ht_block_decode::ht_decode_workspace_bytes(width, height) + })?; + budget.include_bytes(retained_ht_workspace_bytes.max(target_ht_workspace_bytes)) +} diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced/storage.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced/storage.rs new file mode 100644 index 00000000..5124bd75 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced/storage.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::{ + fill_without_allocation, normalize_outer_owner, try_reserve_decode_elements, + DirectComponentBandScratch, DirectComponentPlane, DirectCpuBand, J2kDirectCpuScratch, Result, + Vec, +}; +use super::view::{ + referenced_band_target, referenced_component_band_count, referenced_component_plane_len, + validate_referenced_shape, ReferencedPlanView, +}; + +pub(super) fn normalize_referenced_scratch( + plan: ReferencedPlanView<'_>, + compressed_payload_bytes: usize, + retain_classic_workspace: bool, + retain_ht_workspace: bool, + scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + validate_referenced_shape(plan)?; + let component_count = plan.component_count(); + normalize_outer_owner(&mut scratch.component_band_sets, component_count); + normalize_outer_owner(&mut scratch.component_planes, component_count); + fill_without_allocation( + &mut scratch.component_band_sets, + component_count, + DirectComponentBandScratch::default, + ); + fill_without_allocation( + &mut scratch.component_planes, + component_count, + DirectComponentPlane::default, + ); + + for component_index in 0..component_count { + let band_count = referenced_component_band_count(plan, component_index)?; + if let Some(component) = scratch.component_band_sets.get_mut(component_index) { + normalize_outer_owner(&mut component.bands, band_count); + fill_without_allocation(&mut component.bands, band_count, DirectCpuBand::empty); + for band_index in 0..band_count { + let target_len = referenced_band_target(plan, component_index, band_index)?; + if let Some(band) = component.bands.get_mut(band_index) { + band.coefficients.clear(); + if band.coefficients.capacity() < target_len { + band.coefficients = Vec::new(); + } + } + } + component.active_len = 0; + } + + let plane_len = referenced_component_plane_len(plan, component_index)?; + if let Some(plane) = scratch.component_planes.get_mut(component_index) { + plane.samples.clear(); + if plane.samples.capacity() < plane_len { + plane.samples = Vec::new(); + } + } + } + scratch.compressed_payload.clear(); + if scratch.compressed_payload.capacity() < compressed_payload_bytes { + scratch.compressed_payload = Vec::new(); + } + if !retain_classic_workspace { + scratch.classic_workspace = crate::J2kCodeBlockDecodeWorkspace::default(); + } + if !retain_ht_workspace { + scratch.ht_workspace = crate::HtCodeBlockDecodeWorkspace::default(); + } + scratch.staged_state = None; + Ok(()) +} + +pub(super) fn reserve_referenced_scratch( + plan: ReferencedPlanView<'_>, + compressed_payload_bytes: usize, + retained_classic_workspace_dimensions: Option<(u32, u32)>, + retained_ht_workspace_dimensions: Option<(u32, u32)>, + scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + let component_count = plan.component_count(); + try_reserve_decode_elements(&mut scratch.component_band_sets, component_count)?; + try_reserve_decode_elements(&mut scratch.component_planes, component_count)?; + while scratch.component_band_sets.len() < component_count { + scratch + .component_band_sets + .push(DirectComponentBandScratch::default()); + } + while scratch.component_planes.len() < component_count { + scratch + .component_planes + .push(DirectComponentPlane::default()); + } + for component_index in 0..component_count { + let band_count = referenced_component_band_count(plan, component_index)?; + let component = &mut scratch.component_band_sets[component_index]; + try_reserve_decode_elements(&mut component.bands, band_count)?; + while component.bands.len() < band_count { + component.bands.push(DirectCpuBand::empty()); + } + for band_index in 0..band_count { + try_reserve_decode_elements( + &mut component.bands[band_index].coefficients, + referenced_band_target(plan, component_index, band_index)?, + )?; + } + try_reserve_decode_elements( + &mut scratch.component_planes[component_index].samples, + referenced_component_plane_len(plan, component_index)?, + )?; + } + try_reserve_decode_elements(&mut scratch.compressed_payload, compressed_payload_bytes)?; + if let Some((width, height)) = retained_classic_workspace_dimensions { + scratch.classic_workspace.prepare(width, height)?; + } + if let Some((width, height)) = retained_ht_workspace_dimensions { + scratch.ht_workspace.prepare(width, height)?; + } + Ok(()) +} diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs new file mode 100644 index 00000000..ef989f7b --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::super::{ + bitplane, checked_area, component_band_count, component_plane_len, ht_block_decode, + observe_max_dimensions, DecodingError, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, + J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, Result, ValidationError, +}; + +#[derive(Clone, Copy)] +pub(super) enum ReferencedPlanView<'a> { + Htj2k(&'a J2kReferencedHtj2kPlan), + Classic(&'a J2kReferencedClassicPlan), +} + +impl<'a> ReferencedPlanView<'a> { + pub(super) const fn component_count(self) -> usize { + match self { + Self::Htj2k(J2kReferencedHtj2kPlan::Grayscale { .. }) + | Self::Classic(J2kReferencedClassicPlan::Grayscale { .. }) => 1, + Self::Htj2k(J2kReferencedHtj2kPlan::Color { .. }) + | Self::Classic(J2kReferencedClassicPlan::Color { .. }) => 3, + Self::Htj2k(J2kReferencedHtj2kPlan::Rgba { .. }) + | Self::Classic(J2kReferencedClassicPlan::Rgba { .. }) => 4, + } + } + + pub(super) fn tile_count(self) -> usize { + match self { + Self::Htj2k(plan) => plan.tiles().len(), + Self::Classic(plan) => plan.tiles().len(), + } + } + + pub(super) fn output_dimensions(self) -> (u32, u32) { + let rect = match self { + Self::Htj2k(plan) => plan.output_rect(), + Self::Classic(plan) => plan.output_rect(), + }; + (rect.width(), rect.height()) + } + + pub(super) fn component_plan( + self, + tile_index: usize, + component_index: usize, + ) -> Result<&'a J2kDirectGrayscalePlan> { + match self { + Self::Htj2k(plan) => match plan { + J2kReferencedHtj2kPlan::Grayscale { .. } if component_index == 0 => plan + .tiles() + .get(tile_index) + .and_then(crate::J2kReferencedTilePlan::grayscale_geometry), + J2kReferencedHtj2kPlan::Color { .. } => plan + .tiles() + .get(tile_index) + .and_then(crate::J2kReferencedTilePlan::color_geometry) + .and_then(|geometry| geometry.component_plans.get(component_index)), + J2kReferencedHtj2kPlan::Rgba { .. } => plan + .tiles() + .get(tile_index) + .and_then(crate::J2kReferencedTilePlan::rgba_geometry) + .and_then(|geometry| geometry.component_plans.get(component_index)), + J2kReferencedHtj2kPlan::Grayscale { .. } => None, + }, + Self::Classic(plan) => match plan { + J2kReferencedClassicPlan::Grayscale { .. } if component_index == 0 => plan + .tiles() + .get(tile_index) + .and_then(crate::J2kReferencedTilePlan::grayscale_geometry), + J2kReferencedClassicPlan::Color { .. } => plan + .tiles() + .get(tile_index) + .and_then(crate::J2kReferencedTilePlan::color_geometry) + .and_then(|geometry| geometry.component_plans.get(component_index)), + J2kReferencedClassicPlan::Rgba { .. } => plan + .tiles() + .get(tile_index) + .and_then(crate::J2kReferencedTilePlan::rgba_geometry) + .and_then(|geometry| geometry.component_plans.get(component_index)), + J2kReferencedClassicPlan::Grayscale { .. } => None, + }, + } + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) + } +} + +pub(super) fn validate_referenced_shape(plan: ReferencedPlanView<'_>) -> Result<()> { + if plan.tile_count() == 0 || plan.output_dimensions().0 == 0 || plan.output_dimensions().1 == 0 + { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + for tile_index in 0..plan.tile_count() { + for component_index in 0..plan.component_count() { + let component = plan.component_plan(tile_index, component_index)?; + if component.dimensions != plan.output_dimensions() { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + } + } + Ok(()) +} + +pub(super) fn referenced_component_band_count( + plan: ReferencedPlanView<'_>, + component_index: usize, +) -> Result { + let mut maximum = 0usize; + for tile_index in 0..plan.tile_count() { + maximum = maximum.max(component_band_count( + plan.component_plan(tile_index, component_index)?, + )?); + } + Ok(maximum) +} + +pub(super) fn referenced_band_target( + plan: ReferencedPlanView<'_>, + component_index: usize, + band_index: usize, +) -> Result { + let mut maximum = None; + for tile_index in 0..plan.tile_count() { + let component = plan.component_plan(tile_index, component_index)?; + for_each_staged_band_target(component, |current_index, target_len| { + if current_index == band_index { + maximum = + Some(maximum.map_or(target_len, |current: usize| current.max(target_len))); + } + Ok(()) + })?; + } + maximum.ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +pub(super) fn referenced_component_plane_len( + plan: ReferencedPlanView<'_>, + component_index: usize, +) -> Result { + let expected = checked_area(plan.output_dimensions().0, plan.output_dimensions().1)?; + for tile_index in 0..plan.tile_count() { + let current = component_plane_len(plan.component_plan(tile_index, component_index)?)?; + if current != expected { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + } + Ok(expected) +} + +fn for_each_staged_band_target( + plan: &J2kDirectGrayscalePlan, + mut visit: impl FnMut(usize, usize) -> Result<()>, +) -> Result<()> { + let mut band_index = 0usize; + for entropy_phase in [true, false] { + for step in &plan.steps { + let target_len = match (entropy_phase, step) { + (true, J2kDirectGrayscaleStep::ClassicSubBand(sub_band)) => { + Some(checked_area(sub_band.width, sub_band.height)?) + } + (true, J2kDirectGrayscaleStep::HtSubBand(sub_band)) => { + Some(checked_area(sub_band.width, sub_band.height)?) + } + (false, J2kDirectGrayscaleStep::Idwt(step)) => { + Some(checked_area(step.rect.width(), step.rect.height())?) + } + _ => None, + }; + if let Some(target_len) = target_len { + visit(band_index, target_len)?; + band_index = band_index + .checked_add(1) + .ok_or(ValidationError::ImageTooLarge)?; + } + } + } + Ok(()) +} + +pub(super) fn referenced_temporary_workspace_bytes(plan: ReferencedPlanView<'_>) -> Result { + let classic = max_referenced_dimensions(plan, false).map_or(Ok(0), |(width, height)| { + bitplane::classic_decode_workspace_bytes(width, height) + })?; + let ht = max_referenced_dimensions(plan, true).map_or(Ok(0), |(width, height)| { + ht_block_decode::ht_decode_workspace_bytes(width, height) + })?; + Ok(classic.max(ht)) +} + +pub(super) fn max_referenced_payload_bytes(plan: &J2kReferencedHtj2kPlan) -> Result { + plan.payloads().iter().try_fold(0usize, |maximum, payload| { + payload + .cleanup + .length + .checked_add(payload.refinement.map_or(0, |range| range.length)) + .map(|length| maximum.max(length)) + .ok_or(ValidationError::ImageTooLarge.into()) + }) +} + +pub(super) fn max_referenced_classic_payload_bytes(plan: &J2kReferencedClassicPlan) -> usize { + plan.payloads() + .iter() + .map(|payload| payload.combined_length) + .max() + .unwrap_or(0) +} + +pub(in super::super::super) fn max_referenced_classic_dimensions( + plan: &J2kReferencedClassicPlan, +) -> Option<(u32, u32)> { + max_referenced_dimensions(ReferencedPlanView::Classic(plan), false) +} + +pub(in super::super::super) fn max_referenced_ht_dimensions( + plan: &J2kReferencedHtj2kPlan, +) -> Option<(u32, u32)> { + max_referenced_dimensions(ReferencedPlanView::Htj2k(plan), true) +} + +fn max_referenced_dimensions(plan: ReferencedPlanView<'_>, ht: bool) -> Option<(u32, u32)> { + let mut dimensions = None; + for tile_index in 0..plan.tile_count() { + for component_index in 0..plan.component_count() { + let component = plan.component_plan(tile_index, component_index).ok()?; + for step in &component.steps { + match (ht, step) { + (true, J2kDirectGrayscaleStep::HtSubBand(sub_band)) => { + for job in &sub_band.jobs { + observe_max_dimensions(&mut dimensions, job.width, job.height); + } + } + (false, J2kDirectGrayscaleStep::ClassicSubBand(sub_band)) => { + for job in &sub_band.jobs { + observe_max_dimensions(&mut dimensions, job.width, job.height); + } + } + _ => {} + } + } + } + } + dimensions +} diff --git a/crates/j2k-native/src/direct_cpu/color.rs b/crates/j2k-native/src/direct_cpu/color.rs new file mode 100644 index 00000000..074f9594 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/color.rs @@ -0,0 +1,303 @@ +use super::{ + bail, execute_component_plan, floor_f32, prepare_direct_scratch, round_f32, DecodingError, + DirectComponentPlane, J2kDirectColorPlan, J2kDirectCpuScratch, J2kRect, J2kWaveletTransform, + Result, +}; + +/// Execute a adapter direct RGB plan on the CPU and write an RGB8 output region. +/// +/// # Errors +/// +/// Returns an error for invalid plan geometry, output bounds, or decode-stage failure. +pub fn execute_direct_color_plan_rgb8_into( + plan: &J2kDirectColorPlan, + output_region: J2kRect, + scratch: &mut J2kDirectCpuScratch, + out: &mut [u8], + stride: usize, +) -> Result<()> { + execute_direct_color_plan_u8_into( + plan, + output_region, + scratch, + out, + stride, + DirectColorU8Output::Rgb8, + ) +} + +/// Execute a adapter direct RGB plan on the CPU and write an RGBA8 output region. +/// +/// # Errors +/// +/// Returns an error for invalid plan geometry, output bounds, or decode-stage failure. +pub fn execute_direct_color_plan_rgba8_into( + plan: &J2kDirectColorPlan, + output_region: J2kRect, + scratch: &mut J2kDirectCpuScratch, + out: &mut [u8], + stride: usize, +) -> Result<()> { + execute_direct_color_plan_u8_into( + plan, + output_region, + scratch, + out, + stride, + DirectColorU8Output::Rgba8, + ) +} + +#[derive(Clone, Copy)] +enum DirectColorU8Output { + Rgb8, + Rgba8, +} + +impl DirectColorU8Output { + const fn bytes_per_pixel(self) -> usize { + match self { + Self::Rgb8 => 3, + Self::Rgba8 => 4, + } + } +} + +fn execute_direct_color_plan_u8_into( + plan: &J2kDirectColorPlan, + output_region: J2kRect, + scratch: &mut J2kDirectCpuScratch, + out: &mut [u8], + stride: usize, + output: DirectColorU8Output, +) -> Result<()> { + if plan.component_plans.len() != 3 { + bail!(DecodingError::UnsupportedFeature( + "direct CPU color plan requires three components" + )); + } + validate_output_region(plan, output_region, out.len(), stride, output)?; + + let workspace_budget = prepare_direct_scratch(plan, scratch)?; + for (component_index, component_plan) in plan.component_plans.iter().enumerate() { + let band_scratch = &mut scratch.component_band_sets[component_index]; + let plane = &mut scratch.component_planes[component_index]; + execute_component_plan(component_plan, band_scratch, plane, workspace_budget)?; + } + + let [plane0, plane1, plane2, ..] = scratch.component_planes.as_mut_slice() else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + if plan.mct { + apply_inverse_mct( + plan.transform, + plan.bit_depths, + false, + plane0, + plane1, + plane2, + )?; + } + write_rgb8_region( + [plane0, plane1, plane2], + plan.bit_depths, + output_region, + out, + stride, + output, + ) +} + +fn apply_inverse_mct( + transform: J2kWaveletTransform, + bit_depths: [u8; 3], + signed: bool, + plane0: &mut DirectComponentPlane, + plane1: &mut DirectComponentPlane, + plane2: &mut DirectComponentPlane, +) -> Result<()> { + let region = J2kRect { + x0: 0, + y0: 0, + x1: plane0.width, + y1: plane0.height, + }; + apply_inverse_mct_region( + transform, bit_depths, signed, region, plane0, plane1, plane2, + ) +} + +pub(super) fn apply_inverse_mct_region( + transform: J2kWaveletTransform, + bit_depths: [u8; 3], + signed: bool, + region: J2kRect, + plane0: &mut DirectComponentPlane, + plane1: &mut DirectComponentPlane, + plane2: &mut DirectComponentPlane, +) -> Result<()> { + if plane0.width != plane1.width + || plane1.width != plane2.width + || plane0.height != plane1.height + || plane1.height != plane2.height + || plane0.samples.len() != plane1.samples.len() + || plane1.samples.len() != plane2.samples.len() + || region.x0 > region.x1 + || region.y0 > region.y1 + || region.x1 > plane0.width + || region.y1 > plane0.height + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + let addend0 = if signed { + 0.0 + } else { + sign_addend(bit_depths[0]) + }; + let addend1 = if signed { + 0.0 + } else { + sign_addend(bit_depths[1]) + }; + let addend2 = if signed { + 0.0 + } else { + sign_addend(bit_depths[2]) + }; + let plane_width = plane0.width as usize; + for y in region.y0 as usize..region.y1 as usize { + let row_start = y + .checked_mul(plane_width) + .and_then(|start| start.checked_add(region.x0 as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let row_end = row_start + .checked_add(region.width() as usize) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for index in row_start..row_end { + let src0 = plane0.samples[index]; + let src1 = plane1.samples[index]; + let src2 = plane2.samples[index]; + let (out0, out1, out2) = match transform { + J2kWaveletTransform::Irreversible97 => ( + src0 + 1.402 * src2, + src0 - 0.34413 * src1 - 0.71414 * src2, + src0 + 1.772 * src1, + ), + J2kWaveletTransform::Reversible53 => { + let i1 = src0 - floor_f32((src2 + src1) * 0.25); + (src2 + i1, i1, src1 + i1) + } + }; + plane0.samples[index] = out0 + addend0; + plane1.samples[index] = out1 + addend1; + plane2.samples[index] = out2 + addend2; + } + } + Ok(()) +} + +fn write_rgb8_region( + planes: [&DirectComponentPlane; 3], + bit_depths: [u8; 3], + output_region: J2kRect, + out: &mut [u8], + stride: usize, + output: DirectColorU8Output, +) -> Result<()> { + let width = output_region.width() as usize; + let height = output_region.height() as usize; + let bytes_per_pixel = output.bytes_per_pixel(); + let row_bytes = width + .checked_mul(bytes_per_pixel) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for plane in planes { + if output_region.x1 > plane.width || output_region.y1 > plane.height { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + + for y in 0..height { + let src_y = output_region.y0 as usize + y; + let dst = &mut out[y * stride..y * stride + row_bytes]; + for x in 0..width { + let src_x = output_region.x0 as usize + x; + let dst = &mut dst[x * bytes_per_pixel..x * bytes_per_pixel + bytes_per_pixel]; + for channel in 0..3 { + let plane = planes[channel]; + let sample = plane.samples[src_y * plane.width as usize + src_x]; + dst[channel] = sample_as_u8(sample, bit_depths[channel]); + } + if matches!(output, DirectColorU8Output::Rgba8) { + dst[3] = u8::MAX; + } + } + } + Ok(()) +} + +fn validate_output_region( + plan: &J2kDirectColorPlan, + output_region: J2kRect, + out_len: usize, + stride: usize, + output: DirectColorU8Output, +) -> Result<()> { + if output_region.x1 > plan.dimensions.0 + || output_region.y1 > plan.dimensions.1 + || output_region.x0 > output_region.x1 + || output_region.y0 > output_region.y1 + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let bytes_per_pixel = u32::try_from(output.bytes_per_pixel()) + .map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let row_bytes = output_region + .width() + .checked_mul(bytes_per_pixel) + .and_then(|len| usize::try_from(len).ok()) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if stride < row_bytes { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let height = usize::try_from(output_region.height()) + .map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let required = if height == 0 { + 0 + } else { + stride + .checked_mul(height - 1) + .and_then(|prefix| prefix.checked_add(row_bytes)) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + }; + if out_len < required { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(()) +} + +#[expect( + clippy::cast_precision_loss, + reason = "direct CPU color math intentionally uses the decoder's f32 sample representation" +)] +fn sign_addend(bit_depth: u8) -> f32 { + (1_u32 << (bit_depth - 1)) as f32 +} + +#[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "samples are rounded and clamped before stable 8-bit output quantization" +)] +fn sample_as_u8(sample: f32, bit_depth: u8) -> u8 { + let rounded = round_f32(sample); + if bit_depth == 8 { + return rounded.clamp(0.0, f32::from(u8::MAX)) as u8; + } + let max_value = if bit_depth >= 16 { + f32::from(u16::MAX) + } else { + f32::from(((1_u16 << bit_depth) - 1).max(1)) + }; + round_f32((rounded.clamp(0.0, max_value) / max_value) * f32::from(u8::MAX)) as u8 +} diff --git a/crates/j2k-native/src/direct_cpu/component.rs b/crates/j2k-native/src/direct_cpu/component.rs new file mode 100644 index 00000000..3aaeb6ef --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/component.rs @@ -0,0 +1,352 @@ +use super::{ + bail, decode_ht_code_block_scalar_with_workspace, decode_j2k_code_block_scalar_with_workspace, + idwt, try_resize_decode_elements, DecodingError, DirectComponentBandScratch, + DirectComponentPlane, DirectCpuBand, DirectWorkspaceBudget, HtCodeBlockDecodeJob, + HtCodeBlockDecodeWorkspace, HtOwnedSubBandPlan, J2kCodeBlockDecodeJob, + J2kCodeBlockDecodeWorkspace, J2kDirectBandId, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, + J2kDirectIdwtStep, J2kDirectStoreStep, J2kIdwtBand, J2kOwnedSubBandPlan, J2kRect, + J2kSingleDecompositionIdwtJob, Range, Result, Vec, +}; + +pub(super) fn execute_component_plan( + plan: &J2kDirectGrayscalePlan, + bands: &mut DirectComponentBandScratch, + output: &mut DirectComponentPlane, + workspace_budget: DirectWorkspaceBudget, +) -> Result<()> { + bands.reset(); + let mut output_written = false; + + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { + execute_classic_sub_band(sub_band, bands, workspace_budget)?; + } + J2kDirectGrayscaleStep::HtSubBand(sub_band) => { + execute_ht_sub_band(sub_band, bands, workspace_budget)?; + } + J2kDirectGrayscaleStep::Idwt(step) => execute_idwt_step(step, bands)?, + J2kDirectGrayscaleStep::Store(store) => { + store_component(store, bands.active(), output, &mut output_written)?; + } + } + } + + if output_written { + Ok(()) + } else { + Err(DecodingError::CodeBlockDecodeFailure.into()) + } +} + +fn execute_classic_sub_band( + plan: &J2kOwnedSubBandPlan, + bands: &mut DirectComponentBandScratch, + workspace_budget: DirectWorkspaceBudget, +) -> Result<()> { + let (output, sub_band_width) = + prepare_sub_band_output(bands, plan.band_id, plan.rect, plan.width, plan.height)?; + let mut workspace = J2kCodeBlockDecodeWorkspace::default(); + if let Some((width, height)) = max_classic_job_dimensions(plan) { + workspace.prepare(width, height)?; + workspace_budget.validate_workspace(workspace.allocated_bytes()?)?; + } + + for job in &plan.jobs { + let output_range = checked_sub_band_job_output_range(&SubBandJobOutputRange { + output_x: job.output_x, + output_y: job.output_y, + output_stride: job.output_stride, + width: job.width, + height: job.height, + sub_band_width, + plan_width: plan.width, + plan_height: plan.height, + output_len: output.len(), + })?; + + let code_block = J2kCodeBlockDecodeJob { + data: &job.data, + segments: &job.segments, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + total_bitplanes: job.total_bitplanes, + roi_shift: job.roi_shift, + sub_band_type: job.sub_band_type, + style: job.style, + strict: job.strict, + dequantization_step: job.dequantization_step, + }; + decode_j2k_code_block_scalar_with_workspace( + code_block, + &mut output[output_range], + &mut workspace, + )?; + } + Ok(()) +} + +fn execute_ht_sub_band( + plan: &HtOwnedSubBandPlan, + bands: &mut DirectComponentBandScratch, + workspace_budget: DirectWorkspaceBudget, +) -> Result<()> { + let (output, sub_band_width) = + prepare_sub_band_output(bands, plan.band_id, plan.rect, plan.width, plan.height)?; + let mut workspace = HtCodeBlockDecodeWorkspace::default(); + if let Some((width, height)) = max_ht_job_dimensions(plan) { + workspace.prepare(width, height)?; + workspace_budget.validate_workspace(workspace.allocated_bytes()?)?; + } + + for job in &plan.jobs { + let output_range = checked_sub_band_job_output_range(&SubBandJobOutputRange { + output_x: job.output_x, + output_y: job.output_y, + output_stride: job.output_stride, + width: job.width, + height: job.height, + sub_band_width, + plan_width: plan.width, + plan_height: plan.height, + output_len: output.len(), + })?; + + let code_block = HtCodeBlockDecodeJob { + data: &job.data, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + roi_shift: job.roi_shift, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + }; + decode_ht_code_block_scalar_with_workspace( + code_block, + &mut output[output_range], + &mut workspace, + )?; + } + Ok(()) +} + +fn max_classic_job_dimensions(plan: &J2kOwnedSubBandPlan) -> Option<(u32, u32)> { + plan.jobs.iter().fold(None, |dimensions, job| { + Some( + dimensions.map_or((job.width, job.height), |(width, height)| { + (width.max(job.width), height.max(job.height)) + }), + ) + }) +} + +fn max_ht_job_dimensions(plan: &HtOwnedSubBandPlan) -> Option<(u32, u32)> { + plan.jobs.iter().fold(None, |dimensions, job| { + Some( + dimensions.map_or((job.width, job.height), |(width, height)| { + (width.max(job.width), height.max(job.height)) + }), + ) + }) +} + +pub(super) fn prepare_sub_band_output( + bands: &mut DirectComponentBandScratch, + band_id: J2kDirectBandId, + rect: J2kRect, + width: u32, + height: u32, +) -> Result<(&mut [f32], usize)> { + let required_len = checked_area(width, height)?; + let band_index = bands.prepare_band(band_id, rect, required_len)?; + let output = bands.bands[band_index].coefficients.as_mut_slice(); + let sub_band_width = + usize::try_from(width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + Ok((output, sub_band_width)) +} + +pub(super) fn execute_idwt_step( + step: &J2kDirectIdwtStep, + bands: &mut DirectComponentBandScratch, +) -> Result<()> { + let output_index = bands.prepare_band(step.output_band_id, step.rect, 0)?; + let (input_bands, output_bands) = bands.bands.split_at_mut(output_index); + let output = &mut output_bands[0].coefficients; + let ll = find_idwt_band(input_bands, step.ll_band_id)?; + let hl = find_idwt_band(input_bands, step.hl_band_id)?; + let lh = find_idwt_band(input_bands, step.lh_band_id)?; + let hh = find_idwt_band(input_bands, step.hh_band_id)?; + let job = J2kSingleDecompositionIdwtJob { + rect: step.rect, + transform: step.transform, + ll, + hl, + lh, + hh, + }; + idwt::apply_single_decomposition_idwt_job(job, output) +} + +fn find_idwt_band(bands: &[DirectCpuBand], band_id: J2kDirectBandId) -> Result> { + let band = find_band(bands, band_id)?; + Ok(J2kIdwtBand { + rect: band.rect, + coefficients: &band.coefficients, + }) +} + +pub(super) fn store_component( + store: &J2kDirectStoreStep, + bands: &[DirectCpuBand], + plane: &mut DirectComponentPlane, + output_written: &mut bool, +) -> Result<()> { + let input = find_band(bands, store.input_band_id)?; + if !*output_written { + plane.width = store.output_width; + plane.height = store.output_height; + let required_len = checked_area(store.output_width, store.output_height)?; + resize_and_zero(&mut plane.samples, required_len)?; + *output_written = true; + } + if plane.width != store.output_width + || plane.height != store.output_height + || plane.samples.len() != checked_area(store.output_width, store.output_height)? + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + validate_store_bounds(store, input, plane)?; + let input_width = input.rect.width() as usize; + let output_width = plane.width as usize; + let copy_width = store.copy_width as usize; + for row in 0..store.copy_height as usize { + let src_start = (store.source_y as usize + row) + .checked_mul(input_width) + .and_then(|base| base.checked_add(store.source_x as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let dst_start = (store.output_y as usize + row) + .checked_mul(output_width) + .and_then(|base| base.checked_add(store.output_x as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let src = &input.coefficients[src_start..src_start + copy_width]; + let dst = &mut plane.samples[dst_start..dst_start + copy_width]; + for (src, dst) in src.iter().zip(dst.iter_mut()) { + *dst = *src + store.addend; + } + } + Ok(()) +} + +fn find_band(bands: &[DirectCpuBand], band_id: J2kDirectBandId) -> Result<&DirectCpuBand> { + bands + .iter() + .find(|band| band.band_id == band_id) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +fn validate_store_bounds( + store: &J2kDirectStoreStep, + input: &DirectCpuBand, + output: &DirectComponentPlane, +) -> Result<()> { + if store + .source_x + .checked_add(store.copy_width) + .is_none_or(|x| x > input.rect.width()) + || store + .source_y + .checked_add(store.copy_height) + .is_none_or(|y| y > input.rect.height()) + || store + .output_x + .checked_add(store.copy_width) + .is_none_or(|x| x > output.width) + || store + .output_y + .checked_add(store.copy_height) + .is_none_or(|y| y > output.height) + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(()) +} + +pub(super) fn checked_area(width: u32, height: u32) -> Result { + let height = usize::try_from(height).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + usize::try_from(width) + .ok() + .and_then(|width| width.checked_mul(height)) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +fn checked_block_base(output_x: u32, output_y: u32, stride: usize) -> Result { + let output_x = usize::try_from(output_x).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + usize::try_from(output_y) + .ok() + .and_then(|y| y.checked_mul(stride)) + .and_then(|base| base.checked_add(output_x)) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +pub(super) struct SubBandJobOutputRange { + pub(super) output_x: u32, + pub(super) output_y: u32, + pub(super) output_stride: usize, + pub(super) width: u32, + pub(super) height: u32, + pub(super) sub_band_width: usize, + pub(super) plan_width: u32, + pub(super) plan_height: u32, + pub(super) output_len: usize, +} + +pub(super) fn checked_sub_band_job_output_range( + bounds: &SubBandJobOutputRange, +) -> Result> { + let base_idx = checked_block_base(bounds.output_x, bounds.output_y, bounds.sub_band_width)?; + let block_len = checked_block_output_len(bounds.output_stride, bounds.width, bounds.height)?; + let end_idx = base_idx + .checked_add(block_len) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if end_idx > bounds.output_len + || bounds + .output_x + .checked_add(bounds.width) + .is_none_or(|x| x > bounds.plan_width) + || bounds + .output_y + .checked_add(bounds.height) + .is_none_or(|y| y > bounds.plan_height) + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(base_idx..end_idx) +} + +fn checked_block_output_len(stride: usize, width: u32, height: u32) -> Result { + if height == 0 { + return Ok(0); + } + let height = usize::try_from(height).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let width = usize::try_from(width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + stride + .checked_mul(height - 1) + .and_then(|prefix| prefix.checked_add(width)) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +pub(super) fn resize_and_zero(buffer: &mut Vec, len: usize) -> Result<()> { + try_resize_decode_elements(buffer, len, 0.0)?; + buffer.fill(0.0); + Ok(()) +} diff --git a/crates/j2k-native/src/direct_cpu/referenced.rs b/crates/j2k-native/src/direct_cpu/referenced.rs new file mode 100644 index 00000000..e6c30799 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Parse-free CPU execution for referenced HTJ2K plans. + +mod component; +mod output; +mod payload; + +pub(super) use output::{decoded_color_components, decoded_components, decoded_plane}; +pub use output::{J2kDirectDecodedComponents, J2kDirectDecodedPlane}; +pub(super) use payload::payload_slice; + +use crate::error::{bail, DecodingError, Result}; +use crate::{HtCodeBlockPayloadRanges, J2kReferencedHtj2kPlan}; + +use super::allocation::prepare_referenced_direct_scratch; +use super::J2kDirectCpuScratch; +use component::{execute_color_components_referenced, execute_component_plan_referenced}; +use payload::{validate_payload_ranges, ReferencedPayloadCursor}; + +/// Execute retained per-tile Gray/RGB/RGBA HTJ2K geometry without reparsing packets. +/// +/// Compressed cleanup/refinement ranges are validated against `encoded_input` +/// and combined in one retained scratch buffer. Reconstructed component owners +/// remain in `scratch` and are borrowed by the returned view. +#[doc(hidden)] +pub fn execute_referenced_htj2k_plan<'scratch>( + plan: &J2kReferencedHtj2kPlan, + encoded_input: &[u8], + signed: bool, + scratch: &'scratch mut J2kDirectCpuScratch, +) -> Result> { + execute_referenced_htj2k_plan_with_payloads( + plan, + encoded_input, + plan.payloads(), + signed, + scratch, + ) +} + +/// Execute retained HTJ2K geometry from caller-flattened payload ranges. +/// +/// `payloads` must remain in the plan's component/step/job traversal order, +/// but their ranges may point anywhere inside the shared `payload_arena`. +#[doc(hidden)] +pub fn execute_referenced_htj2k_plan_from_payloads<'scratch>( + plan: &J2kReferencedHtj2kPlan, + payload_arena: &[u8], + payloads: &[HtCodeBlockPayloadRanges], + signed: bool, + scratch: &'scratch mut J2kDirectCpuScratch, +) -> Result> { + execute_referenced_htj2k_plan_with_payloads(plan, payload_arena, payloads, signed, scratch) +} + +fn execute_referenced_htj2k_plan_with_payloads<'scratch>( + plan: &J2kReferencedHtj2kPlan, + encoded_input: &[u8], + payload_ranges: &[HtCodeBlockPayloadRanges], + signed: bool, + scratch: &'scratch mut J2kDirectCpuScratch, +) -> Result> { + if payload_ranges.len() != plan.payloads().len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + validate_payload_ranges(encoded_input, payload_ranges)?; + prepare_referenced_direct_scratch(plan, scratch)?; + { + let J2kDirectCpuScratch { + component_band_sets, + component_planes, + compressed_payload, + classic_workspace: _, + ht_workspace, + staged_state: _, + } = scratch; + let mut payloads = + ReferencedPayloadCursor::new(encoded_input, payload_ranges, compressed_payload); + let mut output_initialized = [false; 4]; + + for tile in plan.tiles() { + if let Some(geometry) = tile.grayscale_geometry() { + let bands = component_band_sets + .first_mut() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let output = component_planes + .first_mut() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + execute_component_plan_referenced( + geometry, + bands, + output, + &mut payloads, + ht_workspace, + &mut output_initialized[0], + )?; + } else if let Some(geometry) = tile.color_geometry() { + execute_color_components_referenced( + &geometry.component_plans, + 3, + geometry.bit_depths, + geometry.mct, + geometry.transform, + signed, + component_band_sets, + component_planes, + &mut payloads, + ht_workspace, + &mut output_initialized, + tile.destination_rect(), + )?; + } else if let Some(geometry) = tile.rgba_geometry() { + execute_color_components_referenced( + &geometry.component_plans, + 4, + [ + geometry.bit_depths[0], + geometry.bit_depths[1], + geometry.bit_depths[2], + ], + geometry.mct, + geometry.transform, + signed, + component_band_sets, + component_planes, + &mut payloads, + ht_workspace, + &mut output_initialized, + tile.destination_rect(), + )?; + } else { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + payloads.ensure_exhausted()?; + } + + decoded_components(plan, scratch) +} diff --git a/crates/j2k-native/src/direct_cpu/referenced/component.rs b/crates/j2k-native/src/direct_cpu/referenced/component.rs new file mode 100644 index 00000000..d50ea693 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced/component.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::error::{bail, DecodingError, Result}; +use crate::{ + decode_ht_code_block_scalar_with_workspace, HtCodeBlockDecodeJob, HtCodeBlockDecodeWorkspace, + HtOwnedSubBandPlan, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kWaveletTransform, +}; + +use super::super::{ + apply_inverse_mct_region, checked_sub_band_job_output_range, execute_idwt_step, + prepare_sub_band_output, store_component, DirectComponentBandScratch, DirectComponentPlane, + SubBandJobOutputRange, +}; +use super::payload::ReferencedPayloadCursor; + +#[expect( + clippy::too_many_arguments, + reason = "the parse-free RGB-family executor keeps geometry, transform metadata, retained owners, payload cursor, and scalar workspace explicit" +)] +pub(super) fn execute_color_components_referenced( + component_plans: &[J2kDirectGrayscalePlan], + expected_component_count: usize, + rgb_bit_depths: [u8; 3], + mct: bool, + transform: J2kWaveletTransform, + signed: bool, + component_band_sets: &mut [DirectComponentBandScratch], + reconstructed_planes: &mut [DirectComponentPlane], + payloads: &mut ReferencedPayloadCursor<'_, '_>, + ht_workspace: &mut HtCodeBlockDecodeWorkspace, + output_initialized: &mut [bool; 4], + destination: crate::J2kRect, +) -> Result<()> { + if component_plans.len() != expected_component_count + || !matches!(expected_component_count, 3 | 4) + || component_band_sets.len() < component_plans.len() + || reconstructed_planes.len() < component_plans.len() + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + for (component_index, component_plan) in component_plans.iter().enumerate() { + execute_component_plan_referenced( + component_plan, + &mut component_band_sets[component_index], + &mut reconstructed_planes[component_index], + payloads, + ht_workspace, + &mut output_initialized[component_index], + )?; + } + if mct { + let [plane0, plane1, plane2, ..] = reconstructed_planes else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + apply_inverse_mct_region( + transform, + rgb_bit_depths, + signed, + destination, + plane0, + plane1, + plane2, + )?; + } + Ok(()) +} + +pub(super) fn execute_component_plan_referenced( + plan: &J2kDirectGrayscalePlan, + bands: &mut DirectComponentBandScratch, + output: &mut DirectComponentPlane, + payloads: &mut ReferencedPayloadCursor<'_, '_>, + ht_workspace: &mut HtCodeBlockDecodeWorkspace, + output_initialized: &mut bool, +) -> Result<()> { + bands.reset(); + let mut stored = false; + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(_) => { + bail!(DecodingError::UnsupportedFeature( + "referenced HTJ2K CPU plan encountered classic code blocks" + )); + } + J2kDirectGrayscaleStep::HtSubBand(sub_band) => { + execute_ht_sub_band_referenced(sub_band, bands, payloads, ht_workspace)?; + } + J2kDirectGrayscaleStep::Idwt(step) => execute_idwt_step(step, bands)?, + J2kDirectGrayscaleStep::Store(store) => { + store_component(store, bands.active(), output, output_initialized)?; + stored = true; + } + } + } + if stored { + Ok(()) + } else { + Err(DecodingError::CodeBlockDecodeFailure.into()) + } +} + +fn execute_ht_sub_band_referenced( + plan: &HtOwnedSubBandPlan, + bands: &mut DirectComponentBandScratch, + payloads: &mut ReferencedPayloadCursor<'_, '_>, + workspace: &mut HtCodeBlockDecodeWorkspace, +) -> Result<()> { + let (output, sub_band_width) = + prepare_sub_band_output(bands, plan.band_id, plan.rect, plan.width, plan.height)?; + for job in &plan.jobs { + let output_range = checked_sub_band_job_output_range(&SubBandJobOutputRange { + output_x: job.output_x, + output_y: job.output_y, + output_stride: job.output_stride, + width: job.width, + height: job.height, + sub_band_width, + plan_width: plan.width, + plan_height: plan.height, + output_len: output.len(), + })?; + let data = payloads.next_data(job.cleanup_length, job.refinement_length)?; + let code_block = HtCodeBlockDecodeJob { + data, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + roi_shift: job.roi_shift, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + }; + decode_ht_code_block_scalar_with_workspace( + code_block, + &mut output[output_range], + workspace, + )?; + } + Ok(()) +} diff --git a/crates/j2k-native/src/direct_cpu/referenced/output.rs b/crates/j2k-native/src/direct_cpu/referenced/output.rs new file mode 100644 index 00000000..831ba837 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced/output.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::error::{bail, DecodingError, Result}; +use crate::J2kReferencedHtj2kPlan; + +use super::super::{DirectComponentPlane, J2kDirectCpuScratch}; + +/// One decoded component plane borrowed from retained direct CPU scratch. +#[doc(hidden)] +#[derive(Debug, Clone, Copy)] +pub struct J2kDirectDecodedPlane<'a> { + pub(in crate::direct_cpu) dimensions: (u32, u32), + pub(in crate::direct_cpu) bit_depth: u8, + pub(in crate::direct_cpu) samples: &'a [f32], +} + +impl<'a> J2kDirectDecodedPlane<'a> { + /// Full reduced-resolution plane dimensions. + #[must_use] + pub const fn dimensions(self) -> (u32, u32) { + self.dimensions + } + + /// Declared component precision. + #[must_use] + pub const fn bit_depth(self) -> u8 { + self.bit_depth + } + + /// Row-major reconstructed samples. + #[must_use] + pub const fn samples(self) -> &'a [f32] { + self.samples + } +} + +/// Gray, RGB, or RGBA decoded planes borrowed from retained direct CPU scratch. +#[doc(hidden)] +#[derive(Debug)] +pub struct J2kDirectDecodedComponents<'a> { + pub(in crate::direct_cpu) dimensions: (u32, u32), + pub(in crate::direct_cpu) planes: [Option>; 4], + pub(in crate::direct_cpu) component_count: usize, +} + +impl J2kDirectDecodedComponents<'_> { + /// Full reduced-resolution plane dimensions. + #[must_use] + pub const fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + /// Number of decoded planes: one for Gray, three for RGB, or four for RGBA. + #[must_use] + pub const fn component_count(&self) -> usize { + self.component_count + } + + /// Return one decoded component plane. + #[must_use] + pub fn plane(&self, index: usize) -> Option> { + self.planes.get(index).copied().flatten() + } +} + +pub(in crate::direct_cpu) fn decoded_components<'scratch>( + plan: &J2kReferencedHtj2kPlan, + scratch: &'scratch J2kDirectCpuScratch, +) -> Result> { + let dimensions = (plan.output_rect().width(), plan.output_rect().height()); + let first = plan + .tiles() + .first() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + match plan { + J2kReferencedHtj2kPlan::Grayscale { .. } => { + let geometry = first + .grayscale_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for tile in plan.tiles() { + let current = tile + .grayscale_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if current.dimensions != dimensions || current.bit_depth != geometry.bit_depth { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + let plane = decoded_plane( + scratch + .component_planes + .first() + .ok_or(DecodingError::CodeBlockDecodeFailure)?, + dimensions, + geometry.bit_depth, + )?; + Ok(J2kDirectDecodedComponents { + dimensions, + planes: [Some(plane), None, None, None], + component_count: 1, + }) + } + J2kReferencedHtj2kPlan::Color { .. } => { + let geometry = first + .color_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for tile in plan.tiles() { + let current = tile + .color_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if current.dimensions != dimensions + || current.bit_depths != geometry.bit_depths + || current.component_plans.len() != 3 + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + decoded_color_components(dimensions, &geometry.bit_depths, 3, scratch) + } + J2kReferencedHtj2kPlan::Rgba { .. } => { + let geometry = first + .rgba_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for tile in plan.tiles() { + let current = tile + .rgba_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if current.dimensions != dimensions + || current.bit_depths != geometry.bit_depths + || current.component_plans.len() != 4 + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + decoded_color_components(dimensions, &geometry.bit_depths, 4, scratch) + } + } +} + +pub(in crate::direct_cpu) fn decoded_color_components<'scratch>( + dimensions: (u32, u32), + bit_depths: &[u8], + component_count: usize, + scratch: &'scratch J2kDirectCpuScratch, +) -> Result> { + if !matches!(component_count, 3 | 4) + || bit_depths.len() != component_count + || scratch.component_planes.len() < component_count + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let mut planes = [None, None, None, None]; + for component in 0..component_count { + planes[component] = Some(decoded_plane( + &scratch.component_planes[component], + dimensions, + bit_depths[component], + )?); + } + Ok(J2kDirectDecodedComponents { + dimensions, + planes, + component_count, + }) +} + +pub(in crate::direct_cpu) fn decoded_plane( + plane: &DirectComponentPlane, + dimensions: (u32, u32), + bit_depth: u8, +) -> Result> { + if (plane.width, plane.height) != dimensions + || plane.samples.len() != super::super::checked_area(dimensions.0, dimensions.1)? + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(J2kDirectDecodedPlane { + dimensions, + bit_depth, + samples: &plane.samples, + }) +} diff --git a/crates/j2k-native/src/direct_cpu/referenced/payload.rs b/crates/j2k-native/src/direct_cpu/referenced/payload.rs new file mode 100644 index 00000000..5b1db864 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced/payload.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::error::{bail, DecodingError, Result}; +use crate::{try_reserve_decode_elements, HtCodeBlockPayloadRanges}; + +pub(super) fn validate_payload_ranges( + encoded_input: &[u8], + payloads: &[HtCodeBlockPayloadRanges], +) -> Result<()> { + for payload in payloads { + payload_slice(encoded_input, payload.cleanup)?; + if let Some(refinement) = payload.refinement { + payload_slice(encoded_input, refinement)?; + } + } + Ok(()) +} + +pub(super) struct ReferencedPayloadCursor<'input, 'scratch> { + encoded_input: &'input [u8], + payloads: &'input [HtCodeBlockPayloadRanges], + combined: &'scratch mut alloc::vec::Vec, + next: usize, +} + +impl<'input, 'scratch> ReferencedPayloadCursor<'input, 'scratch> { + pub(super) fn new( + encoded_input: &'input [u8], + payloads: &'input [HtCodeBlockPayloadRanges], + combined: &'scratch mut alloc::vec::Vec, + ) -> Self { + Self { + encoded_input, + payloads, + combined, + next: 0, + } + } + + pub(super) fn next_data( + &mut self, + cleanup_length: u32, + refinement_length: u32, + ) -> Result<&[u8]> { + let payload = self + .payloads + .get(self.next) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + self.next = self + .next + .checked_add(1) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if payload.cleanup.length != cleanup_length as usize + || payload.refinement.map_or(0, |range| range.length) != refinement_length as usize + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let cleanup = payload_slice(self.encoded_input, payload.cleanup)?; + let Some(refinement_range) = payload.refinement else { + return Ok(cleanup); + }; + let refinement = payload_slice(self.encoded_input, refinement_range)?; + let combined_len = cleanup + .len() + .checked_add(refinement.len()) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if payload.cleanup.end() == Some(refinement_range.offset) { + return payload_slice( + self.encoded_input, + crate::J2kCodestreamRange { + offset: payload.cleanup.offset, + length: combined_len, + }, + ); + } + self.combined.clear(); + try_reserve_decode_elements(self.combined, combined_len)?; + self.combined.extend_from_slice(cleanup); + self.combined.extend_from_slice(refinement); + Ok(self.combined) + } + + pub(super) fn ensure_exhausted(&self) -> Result<()> { + if self.next == self.payloads.len() { + Ok(()) + } else { + Err(DecodingError::CodeBlockDecodeFailure.into()) + } + } +} + +pub(in crate::direct_cpu) fn payload_slice( + input: &[u8], + range: crate::J2kCodestreamRange, +) -> Result<&[u8]> { + let end = range + .offset + .checked_add(range.length) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + input + .get(range.offset..end) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} diff --git a/crates/j2k-native/src/direct_cpu/referenced_classic.rs b/crates/j2k-native/src/direct_cpu/referenced_classic.rs new file mode 100644 index 00000000..a29fb4c9 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced_classic.rs @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Parse-free CPU execution for referenced classic JPEG 2000 plans. + +use crate::error::{bail, DecodingError, Result}; +use crate::{ + decode_j2k_code_block_scalar_with_workspace, try_reserve_decode_elements, + J2kCodeBlockDecodeJob, J2kCodeBlockDecodeWorkspace, J2kDirectGrayscalePlan, + J2kDirectGrayscaleStep, J2kOwnedSubBandPlan, J2kReferencedClassicPlan, J2kWaveletTransform, +}; + +use super::allocation::prepare_referenced_classic_scratch; +use super::referenced::{decoded_color_components, decoded_plane, payload_slice}; +use super::{ + apply_inverse_mct_region, checked_sub_band_job_output_range, execute_idwt_step, + prepare_sub_band_output, store_component, DirectComponentBandScratch, DirectComponentPlane, + J2kDirectCpuScratch, J2kDirectDecodedComponents, SubBandJobOutputRange, +}; + +/// Execute retained per-tile Gray/RGB/RGBA classic JPEG 2000 geometry +/// without reparsing packet headers. +/// +/// Ordered compressed fragments are validated against `encoded_input` and +/// concatenated per code block in one retained scratch buffer. Reconstructed +/// component owners remain in `scratch` and are borrowed by the returned view. +#[doc(hidden)] +pub fn execute_referenced_classic_plan<'scratch>( + plan: &J2kReferencedClassicPlan, + encoded_input: &[u8], + signed: bool, + scratch: &'scratch mut J2kDirectCpuScratch, +) -> Result> { + execute_referenced_classic_plan_with_payloads( + plan, + encoded_input, + plan.payloads(), + plan.ranges(), + signed, + scratch, + ) +} + +/// Execute retained classic geometry from caller-flattened payload ranges. +/// +/// Payload descriptors remain in geometry traversal order. Their fragment +/// ranges may point anywhere inside the shared `payload_arena`. +#[doc(hidden)] +pub fn execute_referenced_classic_plan_from_payloads<'scratch>( + plan: &J2kReferencedClassicPlan, + payload_arena: &[u8], + payloads: &[crate::J2kClassicCodeBlockPayload], + ranges: &[crate::J2kCodestreamRange], + signed: bool, + scratch: &'scratch mut J2kDirectCpuScratch, +) -> Result> { + execute_referenced_classic_plan_with_payloads( + plan, + payload_arena, + payloads, + ranges, + signed, + scratch, + ) +} + +fn execute_referenced_classic_plan_with_payloads<'scratch>( + plan: &J2kReferencedClassicPlan, + encoded_input: &[u8], + payload_descriptors: &[crate::J2kClassicCodeBlockPayload], + payload_ranges: &[crate::J2kCodestreamRange], + signed: bool, + scratch: &'scratch mut J2kDirectCpuScratch, +) -> Result> { + if payload_descriptors.len() != plan.payloads().len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + validate_payload_ranges(encoded_input, payload_descriptors, payload_ranges)?; + prepare_referenced_classic_scratch(plan, scratch)?; + { + let J2kDirectCpuScratch { + component_band_sets, + component_planes, + compressed_payload, + classic_workspace, + ht_workspace: _, + staged_state: _, + } = scratch; + let mut payloads = ClassicPayloadCursor::new( + payload_descriptors, + payload_ranges, + encoded_input, + compressed_payload, + ); + let mut output_initialized = [false; 4]; + + for tile in plan.tiles() { + if let Some(geometry) = tile.grayscale_geometry() { + let bands = component_band_sets + .first_mut() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let output = component_planes + .first_mut() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + execute_component_plan_referenced( + geometry, + bands, + output, + &mut payloads, + classic_workspace, + &mut output_initialized[0], + )?; + } else if let Some(geometry) = tile.color_geometry() { + execute_color_components_referenced( + &geometry.component_plans, + 3, + geometry.bit_depths, + geometry.mct, + geometry.transform, + signed, + component_band_sets, + component_planes, + &mut payloads, + classic_workspace, + &mut output_initialized, + tile.destination_rect(), + )?; + } else if let Some(geometry) = tile.rgba_geometry() { + execute_color_components_referenced( + &geometry.component_plans, + 4, + [ + geometry.bit_depths[0], + geometry.bit_depths[1], + geometry.bit_depths[2], + ], + geometry.mct, + geometry.transform, + signed, + component_band_sets, + component_planes, + &mut payloads, + classic_workspace, + &mut output_initialized, + tile.destination_rect(), + )?; + } else { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + payloads.ensure_exhausted()?; + } + + decoded_components(plan, scratch) +} + +#[expect( + clippy::too_many_arguments, + reason = "the parse-free RGB-family executor keeps geometry, transform metadata, retained owners, payload cursor, and scalar workspace explicit" +)] +fn execute_color_components_referenced( + component_plans: &[J2kDirectGrayscalePlan], + expected_component_count: usize, + rgb_bit_depths: [u8; 3], + mct: bool, + transform: J2kWaveletTransform, + signed: bool, + component_band_sets: &mut [DirectComponentBandScratch], + reconstructed_planes: &mut [DirectComponentPlane], + payloads: &mut ClassicPayloadCursor<'_, '_>, + classic_workspace: &mut J2kCodeBlockDecodeWorkspace, + output_initialized: &mut [bool; 4], + destination: crate::J2kRect, +) -> Result<()> { + if component_plans.len() != expected_component_count + || !matches!(expected_component_count, 3 | 4) + || component_band_sets.len() < component_plans.len() + || reconstructed_planes.len() < component_plans.len() + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + for (component_index, component_plan) in component_plans.iter().enumerate() { + execute_component_plan_referenced( + component_plan, + &mut component_band_sets[component_index], + &mut reconstructed_planes[component_index], + payloads, + classic_workspace, + &mut output_initialized[component_index], + )?; + } + if mct { + let [plane0, plane1, plane2, ..] = reconstructed_planes else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + apply_inverse_mct_region( + transform, + rgb_bit_depths, + signed, + destination, + plane0, + plane1, + plane2, + )?; + } + Ok(()) +} + +fn execute_component_plan_referenced( + plan: &J2kDirectGrayscalePlan, + bands: &mut DirectComponentBandScratch, + output: &mut DirectComponentPlane, + payloads: &mut ClassicPayloadCursor<'_, '_>, + classic_workspace: &mut J2kCodeBlockDecodeWorkspace, + output_initialized: &mut bool, +) -> Result<()> { + bands.reset(); + let mut stored = false; + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { + execute_classic_sub_band_referenced(sub_band, bands, payloads, classic_workspace)?; + } + J2kDirectGrayscaleStep::HtSubBand(_) => { + bail!(DecodingError::UnsupportedFeature( + "referenced classic CPU plan encountered HT code blocks" + )); + } + J2kDirectGrayscaleStep::Idwt(step) => execute_idwt_step(step, bands)?, + J2kDirectGrayscaleStep::Store(store) => { + store_component(store, bands.active(), output, output_initialized)?; + stored = true; + } + } + } + if stored { + Ok(()) + } else { + Err(DecodingError::CodeBlockDecodeFailure.into()) + } +} + +fn execute_classic_sub_band_referenced( + plan: &J2kOwnedSubBandPlan, + bands: &mut DirectComponentBandScratch, + payloads: &mut ClassicPayloadCursor<'_, '_>, + workspace: &mut J2kCodeBlockDecodeWorkspace, +) -> Result<()> { + let (output, sub_band_width) = + prepare_sub_band_output(bands, plan.band_id, plan.rect, plan.width, plan.height)?; + for job in &plan.jobs { + if !job.data.is_empty() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let output_range = checked_sub_band_job_output_range(&SubBandJobOutputRange { + output_x: job.output_x, + output_y: job.output_y, + output_stride: job.output_stride, + width: job.width, + height: job.height, + sub_band_width, + plan_width: plan.width, + plan_height: plan.height, + output_len: output.len(), + })?; + let data = payloads.next_data()?; + let code_block = J2kCodeBlockDecodeJob { + data, + segments: &job.segments, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + total_bitplanes: job.total_bitplanes, + roi_shift: job.roi_shift, + sub_band_type: job.sub_band_type, + style: job.style, + strict: job.strict, + dequantization_step: job.dequantization_step, + }; + decode_j2k_code_block_scalar_with_workspace( + code_block, + &mut output[output_range], + workspace, + )?; + } + Ok(()) +} + +fn validate_payload_ranges( + encoded_input: &[u8], + payloads: &[crate::J2kClassicCodeBlockPayload], + ranges: &[crate::J2kCodestreamRange], +) -> Result<()> { + let mut next_range = 0usize; + for payload in payloads { + if payload.first_range != next_range { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let end_range = payload + .end_range() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let fragments = ranges + .get(payload.first_range..end_range) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let mut combined_length = 0usize; + for range in fragments { + payload_slice(encoded_input, *range)?; + combined_length = combined_length + .checked_add(range.length) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + } + if combined_length != payload.combined_length { + bail!(DecodingError::CodeBlockDecodeFailure); + } + next_range = end_range; + } + if next_range != ranges.len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(()) +} + +struct ClassicPayloadCursor<'plan, 'scratch> { + encoded_input: &'plan [u8], + payloads: &'plan [crate::J2kClassicCodeBlockPayload], + ranges: &'plan [crate::J2kCodestreamRange], + combined: &'scratch mut alloc::vec::Vec, + next: usize, +} + +impl<'plan, 'scratch> ClassicPayloadCursor<'plan, 'scratch> { + fn new( + payloads: &'plan [crate::J2kClassicCodeBlockPayload], + ranges: &'plan [crate::J2kCodestreamRange], + encoded_input: &'plan [u8], + combined: &'scratch mut alloc::vec::Vec, + ) -> Self { + Self { + encoded_input, + payloads, + ranges, + combined, + next: 0, + } + } + + fn next_data(&mut self) -> Result<&[u8]> { + let payload = self + .payloads + .get(self.next) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + self.next = self + .next + .checked_add(1) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let end_range = payload + .end_range() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let fragments = self + .ranges + .get(payload.first_range..end_range) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + + if let [range] = fragments { + if range.length != payload.combined_length { + bail!(DecodingError::CodeBlockDecodeFailure); + } + return payload_slice(self.encoded_input, *range); + } + + self.combined.clear(); + try_reserve_decode_elements(self.combined, payload.combined_length)?; + for range in fragments { + self.combined + .extend_from_slice(payload_slice(self.encoded_input, *range)?); + } + if self.combined.len() != payload.combined_length { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(self.combined) + } + + fn ensure_exhausted(&self) -> Result<()> { + if self.next == self.payloads.len() { + Ok(()) + } else { + Err(DecodingError::CodeBlockDecodeFailure.into()) + } + } +} + +pub(super) fn decoded_components<'scratch>( + plan: &J2kReferencedClassicPlan, + scratch: &'scratch J2kDirectCpuScratch, +) -> Result> { + let dimensions = (plan.output_rect().width(), plan.output_rect().height()); + let first = plan + .tiles() + .first() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + match plan { + J2kReferencedClassicPlan::Grayscale { .. } => { + let geometry = first + .grayscale_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for tile in plan.tiles() { + let current = tile + .grayscale_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if current.dimensions != dimensions || current.bit_depth != geometry.bit_depth { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + let plane = decoded_plane( + scratch + .component_planes + .first() + .ok_or(DecodingError::CodeBlockDecodeFailure)?, + dimensions, + geometry.bit_depth, + )?; + Ok(J2kDirectDecodedComponents { + dimensions, + planes: [Some(plane), None, None, None], + component_count: 1, + }) + } + J2kReferencedClassicPlan::Color { .. } => { + let geometry = first + .color_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for tile in plan.tiles() { + let current = tile + .color_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if current.dimensions != dimensions + || current.bit_depths != geometry.bit_depths + || current.component_plans.len() != 3 + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + decoded_color_components(dimensions, &geometry.bit_depths, 3, scratch) + } + J2kReferencedClassicPlan::Rgba { .. } => { + let geometry = first + .rgba_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for tile in plan.tiles() { + let current = tile + .rgba_geometry() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if current.dimensions != dimensions + || current.bit_depths != geometry.bit_depths + || current.component_plans.len() != 4 + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + decoded_color_components(dimensions, &geometry.bit_depths, 4, scratch) + } + } +} diff --git a/crates/j2k-native/src/direct_cpu/referenced_staged.rs b/crates/j2k-native/src/direct_cpu/referenced_staged.rs new file mode 100644 index 00000000..430958d2 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced_staged.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Staged parse-free CPU execution for cross-image code-block scheduling. + +mod entropy; +mod finish; +mod plan_access; +mod prepare; +mod state; + +pub use entropy::{execute_referenced_classic_entropy_job, execute_referenced_htj2k_entropy_job}; +pub use finish::{ + finish_referenced_classic_staged, finish_referenced_classic_tile_staged, + finish_referenced_htj2k_staged, finish_referenced_htj2k_tile_staged, +}; +pub use prepare::{ + prepare_referenced_classic_entropy_workspace, prepare_referenced_classic_staged, + prepare_referenced_classic_tile_staged, prepare_referenced_htj2k_entropy_workspace, + prepare_referenced_htj2k_staged, prepare_referenced_htj2k_tile_staged, +}; + +use crate::error::{DecodingError, Result}; +use crate::{HtCodeBlockDecodeWorkspace, J2kCodeBlockDecodeWorkspace}; + +use super::allocation::DirectWorkspaceBudget; + +/// Stable location of one code block inside retained component geometry. +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct J2kDirectCodeBlockIndex { + /// Raster-ordered referenced tile-plan index. + pub tile: usize, + /// Component-plan index. + pub component: usize, + /// Step index inside the component plan. + pub step: usize, + /// Code-block index inside the sub-band step. + pub code_block: usize, +} + +/// Per-worker scalar entropy workspace shared by staged images. +#[doc(hidden)] +#[derive(Debug, Default)] +pub struct J2kDirectCpuEntropyWorkspace { + classic: J2kCodeBlockDecodeWorkspace, + ht: HtCodeBlockDecodeWorkspace, +} + +impl J2kDirectCpuEntropyWorkspace { + /// Retained HT scalar workspace bytes. + #[must_use] + pub fn retained_ht_bytes(&self) -> usize { + self.ht.allocated_bytes().unwrap_or(usize::MAX) + } + + /// Retained classic scalar workspace bytes. + #[must_use] + pub fn retained_classic_bytes(&self) -> usize { + self.classic.allocated_bytes().unwrap_or(usize::MAX) + } + + fn prepare_ht( + &mut self, + dimensions: Option<(u32, u32)>, + budget: DirectWorkspaceBudget, + ) -> Result<()> { + let (width, height) = self.prepare_ht_dimensions(dimensions)?; + if budget + .validate_workspace(self.ht.allocated_bytes()?) + .is_err() + { + self.ht = HtCodeBlockDecodeWorkspace::default(); + self.ht.prepare(width, height)?; + budget.validate_workspace(self.ht.allocated_bytes()?)?; + } + Ok(()) + } + + fn prepare_classic( + &mut self, + dimensions: Option<(u32, u32)>, + budget: DirectWorkspaceBudget, + ) -> Result<()> { + let (width, height) = self.prepare_classic_dimensions(dimensions)?; + if budget + .validate_workspace(self.classic.allocated_bytes()?) + .is_err() + { + self.classic = J2kCodeBlockDecodeWorkspace::default(); + self.classic.prepare(width, height)?; + budget.validate_workspace(self.classic.allocated_bytes()?)?; + } + Ok(()) + } + + fn prepare_ht_dimensions(&mut self, dimensions: Option<(u32, u32)>) -> Result<(u32, u32)> { + self.classic = J2kCodeBlockDecodeWorkspace::default(); + let dimensions = dimensions.ok_or(DecodingError::CodeBlockDecodeFailure)?; + self.ht.prepare(dimensions.0, dimensions.1)?; + Ok(dimensions) + } + + fn prepare_classic_dimensions(&mut self, dimensions: Option<(u32, u32)>) -> Result<(u32, u32)> { + self.ht = HtCodeBlockDecodeWorkspace::default(); + let dimensions = dimensions.ok_or(DecodingError::CodeBlockDecodeFailure)?; + self.classic.prepare(dimensions.0, dimensions.1)?; + Ok(dimensions) + } +} diff --git a/crates/j2k-native/src/direct_cpu/referenced_staged/entropy.rs b/crates/j2k-native/src/direct_cpu/referenced_staged/entropy.rs new file mode 100644 index 00000000..adf3c135 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced_staged/entropy.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::error::{bail, DecodingError, Result}; +use crate::{ + decode_ht_code_block_scalar_with_workspace, decode_j2k_code_block_scalar_with_workspace, + HtCodeBlockDecodeJob, HtCodeBlockPayloadRanges, J2kCodeBlockDecodeJob, J2kDirectGrayscaleStep, + J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, +}; + +use super::super::referenced::payload_slice; +use super::super::{ + checked_sub_band_job_output_range, DirectComponentBandScratch, DirectCpuBand, + J2kDirectCpuScratch, StagedDirectRoute, SubBandJobOutputRange, +}; +use super::plan_access::{classic_tile_components, ht_tile_components}; +use super::state::validate_active_tile; +use super::{J2kDirectCodeBlockIndex, J2kDirectCpuEntropyWorkspace}; + +/// Execute one flattened HT code block into its prepared image coefficient owner. +#[doc(hidden)] +pub fn execute_referenced_htj2k_entropy_job( + plan: &J2kReferencedHtj2kPlan, + index: J2kDirectCodeBlockIndex, + payload_arena: &[u8], + payload: HtCodeBlockPayloadRanges, + image_scratch: &mut J2kDirectCpuScratch, + worker_workspace: &mut J2kDirectCpuEntropyWorkspace, +) -> Result<()> { + validate_active_tile(image_scratch, StagedDirectRoute::Htj2k, index.tile)?; + let components = ht_tile_components(plan, index.tile)?; + let component_plan = components + .get(index.component) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let Some(J2kDirectGrayscaleStep::HtSubBand(sub_band)) = component_plan.steps.get(index.step) + else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + let job = sub_band + .jobs + .get(index.code_block) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if payload.cleanup.length != job.cleanup_length as usize + || payload.refinement.map_or(0, |range| range.length) != job.refinement_length as usize + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let data = contiguous_ht_payload(payload_arena, payload)?; + let output = code_block_output( + image_scratch, + index.component, + sub_band.band_id, + sub_band.width, + sub_band.height, + job.output_x, + job.output_y, + job.output_stride, + job.width, + job.height, + )?; + decode_ht_code_block_scalar_with_workspace( + HtCodeBlockDecodeJob { + data, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + roi_shift: job.roi_shift, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + }, + output, + &mut worker_workspace.ht, + ) +} + +/// Execute one flattened classic code block into its prepared image coefficient owner. +#[doc(hidden)] +pub fn execute_referenced_classic_entropy_job( + plan: &J2kReferencedClassicPlan, + index: J2kDirectCodeBlockIndex, + payload_arena: &[u8], + payload: crate::J2kCodestreamRange, + image_scratch: &mut J2kDirectCpuScratch, + worker_workspace: &mut J2kDirectCpuEntropyWorkspace, +) -> Result<()> { + validate_active_tile(image_scratch, StagedDirectRoute::Classic, index.tile)?; + let components = classic_tile_components(plan, index.tile)?; + let component_plan = components + .get(index.component) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let Some(J2kDirectGrayscaleStep::ClassicSubBand(sub_band)) = + component_plan.steps.get(index.step) + else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + let job = sub_band + .jobs + .get(index.code_block) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if !job.data.is_empty() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let data = payload_slice(payload_arena, payload)?; + let output = code_block_output( + image_scratch, + index.component, + sub_band.band_id, + sub_band.width, + sub_band.height, + job.output_x, + job.output_y, + job.output_stride, + job.width, + job.height, + )?; + decode_j2k_code_block_scalar_with_workspace( + J2kCodeBlockDecodeJob { + data, + segments: &job.segments, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + total_bitplanes: job.total_bitplanes, + roi_shift: job.roi_shift, + sub_band_type: job.sub_band_type, + style: job.style, + strict: job.strict, + dequantization_step: job.dequantization_step, + }, + output, + &mut worker_workspace.classic, + ) +} + +#[expect( + clippy::too_many_arguments, + reason = "the code-block destination is validated against both sub-band and block geometry" +)] +fn code_block_output( + scratch: &mut J2kDirectCpuScratch, + component: usize, + band_id: crate::J2kDirectBandId, + band_width: u32, + band_height: u32, + output_x: u32, + output_y: u32, + output_stride: usize, + width: u32, + height: u32, +) -> Result<&mut [f32]> { + let bands = scratch + .component_band_sets + .get_mut(component) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let band = find_active_band_mut(bands, band_id)?; + let sub_band_width = + usize::try_from(band_width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let output_range = checked_sub_band_job_output_range(&SubBandJobOutputRange { + output_x, + output_y, + output_stride, + width, + height, + sub_band_width, + plan_width: band_width, + plan_height: band_height, + output_len: band.coefficients.len(), + })?; + Ok(&mut band.coefficients[output_range]) +} + +fn find_active_band_mut( + bands: &mut DirectComponentBandScratch, + band_id: crate::J2kDirectBandId, +) -> Result<&mut DirectCpuBand> { + bands.bands[..bands.active_len] + .iter_mut() + .find(|band| band.band_id == band_id) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +fn contiguous_ht_payload(input: &[u8], payload: HtCodeBlockPayloadRanges) -> Result<&[u8]> { + let Some(refinement) = payload.refinement else { + return payload_slice(input, payload.cleanup); + }; + if payload.cleanup.end() != Some(refinement.offset) { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let length = payload + .cleanup + .length + .checked_add(refinement.length) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + payload_slice( + input, + crate::J2kCodestreamRange { + offset: payload.cleanup.offset, + length, + }, + ) +} diff --git a/crates/j2k-native/src/direct_cpu/referenced_staged/finish.rs b/crates/j2k-native/src/direct_cpu/referenced_staged/finish.rs new file mode 100644 index 00000000..5e1e2128 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced_staged/finish.rs @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::error::{bail, DecodingError, Result}; +use crate::{ + J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kReferencedClassicPlan, + J2kReferencedHtj2kPlan, J2kWaveletTransform, +}; + +use super::super::referenced::decoded_components as decoded_htj2k_components; +use super::super::referenced_classic::decoded_components as decoded_classic_components; +use super::super::{ + apply_inverse_mct_region, execute_idwt_step, store_component, J2kDirectCpuScratch, + J2kDirectDecodedComponents, StagedDirectRoute, +}; +use super::plan_access::{ + classic_tile_color_transform, classic_tile_components, ht_tile_color_transform, + ht_tile_components, +}; +use super::state::{finish_staged_image, finish_staged_tile}; + +/// Reconstruct and store one staged HT tile, then retire its coefficient bands. +#[doc(hidden)] +pub fn finish_referenced_htj2k_tile_staged( + plan: &J2kReferencedHtj2kPlan, + tile_index: usize, + signed: bool, + image_scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + let components = ht_tile_components(plan, tile_index)?; + let color_transform = ht_tile_color_transform(plan, tile_index)?; + let destination = plan + .tiles() + .get(tile_index) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + .destination_rect(); + finish_staged_tile( + image_scratch, + StagedDirectRoute::Htj2k, + tile_index, + components, + color_transform, + destination, + signed, + ) +} + +/// Reconstruct and store one staged classic tile, then retire its coefficient bands. +#[doc(hidden)] +pub fn finish_referenced_classic_tile_staged( + plan: &J2kReferencedClassicPlan, + tile_index: usize, + signed: bool, + image_scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + let components = classic_tile_components(plan, tile_index)?; + let color_transform = classic_tile_color_transform(plan, tile_index)?; + let destination = plan + .tiles() + .get(tile_index) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + .destination_rect(); + finish_staged_tile( + image_scratch, + StagedDirectRoute::Classic, + tile_index, + components, + color_transform, + destination, + signed, + ) +} + +/// Borrow one complete staged HT image after all tile stores complete. +#[doc(hidden)] +pub fn finish_referenced_htj2k_staged<'scratch>( + plan: &J2kReferencedHtj2kPlan, + signed: bool, + image_scratch: &'scratch mut J2kDirectCpuScratch, +) -> Result> { + let _ = signed; + finish_staged_image(image_scratch, StagedDirectRoute::Htj2k, plan.tiles().len())?; + decoded_htj2k_components(plan, image_scratch) +} + +/// Borrow one complete staged classic image after all tile stores complete. +#[doc(hidden)] +pub fn finish_referenced_classic_staged<'scratch>( + plan: &J2kReferencedClassicPlan, + signed: bool, + image_scratch: &'scratch mut J2kDirectCpuScratch, +) -> Result> { + let _ = signed; + finish_staged_image( + image_scratch, + StagedDirectRoute::Classic, + plan.tiles().len(), + )?; + decoded_classic_components(plan, image_scratch) +} + +pub(super) fn finish_tile_components( + components: &[J2kDirectGrayscalePlan], + color_transform: Option<([u8; 3], bool, J2kWaveletTransform)>, + destination: crate::J2kRect, + signed: bool, + scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + if scratch.component_band_sets.len() < components.len() + || scratch.component_planes.len() < components.len() + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + for (component_index, component) in components.iter().enumerate() { + let bands = &mut scratch.component_band_sets[component_index]; + let output = &mut scratch.component_planes[component_index]; + let mut output_initialized = true; + let mut stored = false; + for step in &component.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(_) + | J2kDirectGrayscaleStep::HtSubBand(_) => {} + J2kDirectGrayscaleStep::Idwt(step) => execute_idwt_step(step, bands)?, + J2kDirectGrayscaleStep::Store(store) => { + store_component(store, bands.active(), output, &mut output_initialized)?; + stored = true; + } + } + } + if !stored { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + if let Some((bit_depths, mct, transform)) = color_transform { + if mct { + let [plane0, plane1, plane2, ..] = scratch.component_planes.as_mut_slice() else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + apply_inverse_mct_region( + transform, + bit_depths, + signed, + destination, + plane0, + plane1, + plane2, + )?; + } + } + Ok(()) +} diff --git a/crates/j2k-native/src/direct_cpu/referenced_staged/plan_access.rs b/crates/j2k-native/src/direct_cpu/referenced_staged/plan_access.rs new file mode 100644 index 00000000..e4ff3b6e --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced_staged/plan_access.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::error::{DecodingError, Result}; +use crate::{ + J2kDirectGrayscalePlan, J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, J2kWaveletTransform, +}; + +pub(super) const fn ht_component_count(plan: &J2kReferencedHtj2kPlan) -> usize { + match plan { + J2kReferencedHtj2kPlan::Grayscale { .. } => 1, + J2kReferencedHtj2kPlan::Color { .. } => 3, + J2kReferencedHtj2kPlan::Rgba { .. } => 4, + } +} + +pub(super) const fn classic_component_count(plan: &J2kReferencedClassicPlan) -> usize { + match plan { + J2kReferencedClassicPlan::Grayscale { .. } => 1, + J2kReferencedClassicPlan::Color { .. } => 3, + J2kReferencedClassicPlan::Rgba { .. } => 4, + } +} + +pub(super) fn ht_tile_components( + plan: &J2kReferencedHtj2kPlan, + tile_index: usize, +) -> Result<&[J2kDirectGrayscalePlan]> { + let tile = plan + .tiles() + .get(tile_index) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + match plan { + J2kReferencedHtj2kPlan::Grayscale { .. } => tile + .grayscale_geometry() + .map(core::slice::from_ref) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + J2kReferencedHtj2kPlan::Color { .. } => tile + .color_geometry() + .map(|geometry| geometry.component_plans.as_slice()) + .filter(|components| components.len() == 3) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + J2kReferencedHtj2kPlan::Rgba { .. } => tile + .rgba_geometry() + .map(|geometry| geometry.component_plans.as_slice()) + .filter(|components| components.len() == 4) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + } +} + +pub(super) fn classic_tile_components( + plan: &J2kReferencedClassicPlan, + tile_index: usize, +) -> Result<&[J2kDirectGrayscalePlan]> { + let tile = plan + .tiles() + .get(tile_index) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + match plan { + J2kReferencedClassicPlan::Grayscale { .. } => tile + .grayscale_geometry() + .map(core::slice::from_ref) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + J2kReferencedClassicPlan::Color { .. } => tile + .color_geometry() + .map(|geometry| geometry.component_plans.as_slice()) + .filter(|components| components.len() == 3) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + J2kReferencedClassicPlan::Rgba { .. } => tile + .rgba_geometry() + .map(|geometry| geometry.component_plans.as_slice()) + .filter(|components| components.len() == 4) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + } +} + +pub(super) fn ht_tile_color_transform( + plan: &J2kReferencedHtj2kPlan, + tile_index: usize, +) -> Result> { + let tile = plan + .tiles() + .get(tile_index) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + match plan { + J2kReferencedHtj2kPlan::Grayscale { .. } if tile.grayscale_geometry().is_some() => Ok(None), + J2kReferencedHtj2kPlan::Color { .. } => tile + .color_geometry() + .map(|geometry| Some((geometry.bit_depths, geometry.mct, geometry.transform))) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + J2kReferencedHtj2kPlan::Rgba { .. } => tile + .rgba_geometry() + .map(|geometry| { + Some(( + [ + geometry.bit_depths[0], + geometry.bit_depths[1], + geometry.bit_depths[2], + ], + geometry.mct, + geometry.transform, + )) + }) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + J2kReferencedHtj2kPlan::Grayscale { .. } => { + Err(DecodingError::CodeBlockDecodeFailure.into()) + } + } +} + +pub(super) fn classic_tile_color_transform( + plan: &J2kReferencedClassicPlan, + tile_index: usize, +) -> Result> { + let tile = plan + .tiles() + .get(tile_index) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + match plan { + J2kReferencedClassicPlan::Grayscale { .. } if tile.grayscale_geometry().is_some() => { + Ok(None) + } + J2kReferencedClassicPlan::Color { .. } => tile + .color_geometry() + .map(|geometry| Some((geometry.bit_depths, geometry.mct, geometry.transform))) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + J2kReferencedClassicPlan::Rgba { .. } => tile + .rgba_geometry() + .map(|geometry| { + Some(( + [ + geometry.bit_depths[0], + geometry.bit_depths[1], + geometry.bit_depths[2], + ], + geometry.mct, + geometry.transform, + )) + }) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()), + J2kReferencedClassicPlan::Grayscale { .. } => { + Err(DecodingError::CodeBlockDecodeFailure.into()) + } + } +} diff --git a/crates/j2k-native/src/direct_cpu/referenced_staged/prepare.rs b/crates/j2k-native/src/direct_cpu/referenced_staged/prepare.rs new file mode 100644 index 00000000..3ce6f925 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced_staged/prepare.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::error::{bail, DecodingError, Result}; +use crate::{ + J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kReferencedClassicPlan, + J2kReferencedHtj2kPlan, +}; + +use super::super::{prepare_sub_band_output, J2kDirectCpuScratch, StagedDirectRoute}; +use super::plan_access::{ + classic_component_count, classic_tile_components, ht_component_count, ht_tile_components, +}; +use super::state::{begin_staged_image, prepare_staged_tile, EntropyRoute}; +use super::J2kDirectCpuEntropyWorkspace; +use crate::direct_cpu::allocation::{ + max_referenced_classic_dimensions, max_referenced_ht_dimensions, + prepare_referenced_classic_staged_scratch, prepare_referenced_htj2k_staged_scratch, +}; + +/// Prepare one retained worker workspace for HT jobs in `plan`. +#[doc(hidden)] +pub fn prepare_referenced_htj2k_entropy_workspace( + plan: &J2kReferencedHtj2kPlan, + worker_workspace: &mut J2kDirectCpuEntropyWorkspace, +) -> Result<()> { + worker_workspace.prepare_ht_dimensions(max_referenced_ht_dimensions(plan))?; + Ok(()) +} + +/// Prepare one retained worker workspace for classic jobs in `plan`. +#[doc(hidden)] +pub fn prepare_referenced_classic_entropy_workspace( + plan: &J2kReferencedClassicPlan, + worker_workspace: &mut J2kDirectCpuEntropyWorkspace, +) -> Result<()> { + worker_workspace.prepare_classic_dimensions(max_referenced_classic_dimensions(plan))?; + Ok(()) +} + +/// Prepare one image's retained coefficient owners for staged HT entropy work. +#[doc(hidden)] +pub fn prepare_referenced_htj2k_staged( + plan: &J2kReferencedHtj2kPlan, + image_scratch: &mut J2kDirectCpuScratch, + worker_workspace: &mut J2kDirectCpuEntropyWorkspace, +) -> Result<()> { + let budget = prepare_referenced_htj2k_staged_scratch(plan, image_scratch)?; + worker_workspace.prepare_ht(max_referenced_ht_dimensions(plan), budget)?; + begin_staged_image( + image_scratch, + StagedDirectRoute::Htj2k, + plan.tiles().len(), + plan.output_rect(), + ht_component_count(plan), + ) +} + +/// Prepare one image's retained coefficient owners for staged classic entropy work. +#[doc(hidden)] +pub fn prepare_referenced_classic_staged( + plan: &J2kReferencedClassicPlan, + image_scratch: &mut J2kDirectCpuScratch, + worker_workspace: &mut J2kDirectCpuEntropyWorkspace, +) -> Result<()> { + let budget = prepare_referenced_classic_staged_scratch(plan, image_scratch)?; + worker_workspace.prepare_classic(max_referenced_classic_dimensions(plan), budget)?; + begin_staged_image( + image_scratch, + StagedDirectRoute::Classic, + plan.tiles().len(), + plan.output_rect(), + classic_component_count(plan), + ) +} + +/// Prepare one HT tile's coefficient owners after the prior tile store has retired. +#[doc(hidden)] +pub fn prepare_referenced_htj2k_tile_staged( + plan: &J2kReferencedHtj2kPlan, + tile_index: usize, + image_scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + prepare_staged_tile( + image_scratch, + StagedDirectRoute::Htj2k, + tile_index, + ht_tile_components(plan, tile_index)?, + EntropyRoute::Htj2k, + ) +} + +/// Prepare one classic tile's coefficient owners after the prior tile store has retired. +#[doc(hidden)] +pub fn prepare_referenced_classic_tile_staged( + plan: &J2kReferencedClassicPlan, + tile_index: usize, + image_scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + prepare_staged_tile( + image_scratch, + StagedDirectRoute::Classic, + tile_index, + classic_tile_components(plan, tile_index)?, + EntropyRoute::Classic, + ) +} + +pub(super) fn prepare_entropy_bands( + components: &[J2kDirectGrayscalePlan], + route: EntropyRoute, + scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + if scratch.component_band_sets.len() < components.len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + for (component_index, component) in components.iter().enumerate() { + let bands = &mut scratch.component_band_sets[component_index]; + bands.reset(); + for step in &component.steps { + let sub_band = match (route, step) { + (EntropyRoute::Classic, J2kDirectGrayscaleStep::ClassicSubBand(sub_band)) => { + Some(( + sub_band.band_id, + sub_band.rect, + sub_band.width, + sub_band.height, + )) + } + (EntropyRoute::Htj2k, J2kDirectGrayscaleStep::HtSubBand(sub_band)) => Some(( + sub_band.band_id, + sub_band.rect, + sub_band.width, + sub_band.height, + )), + (EntropyRoute::Classic, J2kDirectGrayscaleStep::HtSubBand(_)) + | (EntropyRoute::Htj2k, J2kDirectGrayscaleStep::ClassicSubBand(_)) => { + bail!(DecodingError::CodeBlockDecodeFailure) + } + (_, J2kDirectGrayscaleStep::Idwt(_) | J2kDirectGrayscaleStep::Store(_)) => None, + }; + if let Some((band_id, rect, width, height)) = sub_band { + let _ = prepare_sub_band_output(bands, band_id, rect, width, height)?; + } + } + } + Ok(()) +} diff --git a/crates/j2k-native/src/direct_cpu/referenced_staged/state.rs b/crates/j2k-native/src/direct_cpu/referenced_staged/state.rs new file mode 100644 index 00000000..e7778bde --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced_staged/state.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::error::{bail, DecodingError, Result}; +use crate::{J2kDirectGrayscalePlan, J2kWaveletTransform}; + +use super::super::{resize_and_zero, J2kDirectCpuScratch, StagedDirectRoute, StagedDirectState}; +use super::{finish::finish_tile_components, prepare::prepare_entropy_bands}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum EntropyRoute { + Classic, + Htj2k, +} + +pub(super) fn begin_staged_image( + scratch: &mut J2kDirectCpuScratch, + route: StagedDirectRoute, + tile_count: usize, + output_rect: crate::J2kRect, + component_count: usize, +) -> Result<()> { + scratch.staged_state = None; + if tile_count == 0 + || !matches!(component_count, 1 | 3 | 4) + || scratch.component_band_sets.len() < component_count + || scratch.component_planes.len() < component_count + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let dimensions = (output_rect.width(), output_rect.height()); + let plane_len = usize::try_from(dimensions.0) + .ok() + .and_then(|width| { + usize::try_from(dimensions.1) + .ok() + .and_then(|height| width.checked_mul(height)) + }) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for component_index in 0..component_count { + scratch.component_band_sets[component_index].reset(); + let plane = &mut scratch.component_planes[component_index]; + plane.width = dimensions.0; + plane.height = dimensions.1; + resize_and_zero(&mut plane.samples, plane_len)?; + } + scratch.staged_state = Some(StagedDirectState { + route, + next_tile: 0, + active_tile: None, + tile_count, + }); + Ok(()) +} + +pub(super) fn prepare_staged_tile( + scratch: &mut J2kDirectCpuScratch, + route: StagedDirectRoute, + tile_index: usize, + components: &[J2kDirectGrayscalePlan], + entropy_route: EntropyRoute, +) -> Result<()> { + let Some(state) = scratch.staged_state else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + if state.route != route + || state.next_tile != tile_index + || state.active_tile.is_some() + || tile_index >= state.tile_count + { + scratch.staged_state = None; + bail!(DecodingError::CodeBlockDecodeFailure); + } + if let Err(error) = prepare_entropy_bands(components, entropy_route, scratch) { + scratch.staged_state = None; + return Err(error); + } + if let Some(state) = scratch.staged_state.as_mut() { + state.active_tile = Some(tile_index); + } + Ok(()) +} + +pub(super) fn finish_staged_tile( + scratch: &mut J2kDirectCpuScratch, + route: StagedDirectRoute, + tile_index: usize, + components: &[J2kDirectGrayscalePlan], + color_transform: Option<([u8; 3], bool, J2kWaveletTransform)>, + destination: crate::J2kRect, + signed: bool, +) -> Result<()> { + validate_active_tile(scratch, route, tile_index)?; + if let Err(error) = + finish_tile_components(components, color_transform, destination, signed, scratch) + { + scratch.staged_state = None; + return Err(error); + } + for bands in &mut scratch.component_band_sets[..components.len()] { + bands.reset(); + } + let Some(state) = scratch.staged_state.as_mut() else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + state.active_tile = None; + state.next_tile = state + .next_tile + .checked_add(1) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + Ok(()) +} + +pub(super) fn finish_staged_image( + scratch: &mut J2kDirectCpuScratch, + route: StagedDirectRoute, + tile_count: usize, +) -> Result<()> { + let Some(state) = scratch.staged_state else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + if state.route != route + || state.tile_count != tile_count + || state.next_tile != tile_count + || state.active_tile.is_some() + { + scratch.staged_state = None; + bail!(DecodingError::CodeBlockDecodeFailure); + } + scratch.staged_state = None; + Ok(()) +} + +pub(super) fn validate_active_tile( + scratch: &J2kDirectCpuScratch, + route: StagedDirectRoute, + tile_index: usize, +) -> Result<()> { + if scratch.staged_state.is_some_and(|state| { + state.route == route + && state.next_tile == tile_index + && state.active_tile == Some(tile_index) + && tile_index < state.tile_count + }) { + Ok(()) + } else { + Err(DecodingError::CodeBlockDecodeFailure.into()) + } +} diff --git a/crates/j2k-native/src/direct_plan.rs b/crates/j2k-native/src/direct_plan.rs index c94e486a..6910607e 100644 --- a/crates/j2k-native/src/direct_plan.rs +++ b/crates/j2k-native/src/direct_plan.rs @@ -1,8 +1,18 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + use alloc::vec::Vec; use crate::{J2kRect, J2kWaveletTransform}; +pub use j2k_types::{HtCodeBlockPayloadRanges, J2kClassicCodeBlockPayload, J2kCodestreamRange}; mod allocation; +mod referenced_classic; +pub use referenced_classic::J2kReferencedClassicPlan; +mod referenced_ht; +pub use referenced_ht::{ + J2kReferencedHtj2kPlan, J2kReferencedPayloadRecordSpan, J2kReferencedTileGeometry, + J2kReferencedTilePlan, +}; /// Adapter identifier for one device-owned grayscale coefficient band. pub type J2kDirectBandId = u32; @@ -46,6 +56,21 @@ pub struct J2kDirectColorPlan { pub component_plans: Vec, } +/// Adapter RGBA direct device plan with RGB-only inverse MCT semantics. +#[derive(Debug)] +pub struct J2kDirectRgbaPlan { + /// Final output dimensions. + pub dimensions: (u32, u32), + /// Final output bit depths in semantic R, G, B, A order. + pub bit_depths: [u8; 4], + /// Whether inverse MCT must be applied to the first three component stores. + pub mct: bool, + /// Wavelet transform used by the codestream's RGB color transform. + pub transform: J2kWaveletTransform, + /// Per-component direct plans in semantic R, G, B, A order. + pub component_plans: Vec, +} + /// Adapter owned classic J2K sub-band decode job. #[derive(Debug)] pub struct J2kOwnedSubBandPlan { diff --git a/crates/j2k-native/src/direct_plan/allocation.rs b/crates/j2k-native/src/direct_plan/allocation.rs index caa401a2..c7dd7bb6 100644 --- a/crates/j2k-native/src/direct_plan/allocation.rs +++ b/crates/j2k-native/src/direct_plan/allocation.rs @@ -5,8 +5,10 @@ use core::mem::size_of; use crate::{ - HtOwnedCodeBlockBatchJob, J2kCodeBlockSegment, J2kDirectColorPlan, J2kDirectGrayscalePlan, - J2kDirectGrayscaleStep, J2kOwnedCodeBlockBatchJob, Result, ValidationError, + HtCodeBlockPayloadRanges, HtOwnedCodeBlockBatchJob, J2kClassicCodeBlockPayload, + J2kCodeBlockSegment, J2kCodestreamRange, J2kDirectColorPlan, J2kDirectGrayscalePlan, + J2kDirectGrayscaleStep, J2kDirectRgbaPlan, J2kOwnedCodeBlockBatchJob, J2kReferencedClassicPlan, + J2kReferencedHtj2kPlan, J2kReferencedTilePlan, Result, ValidationError, DEFAULT_MAX_DECODE_BYTES, }; @@ -64,16 +66,154 @@ impl J2kDirectColorPlan { /// Returns an error if nested capacity arithmetic overflows or exceeds the /// native decode allocation ceiling. #[doc(hidden)] + pub fn retained_allocation_bytes(&self) -> Result { + retained_color_component_plan_bytes(&self.component_plans, self.component_plans.capacity()) + } +} + +impl J2kDirectRgbaPlan { + /// Return the allocator capacities retained by this direct-plan owner graph. + /// + /// The root plan value itself is not included. Callers that place the root + /// in a separate heap allocation must account that allocation separately. + /// + /// # Errors + /// + /// Returns an error if nested capacity arithmetic overflows or exceeds the + /// native decode allocation ceiling. + #[doc(hidden)] + pub fn retained_allocation_bytes(&self) -> Result { + retained_color_component_plan_bytes(&self.component_plans, self.component_plans.capacity()) + } +} + +impl J2kReferencedHtj2kPlan { + /// Return allocator capacities retained by referenced geometry and payload ranges. + /// + /// The encoded payload bytes themselves remain caller-owned and are not counted. + /// + /// # Errors + /// + /// Returns an error if nested capacity arithmetic overflows or exceeds the + /// native decode allocation ceiling. + #[doc(hidden)] pub fn retained_allocation_bytes(&self) -> Result { let mut budget = RetainedPlanBudget::default(); - budget.include_capacity::(self.component_plans.capacity())?; - for component in &self.component_plans { - include_grayscale_plan(&mut budget, component)?; + match self { + Self::Grayscale { + tiles, payloads, .. + } + | Self::Color { + tiles, payloads, .. + } + | Self::Rgba { + tiles, payloads, .. + } => { + include_referenced_tiles(&mut budget, tiles, tiles.capacity())?; + budget.include_capacity::(payloads.capacity())?; + } } Ok(budget.bytes) } } +impl J2kReferencedClassicPlan { + /// Return allocator capacities retained by referenced geometry and payload fragments. + /// + /// The encoded payload bytes themselves remain caller-owned and are not counted. + /// + /// # Errors + /// + /// Returns an error if nested capacity arithmetic overflows or exceeds the + /// native decode allocation ceiling. + #[doc(hidden)] + pub fn retained_allocation_bytes(&self) -> Result { + let mut budget = RetainedPlanBudget::default(); + match self { + Self::Grayscale { + tiles, + payloads, + ranges, + .. + } + | Self::Color { + tiles, + payloads, + ranges, + .. + } + | Self::Rgba { + tiles, + payloads, + ranges, + .. + } => { + include_referenced_tiles(&mut budget, tiles, tiles.capacity())?; + include_classic_references(&mut budget, payloads.capacity(), ranges.capacity())?; + } + } + Ok(budget.bytes) + } +} + +fn include_referenced_tiles( + budget: &mut RetainedPlanBudget, + tiles: &[J2kReferencedTilePlan], + retained_capacity: usize, +) -> Result<()> { + budget.include_capacity::(retained_capacity)?; + for tile in tiles { + if let Some(geometry) = tile.grayscale_geometry() { + include_grayscale_plan(budget, geometry)?; + } else if let Some(geometry) = tile.color_geometry() { + include_color_component_plans( + budget, + &geometry.component_plans, + geometry.component_plans.capacity(), + )?; + } else if let Some(geometry) = tile.rgba_geometry() { + include_color_component_plans( + budget, + &geometry.component_plans, + geometry.component_plans.capacity(), + )?; + } else { + return Err(ValidationError::ImageTooLarge.into()); + } + } + Ok(()) +} + +fn include_classic_references( + budget: &mut RetainedPlanBudget, + payload_capacity: usize, + range_capacity: usize, +) -> Result<()> { + budget.include_capacity::(payload_capacity)?; + budget.include_capacity::(range_capacity) +} + +fn retained_color_component_plan_bytes( + components: &[J2kDirectGrayscalePlan], + retained_capacity: usize, +) -> Result { + let mut budget = RetainedPlanBudget::default(); + include_color_component_plans(&mut budget, components, retained_capacity)?; + Ok(budget.bytes) +} + +fn include_color_component_plans( + budget: &mut RetainedPlanBudget, + components: &[J2kDirectGrayscalePlan], + retained_capacity: usize, +) -> Result<()> { + budget.include_capacity::(retained_capacity)?; + for component in components { + include_grayscale_plan(budget, component)?; + } + Ok(()) +} + fn include_grayscale_plan( budget: &mut RetainedPlanBudget, plan: &J2kDirectGrayscalePlan, diff --git a/crates/j2k-native/src/direct_plan/referenced_classic.rs b/crates/j2k-native/src/direct_plan/referenced_classic.rs new file mode 100644 index 00000000..883ffce6 --- /dev/null +++ b/crates/j2k-native/src/direct_plan/referenced_classic.rs @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Referenced classic JPEG 2000 plans without duplicated compressed payloads. + +use alloc::vec::Vec; + +use super::{ + J2kClassicCodeBlockPayload, J2kCodestreamRange, J2kDirectColorPlan, J2kDirectGrayscalePlan, + J2kDirectRgbaPlan, J2kReferencedTilePlan, +}; +use crate::J2kRect; + +/// Owned classic execution geometry whose compressed fragments remain ranges +/// into the caller-retained encoded input. +#[derive(Debug)] +pub enum J2kReferencedClassicPlan { + /// One-component grayscale plan. + Grayscale { + /// Per-tile direct execution geometry in raster order. + tiles: Vec, + /// Reduced full-image dimensions before an optional region is applied. + full_dimensions: (u32, u32), + /// Requested output rectangle in reduced full-image coordinates. + output_rect: J2kRect, + /// Code-block payload descriptors in classic step/job traversal order. + payloads: Vec, + /// Ordered encoded-input byte ranges referenced by `payloads`. + ranges: Vec, + }, + /// Three-component RGB plan. + Color { + /// Per-tile direct execution geometry in raster order. + tiles: Vec, + /// Reduced full-image dimensions before an optional region is applied. + full_dimensions: (u32, u32), + /// Requested output rectangle in reduced full-image coordinates. + output_rect: J2kRect, + /// Code-block payload descriptors in component/step/job traversal order. + payloads: Vec, + /// Ordered encoded-input byte ranges referenced by `payloads`. + ranges: Vec, + }, + /// Four-component RGBA plan in semantic R, G, B, A order. + Rgba { + /// Per-tile direct execution geometry in raster order. + tiles: Vec, + /// Reduced full-image dimensions before an optional region is applied. + full_dimensions: (u32, u32), + /// Requested output rectangle in reduced full-image coordinates. + output_rect: J2kRect, + /// Code-block payload descriptors in R/G/B/A component order. + payloads: Vec, + /// Ordered encoded-input byte ranges referenced by `payloads`. + ranges: Vec, + }, +} + +impl J2kReferencedClassicPlan { + /// Grayscale execution geometry for a legacy single-tile plan. + #[must_use] + pub fn grayscale_geometry(&self) -> Option<&J2kDirectGrayscalePlan> { + match self { + Self::Grayscale { tiles, .. } if tiles.len() == 1 => tiles[0].grayscale_geometry(), + _ => None, + } + } + + /// RGB execution geometry for a legacy single-tile plan. + #[must_use] + pub fn color_geometry(&self) -> Option<&J2kDirectColorPlan> { + match self { + Self::Color { tiles, .. } if tiles.len() == 1 => tiles[0].color_geometry(), + _ => None, + } + } + + /// RGBA execution geometry for a legacy single-tile plan. + #[must_use] + pub fn rgba_geometry(&self) -> Option<&J2kDirectRgbaPlan> { + match self { + Self::Rgba { tiles, .. } if tiles.len() == 1 => tiles[0].rgba_geometry(), + _ => None, + } + } + + /// Per-tile direct execution plans in codestream raster order. + #[must_use] + pub fn tiles(&self) -> &[J2kReferencedTilePlan] { + match self { + Self::Grayscale { tiles, .. } + | Self::Color { tiles, .. } + | Self::Rgba { tiles, .. } => tiles, + } + } + + /// Reduced full-image dimensions before an optional output region. + #[must_use] + pub const fn full_dimensions(&self) -> (u32, u32) { + match self { + Self::Grayscale { + full_dimensions, .. + } + | Self::Color { + full_dimensions, .. + } + | Self::Rgba { + full_dimensions, .. + } => *full_dimensions, + } + } + + /// Requested output rectangle in reduced full-image coordinates. + #[must_use] + pub const fn output_rect(&self) -> J2kRect { + match self { + Self::Grayscale { output_rect, .. } + | Self::Color { output_rect, .. } + | Self::Rgba { output_rect, .. } => *output_rect, + } + } + + /// Code-block payload descriptors in geometry traversal order. + #[must_use] + pub fn payloads(&self) -> &[J2kClassicCodeBlockPayload] { + match self { + Self::Grayscale { payloads, .. } + | Self::Color { payloads, .. } + | Self::Rgba { payloads, .. } => payloads, + } + } + + /// Encoded-input fragment ranges referenced by [`Self::payloads`]. + #[must_use] + pub fn ranges(&self) -> &[J2kCodestreamRange] { + match self { + Self::Grayscale { ranges, .. } + | Self::Color { ranges, .. } + | Self::Rgba { ranges, .. } => ranges, + } + } + + pub(crate) fn grayscale( + tiles: Vec, + full_dimensions: (u32, u32), + output_rect: J2kRect, + payloads: Vec, + ranges: Vec, + ) -> Self { + Self::Grayscale { + tiles, + full_dimensions, + output_rect, + payloads, + ranges, + } + } + + pub(crate) fn color( + tiles: Vec, + full_dimensions: (u32, u32), + output_rect: J2kRect, + payloads: Vec, + ranges: Vec, + ) -> Self { + Self::Color { + tiles, + full_dimensions, + output_rect, + payloads, + ranges, + } + } + + pub(crate) fn rgba( + tiles: Vec, + full_dimensions: (u32, u32), + output_rect: J2kRect, + payloads: Vec, + ranges: Vec, + ) -> Self { + Self::Rgba { + tiles, + full_dimensions, + output_rect, + payloads, + ranges, + } + } +} diff --git a/crates/j2k-native/src/direct_plan/referenced_ht.rs b/crates/j2k-native/src/direct_plan/referenced_ht.rs new file mode 100644 index 00000000..eb1570ad --- /dev/null +++ b/crates/j2k-native/src/direct_plan/referenced_ht.rs @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use alloc::vec::Vec; + +use super::{J2kDirectColorPlan, J2kDirectGrayscalePlan, J2kDirectRgbaPlan}; +use crate::{HtCodeBlockPayloadRanges, J2kRect, J2kWaveletTransform}; + +/// Contiguous range of compressed-payload records belonging to one tile plan. +/// +/// The indices address entries in the parent referenced plan's `payloads()` +/// slice. They are record indices, not byte offsets. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct J2kReferencedPayloadRecordSpan { + /// Index of the first payload record for the tile. + pub first_record: usize, + /// Number of payload records for the tile. + pub record_count: usize, +} + +impl J2kReferencedPayloadRecordSpan { + /// Exclusive payload-record index, or `None` when the span overflows. + #[must_use] + pub const fn end_record(self) -> Option { + self.first_record.checked_add(self.record_count) + } +} + +/// Direct execution geometry for one codestream tile. +#[derive(Debug)] +pub enum J2kReferencedTileGeometry { + /// One-component grayscale tile. + Grayscale(J2kDirectGrayscalePlan), + /// Three-component RGB tile. + Color(J2kDirectColorPlan), + /// Four-component RGBA tile. + Rgba(J2kDirectRgbaPlan), +} + +/// One independently executable tile in a referenced direct plan. +/// +/// Tile plans retain a separate coefficient/IDWT/store sequence so an +/// executor can release tile-local coefficient buffers at the Store boundary. +#[derive(Debug)] +pub struct J2kReferencedTilePlan { + tile_index: usize, + decoded_rect: J2kRect, + destination_rect: J2kRect, + payload_records: J2kReferencedPayloadRecordSpan, + wavelet_transform: J2kWaveletTransform, + geometry: J2kReferencedTileGeometry, +} + +impl J2kReferencedTilePlan { + /// Zero-based codestream tile index in raster order. + #[must_use] + pub const fn tile_index(&self) -> usize { + self.tile_index + } + + /// Tile/output-region intersection in reduced full-image coordinates. + #[must_use] + pub const fn decoded_rect(&self) -> J2kRect { + self.decoded_rect + } + + /// Tile/output-region intersection in dense destination coordinates. + #[must_use] + pub const fn destination_rect(&self) -> J2kRect { + self.destination_rect + } + + /// Payload-record span in the parent plan's `payloads()` slice. + #[must_use] + pub const fn payload_records(&self) -> J2kReferencedPayloadRecordSpan { + self.payload_records + } + + /// Grayscale geometry, when this tile is grayscale. + #[must_use] + pub const fn grayscale_geometry(&self) -> Option<&J2kDirectGrayscalePlan> { + match &self.geometry { + J2kReferencedTileGeometry::Grayscale(geometry) => Some(geometry), + J2kReferencedTileGeometry::Color(_) | J2kReferencedTileGeometry::Rgba(_) => None, + } + } + + /// RGB geometry, when this tile is color. + #[must_use] + pub const fn color_geometry(&self) -> Option<&J2kDirectColorPlan> { + match &self.geometry { + J2kReferencedTileGeometry::Color(geometry) => Some(geometry), + J2kReferencedTileGeometry::Grayscale(_) | J2kReferencedTileGeometry::Rgba(_) => None, + } + } + + /// RGBA geometry, when this tile has four components. + #[must_use] + pub const fn rgba_geometry(&self) -> Option<&J2kDirectRgbaPlan> { + match &self.geometry { + J2kReferencedTileGeometry::Rgba(geometry) => Some(geometry), + J2kReferencedTileGeometry::Grayscale(_) | J2kReferencedTileGeometry::Color(_) => None, + } + } + + /// Effective wavelet transform after component and tile coding-style overrides. + #[must_use] + pub const fn wavelet_transform(&self) -> J2kWaveletTransform { + self.wavelet_transform + } + + pub(crate) const fn new( + tile_index: usize, + decoded_rect: J2kRect, + destination_rect: J2kRect, + payload_records: J2kReferencedPayloadRecordSpan, + wavelet_transform: J2kWaveletTransform, + geometry: J2kReferencedTileGeometry, + ) -> Self { + Self { + tile_index, + decoded_rect, + destination_rect, + payload_records, + wavelet_transform, + geometry, + } + } +} + +/// Owned HTJ2K execution geometry whose compressed payloads remain referenced +/// by offset in the caller-retained encoded input. +#[derive(Debug)] +pub enum J2kReferencedHtj2kPlan { + /// One-component grayscale plan. + Grayscale { + /// Per-tile direct execution geometry in raster order. + tiles: Vec, + /// Reduced full-image dimensions before an optional region is applied. + full_dimensions: (u32, u32), + /// Requested output rectangle in reduced full-image coordinates. + output_rect: J2kRect, + /// Payload ranges in HT-step/job traversal order. + payloads: Vec, + }, + /// Three-component RGB plan. + Color { + /// Per-tile direct execution geometry in raster order. + tiles: Vec, + /// Reduced full-image dimensions before an optional region is applied. + full_dimensions: (u32, u32), + /// Requested output rectangle in reduced full-image coordinates. + output_rect: J2kRect, + /// Payload ranges in component/HT-step/job traversal order. + payloads: Vec, + }, + /// Four-component RGBA plan with explicit alpha semantics supplied by the caller. + Rgba { + /// Per-tile direct execution geometry in raster order. + tiles: Vec, + /// Reduced full-image dimensions before an optional region is applied. + full_dimensions: (u32, u32), + /// Requested output rectangle in reduced full-image coordinates. + output_rect: J2kRect, + /// Payload ranges in R/G/B/A component, HT-step, and job traversal order. + payloads: Vec, + }, +} + +impl J2kReferencedHtj2kPlan { + /// Grayscale execution geometry for a legacy single-tile plan. + /// + /// Multi-tile callers must use [`Self::tiles`]; this accessor returns + /// `None` instead of exposing only the first tile. + #[must_use] + pub fn grayscale_geometry(&self) -> Option<&J2kDirectGrayscalePlan> { + match self { + Self::Grayscale { tiles, .. } if tiles.len() == 1 => tiles[0].grayscale_geometry(), + _ => None, + } + } + + /// RGB execution geometry for a legacy single-tile plan. + #[must_use] + pub fn color_geometry(&self) -> Option<&J2kDirectColorPlan> { + match self { + Self::Color { tiles, .. } if tiles.len() == 1 => tiles[0].color_geometry(), + _ => None, + } + } + + /// RGBA execution geometry for a legacy single-tile plan. + #[must_use] + pub fn rgba_geometry(&self) -> Option<&J2kDirectRgbaPlan> { + match self { + Self::Rgba { tiles, .. } if tiles.len() == 1 => tiles[0].rgba_geometry(), + _ => None, + } + } + + /// Per-tile direct execution plans in codestream raster order. + #[must_use] + pub fn tiles(&self) -> &[J2kReferencedTilePlan] { + match self { + Self::Grayscale { tiles, .. } + | Self::Color { tiles, .. } + | Self::Rgba { tiles, .. } => tiles, + } + } + + /// Reduced full-image dimensions before an optional output region. + #[must_use] + pub const fn full_dimensions(&self) -> (u32, u32) { + match self { + Self::Grayscale { + full_dimensions, .. + } + | Self::Color { + full_dimensions, .. + } + | Self::Rgba { + full_dimensions, .. + } => *full_dimensions, + } + } + + /// Requested output rectangle in reduced full-image coordinates. + #[must_use] + pub const fn output_rect(&self) -> J2kRect { + match self { + Self::Grayscale { output_rect, .. } + | Self::Color { output_rect, .. } + | Self::Rgba { output_rect, .. } => *output_rect, + } + } + + /// Referenced payload ranges in geometry traversal order. + #[must_use] + pub fn payloads(&self) -> &[HtCodeBlockPayloadRanges] { + match self { + Self::Grayscale { payloads, .. } + | Self::Color { payloads, .. } + | Self::Rgba { payloads, .. } => payloads, + } + } + + pub(crate) fn grayscale( + tiles: Vec, + full_dimensions: (u32, u32), + output_rect: J2kRect, + payloads: Vec, + ) -> Self { + Self::Grayscale { + tiles, + full_dimensions, + output_rect, + payloads, + } + } + + pub(crate) fn color( + tiles: Vec, + full_dimensions: (u32, u32), + output_rect: J2kRect, + payloads: Vec, + ) -> Self { + Self::Color { + tiles, + full_dimensions, + output_rect, + payloads, + } + } + + pub(crate) fn rgba( + tiles: Vec, + full_dimensions: (u32, u32), + output_rect: J2kRect, + payloads: Vec, + ) -> Self { + Self::Rgba { + tiles, + full_dimensions, + output_rect, + payloads, + } + } +} diff --git a/crates/j2k-native/src/error.rs b/crates/j2k-native/src/error.rs index e7be7bfd..1925247d 100644 --- a/crates/j2k-native/src/error.rs +++ b/crates/j2k-native/src/error.rs @@ -315,6 +315,10 @@ pub enum DirectPlanUnsupportedReason { ColorSingleTileCodestream, /// Color direct plans require three RGB components. ColorThreeComponentRgbCodestream, + /// RGBA direct plans require an RGB image with explicit alpha. + RgbaRgbImageWithAlpha, + /// RGBA direct plans require four components in semantic R, G, B, A order. + RgbaFourComponentRgbCodestream, /// A direct component plan index did not exist. ComponentIndexOutOfRange, /// Direct component plans require unit-sampled components. @@ -514,6 +518,12 @@ const fn direct_plan_unsupported_what(reason: DirectPlanUnsupportedReason) -> &' DirectPlanUnsupportedReason::ColorThreeComponentRgbCodestream => { "direct color plan only supports three-component RGB codestreams" } + DirectPlanUnsupportedReason::RgbaRgbImageWithAlpha => { + "direct RGBA plan only supports RGB images with explicit alpha" + } + DirectPlanUnsupportedReason::RgbaFourComponentRgbCodestream => { + "direct RGBA plan only supports four-component RGB-alpha codestreams" + } DirectPlanUnsupportedReason::ComponentIndexOutOfRange => { "direct component plan index is out of range" } diff --git a/crates/j2k-native/src/image.rs b/crates/j2k-native/src/image.rs index 662c83ac..889133d3 100644 --- a/crates/j2k-native/src/image.rs +++ b/crates/j2k-native/src/image.rs @@ -91,6 +91,9 @@ impl Default for DecodeSettings { /// A JPEG2000 image or codestream. pub struct Image<'a> { + /// Complete encoded input retained by the caller. Referenced execution + /// plans express compressed payload ranges relative to this owner. + pub(crate) encoded_input: &'a [u8], /// The tile-part payload used by the legacy JPEG 2000 decoder. pub(crate) codestream: &'a [u8], /// The header of the J2C codestream. @@ -106,9 +109,24 @@ pub struct Image<'a> { pub(crate) color_space: ColorSpace, } +#[derive(Clone, Copy)] +pub(crate) struct ImageSource<'a> { + encoded_input: &'a [u8], + codestream: &'a [u8], +} + +impl<'a> ImageSource<'a> { + pub(crate) const fn new(encoded_input: &'a [u8], codestream: &'a [u8]) -> Self { + Self { + encoded_input, + codestream, + } + } +} + impl<'a> Image<'a> { pub(crate) fn from_parsed_parts( - codestream: &'a [u8], + source: ImageSource<'a>, header: Header<'a>, boxes: ImageBoxes, settings: DecodeSettings, @@ -116,7 +134,7 @@ impl<'a> Image<'a> { has_alpha: bool, ) -> Result { Self::from_parsed_parts_with_retained_baseline( - codestream, + source, header, boxes, settings, @@ -127,7 +145,7 @@ impl<'a> Image<'a> { } pub(crate) fn from_parsed_parts_with_retained_baseline( - codestream: &'a [u8], + source: ImageSource<'a>, header: Header<'a>, boxes: ImageBoxes, settings: DecodeSettings, @@ -138,7 +156,8 @@ impl<'a> Image<'a> { let metadata_bytes = retained_metadata_bytes(&header, &boxes, &color_space)?; allocation::combine_retained_bytes(retained_baseline_bytes, metadata_bytes)?; Ok(Self { - codestream, + encoded_input: source.encoded_input, + codestream: source.codestream, header, boxes, settings, diff --git a/crates/j2k-native/src/image/direct_api.rs b/crates/j2k-native/src/image/direct_api.rs index a39fb168..fd3d1bdf 100644 --- a/crates/j2k-native/src/image/direct_api.rs +++ b/crates/j2k-native/src/image/direct_api.rs @@ -11,6 +11,8 @@ use crate::{ use super::Image; +mod referenced; + impl<'a> Image<'a> { /// Build an adapter grayscale direct device plan without materializing host component planes. #[doc(hidden)] diff --git a/crates/j2k-native/src/image/direct_api/referenced.rs b/crates/j2k-native/src/image/direct_api/referenced.rs new file mode 100644 index 00000000..95bd06ff --- /dev/null +++ b/crates/j2k-native/src/image/direct_api/referenced.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Referenced HTJ2K plans whose compressed payloads remain in caller-owned input. + +use crate::error::bail; +use crate::j2c; +use crate::{ + ColorSpace, DecoderContext, DecodingError, DirectPlanUnsupportedReason, + J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, Result, +}; + +use super::super::Image; + +impl<'a> Image<'a> { + /// Build owned execution geometry whose classic compressed payload fragments + /// are ranges into this image's borrowed encoded input rather than copied buffers. + #[doc(hidden)] + pub fn build_referenced_classic_plan_region_with_context( + &self, + decoder_context: &mut DecoderContext<'a>, + output_region: (u32, u32, u32, u32), + ) -> Result { + validate_referenced_color_shape(&self.color_space, self.has_alpha)?; + let retained_metadata_bytes = self.retained_metadata_bytes()?; + decoder_context.set_output_region(Some(output_region)); + let result = if matches!(self.color_space, ColorSpace::Gray) { + j2c::build_referenced_classic_grayscale_plan( + self.codestream, + self.encoded_input, + &self.header, + retained_metadata_bytes, + decoder_context, + ) + } else if self.has_alpha { + j2c::build_referenced_classic_rgba_plan( + self.codestream, + self.encoded_input, + &self.header, + retained_metadata_bytes, + decoder_context, + ) + } else { + j2c::build_referenced_classic_color_plan( + self.codestream, + self.encoded_input, + &self.header, + retained_metadata_bytes, + decoder_context, + ) + }; + decoder_context.set_output_region(None); + result + } + + /// Build owned execution geometry whose HT compressed payloads are ranges + /// into this image's borrowed raw codestream rather than copied buffers. + #[doc(hidden)] + pub fn build_referenced_htj2k_plan_region_with_context( + &self, + decoder_context: &mut DecoderContext<'a>, + output_region: (u32, u32, u32, u32), + ) -> Result { + validate_referenced_color_shape(&self.color_space, self.has_alpha)?; + let retained_metadata_bytes = self.retained_metadata_bytes()?; + decoder_context.set_output_region(Some(output_region)); + let result = if matches!(self.color_space, ColorSpace::Gray) { + j2c::build_referenced_htj2k_grayscale_plan( + self.codestream, + self.encoded_input, + &self.header, + retained_metadata_bytes, + decoder_context, + ) + } else if self.has_alpha { + j2c::build_referenced_htj2k_rgba_plan( + self.codestream, + self.encoded_input, + &self.header, + retained_metadata_bytes, + decoder_context, + ) + } else { + j2c::build_referenced_htj2k_color_plan( + self.codestream, + self.encoded_input, + &self.header, + retained_metadata_bytes, + decoder_context, + ) + }; + decoder_context.set_output_region(None); + result + } +} + +fn validate_referenced_color_shape(color_space: &ColorSpace, has_alpha: bool) -> Result<()> { + if matches!( + (color_space, has_alpha), + (ColorSpace::Gray, false) | (ColorSpace::RGB, _) + ) { + return Ok(()); + } + let reason = if has_alpha { + DirectPlanUnsupportedReason::RgbaRgbImageWithAlpha + } else { + DirectPlanUnsupportedReason::ColorRgbImageWithoutAlpha + }; + bail!(DecodingError::DirectPlanUnsupported(reason)); +} diff --git a/crates/j2k-native/src/j2c/arithmetic_decoder.rs b/crates/j2k-native/src/j2c/arithmetic_decoder.rs index 4e4430ad..c87683df 100644 --- a/crates/j2k-native/src/j2c/arithmetic_decoder.rs +++ b/crates/j2k-native/src/j2c/arithmetic_decoder.rs @@ -229,6 +229,10 @@ impl<'a> ArithmeticDecoder<'a> { pub(crate) struct ArithmeticDecoderContext(u8); impl ArithmeticDecoderContext { + pub(crate) const fn empty() -> Self { + Self(0) + } + #[expect( clippy::inline_always, reason = "MQ state transitions are measured per-symbol hot paths" diff --git a/crates/j2k-native/src/j2c/bitplane/state.rs b/crates/j2k-native/src/j2c/bitplane/state.rs index 7fc1eafc..7e57dcbe 100644 --- a/crates/j2k-native/src/j2c/bitplane/state.rs +++ b/crates/j2k-native/src/j2c/bitplane/state.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use alloc::{vec, vec::Vec}; +use alloc::vec::Vec; use core::mem::size_of; use super::super::arithmetic_decoder::ArithmeticDecoderContext; @@ -54,6 +54,14 @@ impl BitPlaneDecodeBuffers { include_capacity::(&mut bytes, self.segment_coding_passes.capacity())?; Ok(bytes) } + + #[cfg(test)] + pub(crate) fn combined_layers_owner_for_test(&self) -> (*const u8, usize) { + ( + self.combined_layers.as_ptr(), + self.combined_layers.capacity(), + ) + } } pub(super) fn push_preallocated(values: &mut Vec, value: T) -> Option<()> { @@ -119,27 +127,38 @@ pub(crate) struct BitPlaneDecodeContext { impl Default for BitPlaneDecodeContext { fn default() -> Self { + Self::empty() + } +} + +impl BitPlaneDecodeContext { + pub(crate) const fn empty() -> Self { Self { - coefficient_states: vec![], - significant_scan_masks: vec![], - zero_coding_scan_masks: vec![], - coefficients: vec![], - neighbor_significances: vec![], + coefficient_states: Vec::new(), + significant_scan_masks: Vec::new(), + zero_coding_scan_masks: Vec::new(), + coefficients: Vec::new(), + neighbor_significances: Vec::new(), width: 0, padded_width: COEFFICIENTS_PADDING * 2, height: 0, - style: CodeBlockStyle::default(), + style: CodeBlockStyle { + selective_arithmetic_coding_bypass: false, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + high_throughput_block_coding: false, + }, bitplanes: 0, max_coding_passes: 0, strict: false, sub_band_type: SubBandType::LowLow, - contexts: [ArithmeticDecoderContext::default(); 19], + contexts: [ArithmeticDecoderContext::empty(); 19], current_bit_position: 0, } } -} -impl BitPlaneDecodeContext { pub(crate) fn prepare(&mut self, width: u32, height: u32) -> Result<()> { workspace::reset_decode_buffers(self, width, height).map(|_| ()) } @@ -215,6 +234,11 @@ impl BitPlaneDecodeContext { self.coefficients.capacity() } + #[cfg(test)] + pub(crate) fn coefficient_ptr_for_test(&self) -> *const Coefficient { + self.coefficients.as_ptr() + } + /// Completely reset context so that it can be reused for a new code-block. #[expect( clippy::trivially_copy_pass_by_ref, diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index b4f9c74a..6d874098 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -43,9 +43,15 @@ mod store; mod subband; mod subband_params; mod tier1; +mod workspace; pub(crate) use self::allocation::DecodeAllocationBudget; use self::direct_plan::collect_classic_code_block_data; -pub(crate) use self::direct_plan::{build_direct_color_plan, build_direct_grayscale_plan}; +pub(crate) use self::direct_plan::{ + build_direct_color_plan, build_direct_grayscale_plan, build_referenced_classic_color_plan, + build_referenced_classic_grayscale_plan, build_referenced_classic_rgba_plan, + build_referenced_htj2k_color_plan, build_referenced_htj2k_grayscale_plan, + build_referenced_htj2k_rgba_plan, +}; use self::store::{apply_sign_shift_after_mct, component_unsigned_level_shift, store}; use self::subband::{code_block_required_by_index, decode_component_tile_bit_planes}; #[cfg(all(test, feature = "parallel"))] @@ -61,8 +67,50 @@ use self::subband_params::{ classic_decode_job_parameters, ht_code_block_has_decodable_passes, sub_band_decode_parameters, SubBandDecodeParameters, }; +pub use self::workspace::{ + CpuDecodeParallelism, DecoderContext, DecoderWorkspace, DecoderWorkspaceStats, +}; -pub(crate) fn decode<'a>( +pub(crate) fn decode_with_capacity_retry<'a>( + data: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, + ht_decoder: &mut Option<&mut dyn HtCodeBlockDecoder>, +) -> Result<()> { + ctx.storage.release_all_allocations(); + let retained_component_bytes = ctx.tile_decode_context.retained_channel_bytes()?; + let retained_tier1_bytes = ctx.tile_decode_context.tier1_capacity_bytes()?; + let retained_idwt_bytes = ctx.tile_decode_context.idwt_capacity_bytes()?; + let retained_scratch_bytes = retained_tier1_bytes + .checked_add(retained_idwt_bytes) + .ok_or(ValidationError::ImageTooLarge)?; + ctx.record_decode_start( + retained_component_bytes, + retained_tier1_bytes, + retained_idwt_bytes, + ); + + let first_result = decode(data, header, retained_image_bytes, ctx, ht_decoder); + let result = ctx.retry_without_retained_scratch_on_capacity( + retained_scratch_bytes, + first_result, + |ctx| decode(data, header, retained_image_bytes, ctx, ht_decoder), + ); + + ctx.storage.release_all_allocations(); + let retained_component_bytes = ctx.tile_decode_context.retained_channel_bytes()?; + let retained_tier1_bytes = ctx.tile_decode_context.tier1_capacity_bytes()?; + let retained_idwt_bytes = ctx.tile_decode_context.idwt_capacity_bytes()?; + ctx.record_decode_complete( + retained_component_bytes, + retained_tier1_bytes, + retained_idwt_bytes, + ); + result +} + +fn decode<'a>( data: &'a [u8], header: &Header<'a>, retained_image_bytes: usize, @@ -75,8 +123,9 @@ pub(crate) fn decode<'a>( let mut profile_timings = DecodeProfileTimings::default(); let stage_start = profile::profile_now(profile_enabled); let mut reader = BitReader::new(data); - let mut parsed_tiles = tile::parse(&mut reader, header, reused_baseline.parser_bytes); - if reused_baseline.retained_channel_bytes != 0 + let mut parsed_tiles = tile::parse(&mut reader, header, reused_baseline.parser_live); + if reused_baseline.scratch_capacity == 0 + && reused_baseline.channel_capacity != 0 && matches!( parsed_tiles.as_ref(), Err(error) if reuse::is_capacity_error(error) @@ -86,11 +135,12 @@ pub(crate) fn decode<'a>( // otherwise valid decode to fail its aggregate allocation cap. ctx.discard_reused_channels(); reused_baseline = reuse::ReusedDecodeBaseline { - parser_bytes: retained_image_bytes, - retained_channel_bytes: 0, + parser_live: retained_image_bytes, + channel_capacity: 0, + scratch_capacity: 0, }; reader = BitReader::new(data); - parsed_tiles = tile::parse(&mut reader, header, reused_baseline.parser_bytes); + parsed_tiles = tile::parse(&mut reader, header, reused_baseline.parser_live); } let tiles = parsed_tiles?; profile_timings.parse_tiles_us += profile::elapsed_us(stage_start); @@ -103,8 +153,11 @@ pub(crate) fn decode<'a>( header, &tiles[0], tiles.structural_workspace_bytes(), - reused_baseline.retained_channel_bytes, + reused_baseline.channel_capacity, )?; + let retained_decode_base_without_scratch = retained_decode_baseline + .checked_sub(reused_baseline.scratch_capacity) + .ok_or(ValidationError::ImageTooLarge)?; let cpu_decode_parallelism = ctx.cpu_decode_parallelism; let (tile_ctx, storage) = (&mut ctx.tile_decode_context, &mut ctx.storage); @@ -128,7 +181,7 @@ pub(crate) fn decode<'a>( cpu_decode_parallelism, profile_enabled, &mut profile_timings, - retained_decode_baseline, + retained_decode_base_without_scratch, )?; } @@ -146,11 +199,10 @@ pub(crate) fn decode<'a>( emit_decode_profile_row(tile_ctx, &profile_timings, total_start); } - // The returned image only borrows channel data. Release tile graph, - // packet, Tier-1, and IDWT owners before callers allocate packed output so - // output conversion does not overlap a completed decode workspace. + // The returned image only borrows channel data. Parsed tile and packet + // graphs borrow the codestream and must be released. Tier-1 and IDWT + // owners are lifetime-free and remain available to the next image. storage.release_all_allocations(); - tile_ctx.release_tile_scratch_allocations(); Ok(()) } @@ -187,55 +239,6 @@ pub(crate) struct DecodeDebugCounters { pub(crate) ht_phase_stats: ht_block_decode::HtBlockDecodeStats, } -/// CPU parallelism policy for native JPEG 2000 decode. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum CpuDecodeParallelism { - /// Allow a single tile decode to use internal code-block parallelism. - #[default] - Auto, - /// Keep code-block decode serial for callers that already parallelize tiles. - Serial, -} - -/// A decoder context for decoding JPEG2000 images. -pub struct DecoderContext<'a> { - pub(crate) tile_decode_context: TileDecodeContext, - pub(crate) storage: DecompositionStorage<'a>, - cpu_decode_parallelism: CpuDecodeParallelism, -} - -impl Default for DecoderContext<'_> { - fn default() -> Self { - Self { - tile_decode_context: TileDecodeContext::default(), - storage: DecompositionStorage::default(), - cpu_decode_parallelism: CpuDecodeParallelism::Auto, - } - } -} - -impl DecoderContext<'_> { - pub(crate) fn release_reusable_allocations(&mut self) { - self.tile_decode_context.release_all_allocations(); - self.storage.release_all_allocations(); - } - - pub(crate) fn set_output_region(&mut self, output_region: Option<(u32, u32, u32, u32)>) { - self.tile_decode_context.output_region = output_region.map(OutputRegion::from_tuple); - } - - /// Return the native CPU decode parallelism policy. - #[must_use] - pub fn cpu_decode_parallelism(&self) -> CpuDecodeParallelism { - self.cpu_decode_parallelism - } - - /// Set the native CPU decode parallelism policy. - pub fn set_cpu_decode_parallelism(&mut self, parallelism: CpuDecodeParallelism) { - self.cpu_decode_parallelism = parallelism; - } -} - #[expect( clippy::too_many_arguments, reason = "this codec boundary keeps geometry, state buffers, and validated options explicit without allocation or indirection" @@ -253,7 +256,13 @@ fn decode_tile<'a, 'b>( retained_decode_baseline: usize, ) -> Result<()> { storage.reset_for_next_tile(); - tile_ctx.release_tile_scratch_allocations(); + let retained_scratch_bytes = tile_ctx.retained_scratch_bytes()?; + let retained_decode_baseline = retained_decode_baseline + .checked_add(retained_scratch_bytes) + .ok_or(ValidationError::ImageTooLarge)?; + if retained_decode_baseline > crate::DEFAULT_MAX_DECODE_BYTES { + return Err(ValidationError::ImageTooLarge.into()); + } storage.exact_integer_decode = tile_requires_exact_integer_decode(tile); if storage.exact_integer_decode { validate_exact_integer_decode_tile(tile)?; @@ -313,7 +322,6 @@ fn decode_tile<'a, 'b>( for (idx, component_info) in header.component_infos.iter().enumerate() { // Next, we apply the inverse discrete wavelet transform. let stage_start = profile::profile_now(profile_enabled); - tile_ctx.release_idwt_allocations(); idwt::apply( storage, tile_ctx, @@ -360,7 +368,7 @@ pub(crate) fn decode_component_tile_bit_planes_budgeted<'a>( cpu_decode_parallelism, profile_enabled, ); - tier1::release_tier1_workspace(tile_ctx, storage, tier1_workspace_bytes)?; + tier1::release_tier1_workspace(tile_ctx, storage, &tier1_workspace_bytes)?; decode_result } @@ -651,6 +659,21 @@ impl TileDecodeContext { .ok_or(ValidationError::ImageTooLarge.into()) } + pub(crate) fn idwt_capacity_bytes(&self) -> Result { + let mut bytes = 0_usize; + include_capacity::(&mut bytes, self.idwt_output.coefficients.capacity())?; + include_capacity::(&mut bytes, self.idwt_output.coefficients_i64.capacity())?; + include_capacity::(&mut bytes, self.idwt_scratch_buffer.capacity())?; + include_capacity::(&mut bytes, self.idwt_scratch_buffer_i64.capacity())?; + Ok(bytes) + } + + pub(crate) fn retained_scratch_bytes(&self) -> Result { + self.tier1_capacity_bytes()? + .checked_add(self.idwt_capacity_bytes()?) + .ok_or(ValidationError::ImageTooLarge.into()) + } + fn release_idwt_allocations(&mut self) { self.idwt_output = IDWTOutput::default(); self.idwt_scratch_buffer = Vec::new(); diff --git a/crates/j2k-native/src/j2c/decode/direct_plan.rs b/crates/j2k-native/src/j2c/decode/direct_plan.rs index 0eb8b68c..4cb7ee7f 100644 --- a/crates/j2k-native/src/j2c/decode/direct_plan.rs +++ b/crates/j2k-native/src/j2c/decode/direct_plan.rs @@ -11,9 +11,70 @@ use super::{ J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, J2kRect, J2kWaveletTransform, ResolutionTile, Result, RoiPlan, SubBand, SubBandDecodeParameters, Tile, ValidationError, Vec, }; +use crate::j2c::rect::IntRect; +use crate::{ + HtCodeBlockPayloadRanges, J2kClassicCodeBlockPayload, J2kCodestreamRange, J2kDirectRgbaPlan, + J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, J2kReferencedPayloadRecordSpan, + J2kReferencedTileGeometry, J2kReferencedTilePlan, +}; mod classic; -pub(super) use self::classic::collect_classic_code_block_data; +pub(super) use self::classic::{ + collect_classic_code_block_data, collect_referenced_classic_code_block_data, +}; +mod color; +pub(crate) use self::color::build_direct_color_plan; +mod referenced_color; +pub(crate) use self::referenced_color::{ + build_referenced_classic_color_plan, build_referenced_classic_rgba_plan, + build_referenced_htj2k_color_plan, build_referenced_htj2k_rgba_plan, +}; +mod referenced_grayscale; +pub(crate) use self::referenced_grayscale::{ + build_referenced_classic_grayscale_plan, build_referenced_htj2k_grayscale_plan, +}; +mod storage; +use self::storage::build_component_plan_from_storage; +use self::storage::sub_band::{strip_classic_payload_owners, strip_grayscale_payload_owners}; + +#[derive(Clone, Copy)] +struct PayloadRangeOwner<'a> { + encoded_input: &'a [u8], + codestream: &'a [u8], +} + +struct ClassicPayloadCollector<'a> { + payloads: &'a mut Vec, + ranges: &'a mut Vec, +} + +impl ClassicPayloadCollector<'_> { + fn prepare( + &mut self, + payload_capacity: usize, + range_capacity: usize, + budget: &mut DecodeAllocationBudget, + ) -> Result<()> { + budget.reserve_new(self.payloads, payload_capacity)?; + budget.reserve_new(self.ranges, range_capacity) + } + + fn push_range(&mut self, range: J2kCodestreamRange) -> Result<()> { + if self.ranges.len() == self.ranges.capacity() { + bail!(DecodingError::HostAllocationFailed); + } + self.ranges.push(range); + Ok(()) + } + + fn push_payload(&mut self, payload: J2kClassicCodeBlockPayload) -> Result<()> { + if self.payloads.len() == self.payloads.capacity() { + bail!(DecodingError::HostAllocationFailed); + } + self.payloads.push(payload); + Ok(()) + } +} pub(crate) fn build_direct_grayscale_plan<'a>( data: &'a [u8], @@ -22,16 +83,27 @@ pub(crate) fn build_direct_grayscale_plan<'a>( ctx: &mut DecoderContext<'a>, ) -> Result { ctx.release_reusable_allocations(); - let result = build_direct_grayscale_plan_inner(data, header, retained_image_bytes, ctx); + let result = build_direct_grayscale_plan_inner( + data, + data, + header, + retained_image_bytes, + ctx, + None, + None, + ); ctx.release_reusable_allocations(); result } fn build_direct_grayscale_plan_inner<'a>( data: &'a [u8], + payload_range_owner: &'a [u8], header: &Header<'a>, retained_image_bytes: usize, ctx: &mut DecoderContext<'a>, + ht_payloads: Option<&mut Vec>, + classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, ) -> Result { let mut reader = BitReader::new(data); let tiles = tile::parse(&mut reader, header, retained_image_bytes)?; @@ -42,7 +114,40 @@ fn build_direct_grayscale_plan_inner<'a>( )); } - let tile = &tiles[0]; + let mut next_band_id = 0; + let output_region = ctx.tile_decode_context.output_region; + build_direct_grayscale_tile_plan( + data, + payload_range_owner, + &tiles[0], + header, + tiles.structural_workspace_bytes(), + ctx, + &mut next_band_id, + output_region, + None, + ht_payloads, + classic_payloads, + ) +} + +#[expect( + clippy::too_many_arguments, + reason = "tile-local planning keeps the borrowed input, retained baseline, global band namespace, output region, and payload collector explicit" +)] +fn build_direct_grayscale_tile_plan<'a>( + data: &'a [u8], + payload_range_owner: &'a [u8], + tile: &Tile<'a>, + header: &Header<'a>, + structural_workspace_bytes: usize, + ctx: &mut DecoderContext<'a>, + next_band_id: &mut J2kDirectBandId, + decode_region: Option, + store_region: Option, + ht_payloads: Option<&mut Vec>, + mut classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, +) -> Result { if tile.component_infos.len() != 1 { bail!(DecodingError::DirectPlanUnsupported( DirectPlanUnsupportedReason::GrayscaleSingleComponentCodestream @@ -54,11 +159,11 @@ fn build_direct_grayscale_plan_inner<'a>( build::build( tile, &mut ctx.storage, - tiles.structural_workspace_bytes(), - ctx.tile_decode_context.output_region.is_some(), + structural_workspace_bytes, + decode_region.is_some(), build::BuildWorkspace::CoefficientsOnly, )?; - if let Some(output_region) = ctx.tile_decode_context.output_region { + if let Some(output_region) = decode_region { ctx.storage.roi_plan = RoiPlan::build(tile, header, &ctx.storage, output_region)?; if ctx.storage.roi_plan.is_none() { build::release_unused_roi_workspace(&mut ctx.storage, tile.component_infos.len())?; @@ -69,48 +174,69 @@ fn build_direct_grayscale_plan_inner<'a>( let component_info = &tile.component_infos[0]; let mut budget = DecodeAllocationBudget::for_storage(&ctx.storage)?; + if let Some(collector) = classic_payloads.as_deref_mut() { + collector.prepare( + ctx.storage.code_blocks.len(), + ctx.storage.segments.len(), + &mut budget, + )?; + } build_component_plan_from_storage( + PayloadRangeOwner { + encoded_input: payload_range_owner, + codestream: data, + }, tile, header, &ctx.storage, 0, component_unsigned_level_shift(component_info), &mut budget, + next_band_id, + store_region, + ht_payloads, + classic_payloads, ) } -pub(crate) fn build_direct_color_plan<'a>( - data: &'a [u8], - header: &Header<'a>, - retained_image_bytes: usize, - ctx: &mut DecoderContext<'a>, -) -> Result { - ctx.release_reusable_allocations(); - let result = build_direct_color_plan_inner(data, header, retained_image_bytes, ctx); - ctx.release_reusable_allocations(); - result +fn referenced_output_region(header: &Header<'_>, ctx: &DecoderContext<'_>) -> super::OutputRegion { + ctx.tile_decode_context + .output_region + .unwrap_or(super::OutputRegion { + x: 0, + y: 0, + width: header.size_data.image_width(), + height: header.size_data.image_height(), + }) } -fn build_direct_color_plan_inner<'a>( - data: &'a [u8], - header: &Header<'a>, - retained_image_bytes: usize, - ctx: &mut DecoderContext<'a>, -) -> Result { - let mut reader = BitReader::new(data); - let tiles = tile::parse(&mut reader, header, retained_image_bytes)?; +fn output_region_rect(output_region: super::OutputRegion) -> J2kRect { + J2kRect { + x0: output_region.x, + y0: output_region.y, + x1: output_region.x.saturating_add(output_region.width), + y1: output_region.y.saturating_add(output_region.height), + } +} - if tiles.len() != 1 { +fn validate_grayscale_tile(tile: &Tile<'_>) -> Result<()> { + if tile.component_infos.len() != 1 { bail!(DecodingError::DirectPlanUnsupported( - DirectPlanUnsupportedReason::ColorSingleTileCodestream + DirectPlanUnsupportedReason::GrayscaleSingleComponentCodestream )); } + validate_unit_sampled_component(&tile.component_infos[0]) +} - let tile = &tiles[0]; - if tile.component_infos.len() != 3 { - bail!(DecodingError::DirectPlanUnsupported( - DirectPlanUnsupportedReason::ColorThreeComponentRgbCodestream - )); +fn validate_color_tile( + tile: &Tile<'_>, + component_count_error: DirectPlanUnsupportedReason, +) -> Result<()> { + if tile.component_infos.len() != COMPONENT_COUNT { + bail!(DecodingError::DirectPlanUnsupported(component_count_error)); + } + for component_info in &tile.component_infos { + validate_unit_sampled_component(component_info)?; } let transform = tile.component_infos[0].wavelet_transform(); if tile.mct @@ -119,78 +245,10 @@ fn build_direct_color_plan_inner<'a>( { bail!(ColorError::Mct); } - - ctx.tile_decode_context.channel_data.clear(); - ctx.storage.reset_for_next_tile(); - - build::build( - tile, - &mut ctx.storage, - tiles.structural_workspace_bytes(), - ctx.tile_decode_context.output_region.is_some(), - build::BuildWorkspace::CoefficientsOnly, - )?; - if let Some(output_region) = ctx.tile_decode_context.output_region { - ctx.storage.roi_plan = RoiPlan::build(tile, header, &ctx.storage, output_region)?; - if ctx.storage.roi_plan.is_none() { - build::release_unused_roi_workspace(&mut ctx.storage, tile.component_infos.len())?; - } - } - - segment::parse(tile, progression_iterator(tile)?, header, &mut ctx.storage)?; - - let mut bit_depths = [0_u8; 3]; - let mut budget = DecodeAllocationBudget::for_storage(&ctx.storage)?; - let mut component_plans = Vec::new(); - budget.reserve_new(&mut component_plans, bit_depths.len())?; - for (component_idx, bit_depth) in bit_depths.iter_mut().enumerate() { - let component_info = &tile.component_infos[component_idx]; - *bit_depth = component_info.size_info.precision; - let addend = if tile.mct { - 0.0 - } else { - component_unsigned_level_shift(component_info) - }; - component_plans.push(build_component_plan_from_storage( - tile, - header, - &ctx.storage, - component_idx, - addend, - &mut budget, - )?); - } - - Ok(J2kDirectColorPlan { - dimensions: ( - header.size_data.image_width(), - header.size_data.image_height(), - ), - bit_depths, - mct: tile.mct, - transform: J2kWaveletTransform::from(transform), - component_plans, - }) + Ok(()) } -#[expect( - clippy::too_many_lines, - reason = "the ordered JPEG 2000 state machine stays cohesive to preserve marker, packet, pass, and sample order" -)] -fn build_component_plan_from_storage( - tile: &Tile<'_>, - header: &Header<'_>, - storage: &DecompositionStorage<'_>, - component_idx: usize, - store_addend: f32, - budget: &mut DecodeAllocationBudget, -) -> Result { - let component_info = - tile.component_infos - .get(component_idx) - .ok_or(DecodingError::DirectPlanUnsupported( - DirectPlanUnsupportedReason::ComponentIndexOutOfRange, - ))?; +fn validate_unit_sampled_component(component_info: &ComponentInfo) -> Result<()> { if component_info.size_info.horizontal_resolution != 1 || component_info.size_info.vertical_resolution != 1 { @@ -198,290 +256,162 @@ fn build_component_plan_from_storage( DirectPlanUnsupportedReason::ComponentUnitSampled )); } + Ok(()) +} - let tile_decompositions = storage.tile_decompositions.get(component_idx).ok_or( - DecodingError::DirectPlanUnsupported( - DirectPlanUnsupportedReason::ComponentDecompositionIndexOutOfRange, - ), - )?; - let decompositions = &storage.decompositions[tile_decompositions.decompositions.clone()]; - let active_decomposition_count = decompositions - .len() - .saturating_sub(header.skipped_resolution_levels as usize); - let sub_band_step_count = (0..component_info.num_resolution_levels() - - header.skipped_resolution_levels) - .try_fold(0_usize, |total, resolution| { - tile_decompositions - .sub_band_iter(resolution, &storage.decompositions) - .count() - .checked_add(total) - .ok_or(ValidationError::ImageTooLarge) - })?; - let step_capacity = sub_band_step_count - .checked_add(active_decomposition_count) - .and_then(|count| count.checked_add(1)) - .ok_or(ValidationError::ImageTooLarge)?; - let mut steps = Vec::new(); - budget.reserve_new(&mut steps, step_capacity)?; - let mut next_band_id: J2kDirectBandId = 0; - let mut sub_band_ids = Vec::new(); - budget.resize_new(&mut sub_band_ids, storage.sub_bands.len(), None)?; - - for resolution in 0..component_info.num_resolution_levels() - header.skipped_resolution_levels { - let sub_band_iter = tile_decompositions.sub_band_iter(resolution, &storage.decompositions); - for sub_band_idx in sub_band_iter { - if let Some(step) = build_grayscale_sub_band_step( - &storage.sub_bands[sub_band_idx], - sub_band_idx, - next_band_id, - resolution, - component_info, - storage, - header, - budget, - )? { - sub_band_ids[sub_band_idx] = Some(next_band_id); - next_band_id = next_band_id - .checked_add(1) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - steps.push(step); - } - } - } - - let mut current_ll_rect = storage.sub_bands[tile_decompositions.first_ll_sub_band].rect; - let mut current_ll_band_id = sub_band_ids[tile_decompositions.first_ll_sub_band] +fn tile_intersects_output( + tile: &Tile<'_>, + header: &Header<'_>, + output_region: super::OutputRegion, +) -> Result { + let component_info = tile + .component_infos + .first() .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let decompositions = &decompositions[..active_decomposition_count]; - for decomposition in decompositions { - let hl = &storage.sub_bands[decomposition.sub_bands[0]]; - let lh = &storage.sub_bands[decomposition.sub_bands[1]]; - let hh = &storage.sub_bands[decomposition.sub_bands[2]]; - let output_band_id = next_band_id; - next_band_id = next_band_id - .checked_add(1) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - steps.push(J2kDirectGrayscaleStep::Idwt(J2kDirectIdwtStep { - output_band_id, - rect: J2kRect::from(decomposition.rect), - transform: J2kWaveletTransform::from(component_info.wavelet_transform()), - ll_band_id: current_ll_band_id, - ll: J2kRect::from(current_ll_rect), - hl_band_id: sub_band_ids[decomposition.sub_bands[0]] - .ok_or(DecodingError::CodeBlockDecodeFailure)?, - hl: J2kRect::from(hl.rect), - lh_band_id: sub_band_ids[decomposition.sub_bands[1]] - .ok_or(DecodingError::CodeBlockDecodeFailure)?, - lh: J2kRect::from(lh.rect), - hh_band_id: sub_band_ids[decomposition.sub_bands[2]] - .ok_or(DecodingError::CodeBlockDecodeFailure)?, - hh: J2kRect::from(hh.rect), - })); - current_ll_rect = decomposition.rect; - current_ll_band_id = output_band_id; - } - + validate_unit_sampled_component(component_info)?; let component_tile = ComponentTile::new(tile, component_info); let resolution_tile = ResolutionTile::new( component_tile, component_info.num_resolution_levels() - 1 - header.skipped_resolution_levels, ); - let image_x_offset = header.size_data.image_area_x_offset; - let image_y_offset = header.size_data.image_area_y_offset; - let source_x = image_x_offset.saturating_sub(current_ll_rect.x0); - let source_y = image_y_offset.saturating_sub(current_ll_rect.y0); - let copy_width = resolution_tile - .rect - .width() - .min(current_ll_rect.width().saturating_sub(source_x)); - let copy_height = resolution_tile - .rect - .height() - .min(current_ll_rect.height().saturating_sub(source_y)); - let output_x = resolution_tile.rect.x0.saturating_sub(image_x_offset); - let output_y = resolution_tile.rect.y0.saturating_sub(image_y_offset); - steps.push(J2kDirectGrayscaleStep::Store(J2kDirectStoreStep { - input_band_id: current_ll_band_id, - input_rect: J2kRect::from(current_ll_rect), - source_x, - source_y, - copy_width, - copy_height, - output_width: header.size_data.image_width(), - output_height: header.size_data.image_height(), - output_x, - output_y, - addend: store_addend, - })); - - let sub_band_id_capacity = sub_band_ids.capacity(); - drop(sub_band_ids); - budget.release_elements::>(sub_band_id_capacity)?; + let x_offset = header + .size_data + .image_area_x_offset + .div_ceil(header.size_data.x_shrink_factor); + let y_offset = header + .size_data + .image_area_y_offset + .div_ceil(header.size_data.y_shrink_factor); + let request_left = output_region.x.saturating_add(x_offset); + let request_top = output_region.y.saturating_add(y_offset); + let request_right = request_left.saturating_add(output_region.width); + let request_bottom = request_top.saturating_add(output_region.height); + Ok(resolution_tile.rect.x0 < request_right + && request_left < resolution_tile.rect.x1 + && resolution_tile.rect.y0 < request_bottom + && request_top < resolution_tile.rect.y1) +} - Ok(J2kDirectGrayscalePlan { - dimensions: ( - header.size_data.image_width(), - header.size_data.image_height(), - ), - bit_depth: component_info.size_info.precision, - steps, +fn payload_record_span( + first_record: usize, + record_count: usize, +) -> Result { + first_record + .checked_add(record_count) + .ok_or(ValidationError::ImageTooLarge)?; + Ok(J2kReferencedPayloadRecordSpan { + first_record, + record_count, }) } -#[expect( - clippy::too_many_arguments, - clippy::too_many_lines, - reason = "the direct-plan boundary keeps validated band identity, geometry, storage, and the shared live budget explicit" -)] -fn build_grayscale_sub_band_step( - sub_band: &SubBand, - sub_band_idx: usize, - band_id: J2kDirectBandId, - resolution: u8, - component_info: &ComponentInfo, - storage: &DecompositionStorage<'_>, - header: &Header<'_>, - budget: &mut DecodeAllocationBudget, -) -> Result> { - let SubBandDecodeParameters { - dequantization_step, - num_bitplanes, - } = sub_band_decode_parameters(sub_band, resolution, component_info)?; - - if component_info - .coding_style - .parameters - .code_block_style - .uses_high_throughput_block_coding() - { - let coded_bitplanes = - add_roi_shift_to_bitplanes(num_bitplanes, component_info.roi_shift, 31)?; - let stripe_causal = component_info - .coding_style - .parameters - .code_block_style - .vertically_causal_context; - let job_capacity = direct_sub_band_job_capacity(sub_band, storage)?; - let mut jobs = Vec::new(); - budget.reserve_new(&mut jobs, job_capacity)?; - for precinct in sub_band - .precincts - .clone() - .map(|idx| &storage.precincts[idx]) - { - for code_block in precinct - .code_blocks - .clone() - .map(|idx| &storage.code_blocks[idx]) - { - if !code_block_required_by_index(storage, sub_band_idx, code_block) { - continue; - } - if !ht_code_block_has_decodable_passes(code_block, coded_bitplanes, header.strict)? - { - continue; - } +fn append_decode_elements(destination: &mut Vec, source: &mut Vec) -> Result<()> { + let target_len = destination + .len() + .checked_add(source.len()) + .ok_or(ValidationError::ImageTooLarge)?; + crate::try_reserve_decode_elements(destination, target_len)?; + destination.append(source); + Ok(()) +} - let combined = - ht_block_decode::collect_code_block_data(code_block, storage, budget)?; - jobs.push(HtOwnedCodeBlockBatchJob { - output_x: code_block.rect.x0 - sub_band.rect.x0, - output_y: code_block.rect.y0 - sub_band.rect.y0, - data: combined.data, - cleanup_length: combined.cleanup_length, - refinement_length: combined.refinement_length, - width: code_block.rect.width(), - height: code_block.rect.height(), - output_stride: sub_band.rect.width() as usize, - missing_bit_planes: code_block.missing_bit_planes, - number_of_coding_passes: code_block.number_of_coding_passes, - num_bitplanes, - roi_shift: component_info.roi_shift, - stripe_causal, - strict: header.strict, - dequantization_step, - }); - } +fn append_classic_payload_records( + payloads: &mut Vec, + ranges: &mut Vec, + tile_payloads: &mut Vec, + tile_ranges: &mut Vec, +) -> Result<()> { + let range_base = ranges.len(); + for payload in tile_payloads.iter_mut() { + payload.first_range = payload + .first_range + .checked_add(range_base) + .ok_or(ValidationError::ImageTooLarge)?; + let end_range = payload.end_range().ok_or(ValidationError::ImageTooLarge)?; + let combined_range_len = range_base + .checked_add(tile_ranges.len()) + .ok_or(ValidationError::ImageTooLarge)?; + if end_range > combined_range_len { + bail!(DecodingError::CodeBlockDecodeFailure); } - - return Ok(Some(J2kDirectGrayscaleStep::HtSubBand( - HtOwnedSubBandPlan { - band_id, - rect: J2kRect::from(sub_band.rect), - width: sub_band.rect.width(), - height: sub_band.rect.height(), - jobs, - }, - ))); } + append_decode_elements(payloads, tile_payloads)?; + append_decode_elements(ranges, tile_ranges) +} - let (classic_job_sub_band_type, classic_job_style) = - classic_decode_job_parameters(sub_band.sub_band_type, component_info); +fn grayscale_plan_rects( + geometry: &J2kDirectGrayscalePlan, + output_rect: J2kRect, +) -> Result<(J2kRect, J2kRect)> { + let mut stores = geometry.steps.iter().filter_map(|step| match step { + J2kDirectGrayscaleStep::Store(store) => Some(store), + J2kDirectGrayscaleStep::ClassicSubBand(_) + | J2kDirectGrayscaleStep::HtSubBand(_) + | J2kDirectGrayscaleStep::Idwt(_) => None, + }); + let store = stores.next().ok_or(DecodingError::CodeBlockDecodeFailure)?; + if stores.next().is_some() || store.copy_width == 0 || store.copy_height == 0 { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let destination_rect = J2kRect { + x0: store.output_x, + y0: store.output_y, + x1: store.output_x.saturating_add(store.copy_width), + y1: store.output_y.saturating_add(store.copy_height), + }; + let decoded_rect = J2kRect { + x0: output_rect.x0.saturating_add(destination_rect.x0), + y0: output_rect.y0.saturating_add(destination_rect.y0), + x1: output_rect.x0.saturating_add(destination_rect.x1), + y1: output_rect.y0.saturating_add(destination_rect.y1), + }; + Ok((decoded_rect, destination_rect)) +} - let job_capacity = direct_sub_band_job_capacity(sub_band, storage)?; - let mut jobs = Vec::new(); - budget.reserve_new(&mut jobs, job_capacity)?; - for precinct in sub_band - .precincts - .clone() - .map(|idx| &storage.precincts[idx]) - { - for code_block in precinct - .code_blocks - .clone() - .map(|idx| &storage.code_blocks[idx]) - { - if !code_block_required_by_index(storage, sub_band_idx, code_block) { - continue; - } - let (combined_data, segments) = collect_classic_code_block_data( - code_block, - &component_info.coding_style.parameters.code_block_style, - storage, - budget, - )?; - jobs.push(J2kOwnedCodeBlockBatchJob { - output_x: code_block.rect.x0 - sub_band.rect.x0, - output_y: code_block.rect.y0 - sub_band.rect.y0, - data: combined_data, - segments, - width: code_block.rect.width(), - height: code_block.rect.height(), - output_stride: sub_band.rect.width() as usize, - missing_bit_planes: code_block.missing_bit_planes, - number_of_coding_passes: code_block.number_of_coding_passes, - total_bitplanes: num_bitplanes, - roi_shift: component_info.roi_shift, - sub_band_type: classic_job_sub_band_type, - style: classic_job_style, - strict: header.strict, - dequantization_step, - }); +fn color_plan_rects( + component_plans: &[J2kDirectGrayscalePlan], + output_rect: J2kRect, +) -> Result<(J2kRect, J2kRect)> { + if component_plans.len() != COMPONENT_COUNT { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let expected = grayscale_plan_rects(&component_plans[0], output_rect)?; + for component in &component_plans[1..] { + if grayscale_plan_rects(component, output_rect)? != expected { + bail!(DecodingError::CodeBlockDecodeFailure); } } + Ok(expected) +} - Ok(Some(J2kDirectGrayscaleStep::ClassicSubBand( - J2kOwnedSubBandPlan { - band_id, - rect: J2kRect::from(sub_band.rect), - width: sub_band.rect.width(), - height: sub_band.rect.height(), - jobs, - }, - ))) +fn validate_and_strip_referenced_payload_owners( + component_plans: &mut [J2kDirectGrayscalePlan], + payload_count: usize, +) -> Result<()> { + let mut job_count = 0_usize; + for component in component_plans { + job_count = job_count + .checked_add(strip_grayscale_payload_owners(component)?) + .ok_or(ValidationError::ImageTooLarge)?; + } + if job_count != payload_count { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(()) } -fn direct_sub_band_job_capacity( - sub_band: &SubBand, - storage: &DecompositionStorage<'_>, -) -> Result { - sub_band - .precincts - .clone() - .map(|idx| storage.precincts[idx].code_blocks.len()) - .try_fold(0_usize, |total, count| { - total - .checked_add(count) - .ok_or(ValidationError::ImageTooLarge.into()) - }) +fn validate_and_strip_classic_payload_owners( + component_plans: &mut [J2kDirectGrayscalePlan], + payload_count: usize, +) -> Result<()> { + let mut job_count = 0usize; + for component in component_plans { + job_count = job_count + .checked_add(strip_classic_payload_owners(component)?) + .ok_or(ValidationError::ImageTooLarge)?; + } + if job_count != payload_count { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(()) } diff --git a/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs b/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs index 77a26b3c..84f7cd28 100644 --- a/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs +++ b/crates/j2k-native/src/j2c/decode/direct_plan/classic.rs @@ -13,6 +13,7 @@ use alloc::vec::Vec; struct ClassicAllocationCounts { data_bytes: usize, segments: usize, + fragments: usize, } #[expect( @@ -107,6 +108,104 @@ pub(crate) fn collect_classic_code_block_data( Ok((combined_data, collected_segments)) } +#[expect( + clippy::trivially_copy_pass_by_ref, + reason = "the stable codec boundary borrows shared Copy metadata used across nested calls" +)] +pub(crate) fn collect_referenced_classic_code_block_data( + code_block: &CodeBlock, + style: &CodeBlockStyle, + storage: &DecompositionStorage<'_>, + budget: &mut DecodeAllocationBudget, + mut append_fragment: impl FnMut(&[u8]) -> Result<()>, +) -> Result<(usize, usize, Vec)> { + let counts = classic_allocation_counts(code_block, *style, storage)?; + budget.include_elements::(counts.segments)?; + + let mut collected_segments = Vec::new(); + try_reserve_decode_elements(&mut collected_segments, counts.segments)?; + budget.include_capacity_overage::( + counts.segments, + collected_segments.capacity(), + )?; + + let mut data_bytes = 0usize; + let mut fragment_count = 0usize; + let mut last_segment_idx = 0u8; + let mut segment_start_offset = 0usize; + let mut segment_start_coding_pass = 0u8; + let mut coding_passes = 0u8; + let is_normal_mode = + !style.selective_arithmetic_coding_bypass && !style.termination_on_each_pass; + + for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { + let Some(range) = layer.segments.clone() else { + continue; + }; + + for segment in &storage.segments[range] { + if segment.idx != last_segment_idx { + validate_next_segment_index(last_segment_idx, segment.idx)?; + if coding_passes > segment_start_coding_pass || data_bytes > segment_start_offset { + collected_segments.push(classic_segment( + *style, + segment_start_offset, + data_bytes, + segment_start_coding_pass, + coding_passes, + )?); + } + segment_start_offset = data_bytes; + segment_start_coding_pass = coding_passes; + last_segment_idx = segment.idx; + } + + if !segment.data.is_empty() { + append_fragment(segment.data)?; + fragment_count = fragment_count + .checked_add(1) + .ok_or(ValidationError::ImageTooLarge)?; + } + data_bytes = data_bytes + .checked_add(segment.data.len()) + .ok_or(ValidationError::ImageTooLarge)?; + coding_passes = coding_passes.saturating_add(segment.coding_pases); + } + } + + if coding_passes > segment_start_coding_pass || data_bytes > segment_start_offset { + collected_segments.push(classic_segment( + *style, + segment_start_offset, + data_bytes, + segment_start_coding_pass, + coding_passes, + )?); + } + + if is_normal_mode { + collected_segments.clear(); + collected_segments.push(J2kCodeBlockSegment { + data_offset: 0, + data_length: u32::try_from(data_bytes) + .map_err(|_| DecodingError::CodeBlockDecodeFailure)?, + start_coding_pass: 0, + end_coding_pass: coding_passes, + use_arithmetic: true, + }); + } + + if coding_passes != code_block.number_of_coding_passes + || data_bytes != counts.data_bytes + || fragment_count != counts.fragments + || collected_segments.len() > counts.segments + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + Ok((fragment_count, data_bytes, collected_segments)) +} + fn classic_allocation_counts( code_block: &CodeBlock, style: CodeBlockStyle, @@ -118,6 +217,7 @@ fn classic_allocation_counts( let mut segment_start_offset = 0_usize; let mut segment_start_coding_pass = 0_u8; let mut coding_passes = 0_u8; + let mut fragment_count = 0_usize; for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { let Some(range) = layer.segments.clone() else { @@ -138,6 +238,11 @@ fn classic_allocation_counts( data_bytes = data_bytes .checked_add(segment.data.len()) .ok_or(ValidationError::ImageTooLarge)?; + if !segment.data.is_empty() { + fragment_count = fragment_count + .checked_add(1) + .ok_or(ValidationError::ImageTooLarge)?; + } coding_passes = coding_passes.saturating_add(segment.coding_pases); } } @@ -154,6 +259,7 @@ fn classic_allocation_counts( Ok(ClassicAllocationCounts { data_bytes, segments: segment_count, + fragments: fragment_count, }) } diff --git a/crates/j2k-native/src/j2c/decode/direct_plan/color.rs b/crates/j2k-native/src/j2c/decode/direct_plan/color.rs new file mode 100644 index 00000000..682c55bb --- /dev/null +++ b/crates/j2k-native/src/j2c/decode/direct_plan/color.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Direct RGB/RGBA plan construction. + +use super::{ + bail, build, build_component_plan_from_storage, component_unsigned_level_shift, + progression_iterator, segment, tile, BitReader, ClassicPayloadCollector, ColorError, + DecodeAllocationBudget, DecoderContext, DecodingError, DirectPlanUnsupportedReason, Header, + HtCodeBlockPayloadRanges, J2kDirectBandId, J2kDirectColorPlan, J2kDirectGrayscalePlan, + J2kDirectRgbaPlan, J2kWaveletTransform, PayloadRangeOwner, Result, RoiPlan, Tile, Vec, +}; + +pub(crate) fn build_direct_color_plan<'a>( + data: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, +) -> Result { + ctx.release_reusable_allocations(); + let result = build_direct_color_components_plan_inner::<3>( + data, + data, + header, + retained_image_bytes, + ctx, + DirectPlanUnsupportedReason::ColorThreeComponentRgbCodestream, + None, + None, + ) + .map(DirectColorComponentPlans::into_rgb); + ctx.release_reusable_allocations(); + result +} + +pub(super) struct DirectColorComponentPlans { + dimensions: (u32, u32), + bit_depths: [u8; COMPONENT_COUNT], + mct: bool, + transform: J2kWaveletTransform, + pub(super) component_plans: Vec, +} + +impl DirectColorComponentPlans<3> { + pub(super) fn into_rgb(self) -> J2kDirectColorPlan { + J2kDirectColorPlan { + dimensions: self.dimensions, + bit_depths: self.bit_depths, + mct: self.mct, + transform: self.transform, + component_plans: self.component_plans, + } + } +} + +impl DirectColorComponentPlans<4> { + pub(super) fn into_rgba(self) -> J2kDirectRgbaPlan { + J2kDirectRgbaPlan { + dimensions: self.dimensions, + bit_depths: self.bit_depths, + mct: self.mct, + transform: self.transform, + component_plans: self.component_plans, + } + } +} + +#[expect( + clippy::too_many_arguments, + reason = "color planning keeps the borrowed codestream, decode context, component contract, and mutually exclusive referenced payload collectors explicit" +)] +fn build_direct_color_components_plan_inner<'a, const COMPONENT_COUNT: usize>( + data: &'a [u8], + payload_range_owner: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, + component_count_error: DirectPlanUnsupportedReason, + ht_payloads: Option<&mut Vec>, + classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, +) -> Result> { + let mut reader = BitReader::new(data); + let tiles = tile::parse(&mut reader, header, retained_image_bytes)?; + + if tiles.len() != 1 { + bail!(DecodingError::DirectPlanUnsupported( + DirectPlanUnsupportedReason::ColorSingleTileCodestream + )); + } + + let mut next_band_id = 0; + let output_region = ctx.tile_decode_context.output_region; + build_direct_color_tile_components_plan::( + data, + payload_range_owner, + &tiles[0], + header, + tiles.structural_workspace_bytes(), + ctx, + component_count_error, + &mut next_band_id, + output_region, + None, + ht_payloads, + classic_payloads, + ) +} + +#[expect( + clippy::too_many_arguments, + reason = "tile-local color planning keeps the borrowed input, component contract, global band namespace, output region, and payload collector explicit" +)] +pub(super) fn build_direct_color_tile_components_plan<'a, const COMPONENT_COUNT: usize>( + data: &'a [u8], + payload_range_owner: &'a [u8], + tile: &Tile<'a>, + header: &Header<'a>, + structural_workspace_bytes: usize, + ctx: &mut DecoderContext<'a>, + component_count_error: DirectPlanUnsupportedReason, + next_band_id: &mut J2kDirectBandId, + decode_region: Option, + store_region: Option, + mut ht_payloads: Option<&mut Vec>, + mut classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, +) -> Result> { + if tile.component_infos.len() != COMPONENT_COUNT { + bail!(DecodingError::DirectPlanUnsupported(component_count_error)); + } + let transform = tile.component_infos[0].wavelet_transform(); + if tile.mct + && (transform != tile.component_infos[1].wavelet_transform() + || transform != tile.component_infos[2].wavelet_transform()) + { + bail!(ColorError::Mct); + } + + ctx.tile_decode_context.channel_data.clear(); + ctx.storage.reset_for_next_tile(); + + build::build( + tile, + &mut ctx.storage, + structural_workspace_bytes, + decode_region.is_some(), + build::BuildWorkspace::CoefficientsOnly, + )?; + if let Some(output_region) = decode_region { + ctx.storage.roi_plan = RoiPlan::build(tile, header, &ctx.storage, output_region)?; + if ctx.storage.roi_plan.is_none() { + build::release_unused_roi_workspace(&mut ctx.storage, tile.component_infos.len())?; + } + } + + segment::parse(tile, progression_iterator(tile)?, header, &mut ctx.storage)?; + + let mut bit_depths = [0_u8; COMPONENT_COUNT]; + let mut budget = DecodeAllocationBudget::for_storage(&ctx.storage)?; + if let Some(collector) = classic_payloads.as_deref_mut() { + collector.prepare( + ctx.storage.code_blocks.len(), + ctx.storage.segments.len(), + &mut budget, + )?; + } + let mut component_plans = Vec::new(); + budget.reserve_new(&mut component_plans, bit_depths.len())?; + for (component_idx, bit_depth) in bit_depths.iter_mut().enumerate() { + let component_info = &tile.component_infos[component_idx]; + *bit_depth = component_info.size_info.precision; + let addend = if tile.mct && component_idx < 3 { + 0.0 + } else { + component_unsigned_level_shift(component_info) + }; + component_plans.push(build_component_plan_from_storage( + PayloadRangeOwner { + encoded_input: payload_range_owner, + codestream: data, + }, + tile, + header, + &ctx.storage, + component_idx, + addend, + &mut budget, + next_band_id, + store_region, + ht_payloads.as_deref_mut(), + classic_payloads.as_deref_mut(), + )?); + } + + let dimensions = store_region.map_or_else( + || { + ( + header.size_data.image_width(), + header.size_data.image_height(), + ) + }, + |region| (region.width, region.height), + ); + Ok(DirectColorComponentPlans { + dimensions, + bit_depths, + mct: tile.mct, + transform: J2kWaveletTransform::from(transform), + component_plans, + }) +} diff --git a/crates/j2k-native/src/j2c/decode/direct_plan/referenced_color.rs b/crates/j2k-native/src/j2c/decode/direct_plan/referenced_color.rs new file mode 100644 index 00000000..6edadfd5 --- /dev/null +++ b/crates/j2k-native/src/j2c/decode/direct_plan/referenced_color.rs @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Retained RGB/RGBA plan assembly for HTJ2K and classic codestreams. + +use super::{ + append_classic_payload_records, append_decode_elements, + color::{build_direct_color_tile_components_plan, DirectColorComponentPlans}, + color_plan_rects, output_region_rect, payload_record_span, referenced_output_region, tile, + tile_intersects_output, validate_and_strip_classic_payload_owners, + validate_and_strip_referenced_payload_owners, validate_color_tile, BitReader, + ClassicPayloadCollector, DecoderContext, DirectPlanUnsupportedReason, Header, + HtCodeBlockPayloadRanges, J2kClassicCodeBlockPayload, J2kCodestreamRange, J2kDirectBandId, + J2kRect, J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, J2kReferencedTileGeometry, + J2kReferencedTilePlan, Result, ValidationError, Vec, +}; + +pub(crate) fn build_referenced_htj2k_color_plan<'a>( + data: &'a [u8], + payload_range_owner: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, +) -> Result { + build_referenced_htj2k_color_components_plan::<3>( + data, + payload_range_owner, + header, + retained_image_bytes, + ctx, + DirectPlanUnsupportedReason::ColorThreeComponentRgbCodestream, + |plans| J2kReferencedTileGeometry::Color(plans.into_rgb()), + |tiles, full_dimensions, output_rect, payloads| { + J2kReferencedHtj2kPlan::color(tiles, full_dimensions, output_rect, payloads) + }, + ) +} + +pub(crate) fn build_referenced_htj2k_rgba_plan<'a>( + data: &'a [u8], + payload_range_owner: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, +) -> Result { + build_referenced_htj2k_color_components_plan::<4>( + data, + payload_range_owner, + header, + retained_image_bytes, + ctx, + DirectPlanUnsupportedReason::RgbaFourComponentRgbCodestream, + |plans| J2kReferencedTileGeometry::Rgba(plans.into_rgba()), + |tiles, full_dimensions, output_rect, payloads| { + J2kReferencedHtj2kPlan::rgba(tiles, full_dimensions, output_rect, payloads) + }, + ) +} + +#[expect( + clippy::too_many_arguments, + reason = "referenced color planning keeps the encoded owner, retained baseline, component contract, and typed plan constructor explicit" +)] +fn build_referenced_htj2k_color_components_plan<'a, const COMPONENT_COUNT: usize>( + data: &'a [u8], + payload_range_owner: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, + component_count_error: DirectPlanUnsupportedReason, + into_geometry: impl Fn(DirectColorComponentPlans) -> J2kReferencedTileGeometry, + into_plan: impl FnOnce( + Vec, + (u32, u32), + J2kRect, + Vec, + ) -> J2kReferencedHtj2kPlan, +) -> Result { + ctx.release_reusable_allocations(); + let result = (|| { + let mut reader = BitReader::new(data); + let parsed_tiles = tile::parse(&mut reader, header, retained_image_bytes)?; + let output_region = referenced_output_region(header, ctx); + let output_rect = output_region_rect(output_region); + let mut payloads = Vec::new(); + let mut tile_plans = Vec::new(); + crate::try_reserve_decode_elements(&mut tile_plans, parsed_tiles.len())?; + let mut next_band_id: J2kDirectBandId = 0; + + for tile in parsed_tiles.iter() { + validate_color_tile::(tile, component_count_error)?; + if !tile_intersects_output(tile, header, output_region)? { + continue; + } + let first_record = payloads.len(); + let mut tile_payloads = Vec::new(); + let mut plans = build_direct_color_tile_components_plan::( + data, + payload_range_owner, + tile, + header, + parsed_tiles.structural_workspace_bytes(), + ctx, + component_count_error, + &mut next_band_id, + Some(output_region), + Some(output_region), + Some(&mut tile_payloads), + None, + )?; + validate_and_strip_referenced_payload_owners( + &mut plans.component_plans, + tile_payloads.len(), + )?; + let record_count = tile_payloads.len(); + append_decode_elements(&mut payloads, &mut tile_payloads)?; + let payload_records = payload_record_span(first_record, record_count)?; + let (decoded_rect, destination_rect) = + color_plan_rects::(&plans.component_plans, output_rect)?; + let geometry = into_geometry(plans); + tile_plans.push(J2kReferencedTilePlan::new( + usize::try_from(tile.idx).map_err(|_| ValidationError::ImageTooLarge)?, + decoded_rect, + destination_rect, + payload_records, + super::J2kWaveletTransform::from(tile.component_infos[0].wavelet_transform()), + geometry, + )); + } + + let full_dimensions = ( + header.size_data.image_width(), + header.size_data.image_height(), + ); + Ok(into_plan( + tile_plans, + full_dimensions, + output_rect, + payloads, + )) + })(); + ctx.release_reusable_allocations(); + result +} + +pub(crate) fn build_referenced_classic_color_plan<'a>( + data: &'a [u8], + payload_range_owner: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, +) -> Result { + build_referenced_classic_color_components_plan::<3>( + data, + payload_range_owner, + header, + retained_image_bytes, + ctx, + DirectPlanUnsupportedReason::ColorThreeComponentRgbCodestream, + |plans| J2kReferencedTileGeometry::Color(plans.into_rgb()), + |tiles, full_dimensions, output_rect, payloads, ranges| { + J2kReferencedClassicPlan::color(tiles, full_dimensions, output_rect, payloads, ranges) + }, + ) +} + +pub(crate) fn build_referenced_classic_rgba_plan<'a>( + data: &'a [u8], + payload_range_owner: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, +) -> Result { + build_referenced_classic_color_components_plan::<4>( + data, + payload_range_owner, + header, + retained_image_bytes, + ctx, + DirectPlanUnsupportedReason::RgbaFourComponentRgbCodestream, + |plans| J2kReferencedTileGeometry::Rgba(plans.into_rgba()), + |tiles, full_dimensions, output_rect, payloads, ranges| { + J2kReferencedClassicPlan::rgba(tiles, full_dimensions, output_rect, payloads, ranges) + }, + ) +} + +#[expect( + clippy::too_many_arguments, + reason = "referenced classic color planning keeps retained-byte ownership and typed geometry construction explicit" +)] +fn build_referenced_classic_color_components_plan<'a, const COMPONENT_COUNT: usize>( + data: &'a [u8], + payload_range_owner: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, + component_count_error: DirectPlanUnsupportedReason, + into_geometry: impl Fn(DirectColorComponentPlans) -> J2kReferencedTileGeometry, + into_plan: impl FnOnce( + Vec, + (u32, u32), + J2kRect, + Vec, + Vec, + ) -> J2kReferencedClassicPlan, +) -> Result { + ctx.release_reusable_allocations(); + let result = (|| { + let mut reader = BitReader::new(data); + let parsed_tiles = tile::parse(&mut reader, header, retained_image_bytes)?; + let output_region = referenced_output_region(header, ctx); + let output_rect = output_region_rect(output_region); + let mut payloads = Vec::new(); + let mut ranges = Vec::new(); + let mut tile_plans = Vec::new(); + crate::try_reserve_decode_elements(&mut tile_plans, parsed_tiles.len())?; + let mut next_band_id: J2kDirectBandId = 0; + + for tile in parsed_tiles.iter() { + validate_color_tile::(tile, component_count_error)?; + if !tile_intersects_output(tile, header, output_region)? { + continue; + } + let first_record = payloads.len(); + let mut tile_payloads = Vec::new(); + let mut tile_ranges = Vec::new(); + let mut collector = ClassicPayloadCollector { + payloads: &mut tile_payloads, + ranges: &mut tile_ranges, + }; + let mut plans = build_direct_color_tile_components_plan::( + data, + payload_range_owner, + tile, + header, + parsed_tiles.structural_workspace_bytes(), + ctx, + component_count_error, + &mut next_band_id, + Some(output_region), + Some(output_region), + None, + Some(&mut collector), + )?; + validate_and_strip_classic_payload_owners( + &mut plans.component_plans, + tile_payloads.len(), + )?; + let record_count = tile_payloads.len(); + append_classic_payload_records( + &mut payloads, + &mut ranges, + &mut tile_payloads, + &mut tile_ranges, + )?; + let payload_records = payload_record_span(first_record, record_count)?; + let (decoded_rect, destination_rect) = + color_plan_rects::(&plans.component_plans, output_rect)?; + tile_plans.push(J2kReferencedTilePlan::new( + usize::try_from(tile.idx).map_err(|_| ValidationError::ImageTooLarge)?, + decoded_rect, + destination_rect, + payload_records, + super::J2kWaveletTransform::from(tile.component_infos[0].wavelet_transform()), + into_geometry(plans), + )); + } + + Ok(into_plan( + tile_plans, + ( + header.size_data.image_width(), + header.size_data.image_height(), + ), + output_rect, + payloads, + ranges, + )) + })(); + ctx.release_reusable_allocations(); + result +} diff --git a/crates/j2k-native/src/j2c/decode/direct_plan/referenced_grayscale.rs b/crates/j2k-native/src/j2c/decode/direct_plan/referenced_grayscale.rs new file mode 100644 index 00000000..3dee89c5 --- /dev/null +++ b/crates/j2k-native/src/j2c/decode/direct_plan/referenced_grayscale.rs @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Retained HTJ2K and classic grayscale plan construction. + +use super::{ + append_classic_payload_records, append_decode_elements, bail, build_direct_grayscale_tile_plan, + grayscale_plan_rects, output_region_rect, payload_record_span, referenced_output_region, + strip_classic_payload_owners, strip_grayscale_payload_owners, tile, tile_intersects_output, + validate_grayscale_tile, BitReader, ClassicPayloadCollector, DecoderContext, DecodingError, + Header, J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, J2kReferencedTileGeometry, + J2kReferencedTilePlan, Result, ValidationError, Vec, +}; + +pub(crate) fn build_referenced_htj2k_grayscale_plan<'a>( + data: &'a [u8], + payload_range_owner: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, +) -> Result { + ctx.release_reusable_allocations(); + let result = (|| { + let mut reader = BitReader::new(data); + let parsed_tiles = tile::parse(&mut reader, header, retained_image_bytes)?; + let output_region = referenced_output_region(header, ctx); + let output_rect = output_region_rect(output_region); + let mut payloads = Vec::new(); + let mut tile_plans = Vec::new(); + crate::try_reserve_decode_elements(&mut tile_plans, parsed_tiles.len())?; + let mut next_band_id = 0; + + for tile in parsed_tiles.iter() { + validate_grayscale_tile(tile)?; + if !tile_intersects_output(tile, header, output_region)? { + continue; + } + let first_record = payloads.len(); + let mut tile_payloads = Vec::new(); + let mut geometry = build_direct_grayscale_tile_plan( + data, + payload_range_owner, + tile, + header, + parsed_tiles.structural_workspace_bytes(), + ctx, + &mut next_band_id, + Some(output_region), + Some(output_region), + Some(&mut tile_payloads), + None, + )?; + let job_count = strip_grayscale_payload_owners(&mut geometry)?; + if job_count != tile_payloads.len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + append_decode_elements(&mut payloads, &mut tile_payloads)?; + let payload_records = payload_record_span(first_record, job_count)?; + let (decoded_rect, destination_rect) = grayscale_plan_rects(&geometry, output_rect)?; + tile_plans.push(J2kReferencedTilePlan::new( + usize::try_from(tile.idx).map_err(|_| ValidationError::ImageTooLarge)?, + decoded_rect, + destination_rect, + payload_records, + super::J2kWaveletTransform::from(tile.component_infos[0].wavelet_transform()), + J2kReferencedTileGeometry::Grayscale(geometry), + )); + } + + Ok(J2kReferencedHtj2kPlan::grayscale( + tile_plans, + ( + header.size_data.image_width(), + header.size_data.image_height(), + ), + output_rect, + payloads, + )) + })(); + ctx.release_reusable_allocations(); + result +} + +pub(crate) fn build_referenced_classic_grayscale_plan<'a>( + data: &'a [u8], + payload_range_owner: &'a [u8], + header: &Header<'a>, + retained_image_bytes: usize, + ctx: &mut DecoderContext<'a>, +) -> Result { + ctx.release_reusable_allocations(); + let result = (|| { + let mut reader = BitReader::new(data); + let parsed_tiles = tile::parse(&mut reader, header, retained_image_bytes)?; + let output_region = referenced_output_region(header, ctx); + let output_rect = output_region_rect(output_region); + let mut payloads = Vec::new(); + let mut ranges = Vec::new(); + let mut tile_plans = Vec::new(); + crate::try_reserve_decode_elements(&mut tile_plans, parsed_tiles.len())?; + let mut next_band_id = 0; + + for tile in parsed_tiles.iter() { + validate_grayscale_tile(tile)?; + if !tile_intersects_output(tile, header, output_region)? { + continue; + } + let first_record = payloads.len(); + let mut tile_payloads = Vec::new(); + let mut tile_ranges = Vec::new(); + let mut collector = ClassicPayloadCollector { + payloads: &mut tile_payloads, + ranges: &mut tile_ranges, + }; + let mut geometry = build_direct_grayscale_tile_plan( + data, + payload_range_owner, + tile, + header, + parsed_tiles.structural_workspace_bytes(), + ctx, + &mut next_band_id, + Some(output_region), + Some(output_region), + None, + Some(&mut collector), + )?; + let job_count = strip_classic_payload_owners(&mut geometry)?; + if job_count != tile_payloads.len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + append_classic_payload_records( + &mut payloads, + &mut ranges, + &mut tile_payloads, + &mut tile_ranges, + )?; + let payload_records = payload_record_span(first_record, job_count)?; + let (decoded_rect, destination_rect) = grayscale_plan_rects(&geometry, output_rect)?; + tile_plans.push(J2kReferencedTilePlan::new( + usize::try_from(tile.idx).map_err(|_| ValidationError::ImageTooLarge)?, + decoded_rect, + destination_rect, + payload_records, + super::J2kWaveletTransform::from(tile.component_infos[0].wavelet_transform()), + J2kReferencedTileGeometry::Grayscale(geometry), + )); + } + + Ok(J2kReferencedClassicPlan::grayscale( + tile_plans, + ( + header.size_data.image_width(), + header.size_data.image_height(), + ), + output_rect, + payloads, + ranges, + )) + })(); + ctx.release_reusable_allocations(); + result +} diff --git a/crates/j2k-native/src/j2c/decode/direct_plan/storage.rs b/crates/j2k-native/src/j2c/decode/direct_plan/storage.rs new file mode 100644 index 00000000..ecadf72e --- /dev/null +++ b/crates/j2k-native/src/j2c/decode/direct_plan/storage.rs @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Component-plan assembly from parsed decomposition storage. + +use super::{ + bail, ClassicPayloadCollector, ComponentInfo, ComponentTile, DecodeAllocationBudget, + DecodingError, DecompositionStorage, DirectPlanUnsupportedReason, Header, + HtCodeBlockPayloadRanges, IntRect, J2kDirectBandId, J2kDirectGrayscalePlan, + J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, J2kRect, J2kWaveletTransform, + PayloadRangeOwner, ResolutionTile, Result, Tile, ValidationError, Vec, +}; +use crate::j2c::decode::TileDecompositions; + +pub(super) mod sub_band; +use self::sub_band::build_grayscale_sub_band_step; + +#[expect( + clippy::too_many_arguments, + reason = "component planning keeps its validated storage, shared budget, output region, and optional payload collectors explicit" +)] +pub(super) fn build_component_plan_from_storage( + payload_range_owner: PayloadRangeOwner<'_>, + tile: &Tile<'_>, + header: &Header<'_>, + storage: &DecompositionStorage<'_>, + component_idx: usize, + store_addend: f32, + budget: &mut DecodeAllocationBudget, + next_band_id: &mut J2kDirectBandId, + output_region: Option, + ht_payloads: Option<&mut Vec>, + classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, +) -> Result { + let component_info = component_info(tile, component_idx)?; + let tile_decompositions = component_decompositions(storage, component_idx)?; + let (step_capacity, active_decomposition_count) = component_step_capacity( + component_info, + tile_decompositions, + storage, + header.skipped_resolution_levels, + )?; + let mut steps = Vec::new(); + budget.reserve_new(&mut steps, step_capacity)?; + let mut sub_band_ids = Vec::new(); + budget.resize_new(&mut sub_band_ids, storage.sub_bands.len(), None)?; + + append_sub_band_steps( + payload_range_owner, + component_info, + tile_decompositions, + storage, + header, + budget, + next_band_id, + &mut steps, + &mut sub_band_ids, + ht_payloads, + classic_payloads, + )?; + let (final_rect, final_band_id) = append_idwt_steps( + component_info, + tile_decompositions, + active_decomposition_count, + storage, + next_band_id, + &sub_band_ids, + &mut steps, + )?; + let store = component_store_step( + tile, + component_info, + final_rect, + final_band_id, + header, + output_region, + store_addend, + ); + let dimensions = (store.output_width, store.output_height); + steps.push(J2kDirectGrayscaleStep::Store(store)); + + let sub_band_id_capacity = sub_band_ids.capacity(); + drop(sub_band_ids); + budget.release_elements::>(sub_band_id_capacity)?; + + Ok(J2kDirectGrayscalePlan { + dimensions, + bit_depth: component_info.size_info.precision, + steps, + }) +} + +fn component_info<'a>(tile: &'a Tile<'_>, component_idx: usize) -> Result<&'a ComponentInfo> { + let component_info = + tile.component_infos + .get(component_idx) + .ok_or(DecodingError::DirectPlanUnsupported( + DirectPlanUnsupportedReason::ComponentIndexOutOfRange, + ))?; + if component_info.size_info.horizontal_resolution != 1 + || component_info.size_info.vertical_resolution != 1 + { + bail!(DecodingError::DirectPlanUnsupported( + DirectPlanUnsupportedReason::ComponentUnitSampled + )); + } + Ok(component_info) +} + +fn component_decompositions<'a>( + storage: &'a DecompositionStorage<'_>, + component_idx: usize, +) -> Result<&'a TileDecompositions> { + storage + .tile_decompositions + .get(component_idx) + .ok_or(DecodingError::DirectPlanUnsupported( + DirectPlanUnsupportedReason::ComponentDecompositionIndexOutOfRange, + )) + .map_err(Into::into) +} + +fn component_step_capacity( + component_info: &ComponentInfo, + tile_decompositions: &TileDecompositions, + storage: &DecompositionStorage<'_>, + skipped_resolution_levels: u8, +) -> Result<(usize, usize)> { + let active_decomposition_count = tile_decompositions + .decompositions + .len() + .saturating_sub(skipped_resolution_levels as usize); + let sub_band_step_count = (0..component_info.num_resolution_levels() + - skipped_resolution_levels) + .try_fold(0_usize, |total, resolution| { + tile_decompositions + .sub_band_iter(resolution, &storage.decompositions) + .count() + .checked_add(total) + .ok_or(ValidationError::ImageTooLarge) + })?; + let step_capacity = sub_band_step_count + .checked_add(active_decomposition_count) + .and_then(|count| count.checked_add(1)) + .ok_or(ValidationError::ImageTooLarge)?; + Ok((step_capacity, active_decomposition_count)) +} + +#[expect( + clippy::too_many_arguments, + reason = "sub-band assembly advances the shared band namespace and both optional payload collectors" +)] +fn append_sub_band_steps( + payload_range_owner: PayloadRangeOwner<'_>, + component_info: &ComponentInfo, + tile_decompositions: &TileDecompositions, + storage: &DecompositionStorage<'_>, + header: &Header<'_>, + budget: &mut DecodeAllocationBudget, + next_band_id: &mut J2kDirectBandId, + steps: &mut Vec, + sub_band_ids: &mut [Option], + mut ht_payloads: Option<&mut Vec>, + mut classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, +) -> Result<()> { + for resolution in 0..component_info.num_resolution_levels() - header.skipped_resolution_levels { + for sub_band_idx in tile_decompositions.sub_band_iter(resolution, &storage.decompositions) { + let Some(step) = build_grayscale_sub_band_step( + payload_range_owner, + &storage.sub_bands[sub_band_idx], + sub_band_idx, + *next_band_id, + resolution, + component_info, + storage, + header, + budget, + ht_payloads.as_deref_mut(), + classic_payloads.as_deref_mut(), + )? + else { + continue; + }; + sub_band_ids[sub_band_idx] = Some(*next_band_id); + *next_band_id = next_band_id + .checked_add(1) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + steps.push(step); + } + } + Ok(()) +} + +fn append_idwt_steps( + component_info: &ComponentInfo, + tile_decompositions: &TileDecompositions, + active_decomposition_count: usize, + storage: &DecompositionStorage<'_>, + next_band_id: &mut J2kDirectBandId, + sub_band_ids: &[Option], + steps: &mut Vec, +) -> Result<(IntRect, J2kDirectBandId)> { + let mut ll_rect = storage.sub_bands[tile_decompositions.first_ll_sub_band].rect; + let mut ll_band_id = sub_band_ids[tile_decompositions.first_ll_sub_band] + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let decompositions = &storage.decompositions[tile_decompositions.decompositions.clone()]; + + for decomposition in &decompositions[..active_decomposition_count] { + let [horizontal_band, vertical_band, diagonal_band] = decomposition.sub_bands; + let output_band_id = *next_band_id; + *next_band_id = next_band_id + .checked_add(1) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + steps.push(J2kDirectGrayscaleStep::Idwt(J2kDirectIdwtStep { + output_band_id, + rect: J2kRect::from(decomposition.rect), + transform: J2kWaveletTransform::from(component_info.wavelet_transform()), + ll_band_id, + ll: J2kRect::from(ll_rect), + hl_band_id: sub_band_ids[horizontal_band] + .ok_or(DecodingError::CodeBlockDecodeFailure)?, + hl: J2kRect::from(storage.sub_bands[horizontal_band].rect), + lh_band_id: sub_band_ids[vertical_band].ok_or(DecodingError::CodeBlockDecodeFailure)?, + lh: J2kRect::from(storage.sub_bands[vertical_band].rect), + hh_band_id: sub_band_ids[diagonal_band].ok_or(DecodingError::CodeBlockDecodeFailure)?, + hh: J2kRect::from(storage.sub_bands[diagonal_band].rect), + })); + ll_rect = decomposition.rect; + ll_band_id = output_band_id; + } + Ok((ll_rect, ll_band_id)) +} + +fn component_store_step( + tile: &Tile<'_>, + component_info: &ComponentInfo, + input_rect: IntRect, + input_band_id: J2kDirectBandId, + header: &Header<'_>, + output_region: Option, + addend: f32, +) -> J2kDirectStoreStep { + let component_tile = ComponentTile::new(tile, component_info); + let resolution_tile = ResolutionTile::new( + component_tile, + component_info.num_resolution_levels() - 1 - header.skipped_resolution_levels, + ); + let ( + source_x, + source_y, + copy_width, + copy_height, + output_width, + output_height, + output_x, + output_y, + ) = direct_store_geometry(input_rect, resolution_tile.rect, header, output_region); + J2kDirectStoreStep { + input_band_id, + input_rect: J2kRect::from(input_rect), + source_x, + source_y, + copy_width, + copy_height, + output_width, + output_height, + output_x, + output_y, + addend, + } +} + +#[expect( + clippy::similar_names, + reason = "paired source, destination, image, and region coordinates mirror the JPEG 2000 store boundary" +)] +fn direct_store_geometry( + input_rect: IntRect, + resolution_rect: IntRect, + header: &Header<'_>, + output_region: Option, +) -> (u32, u32, u32, u32, u32, u32, u32, u32) { + let output_region = output_region.unwrap_or(super::super::OutputRegion { + x: 0, + y: 0, + width: header.size_data.image_width(), + height: header.size_data.image_height(), + }); + let x_offset = header + .size_data + .image_area_x_offset + .div_ceil(header.size_data.x_shrink_factor); + let y_offset = header + .size_data + .image_area_y_offset + .div_ceil(header.size_data.y_shrink_factor); + let region_x0 = output_region.x.saturating_add(x_offset); + let region_y0 = output_region.y.saturating_add(y_offset); + let region_x1 = region_x0.saturating_add(output_region.width); + let region_y1 = region_y0.saturating_add(output_region.height); + let copy_x0 = input_rect.x0.max(resolution_rect.x0).max(region_x0); + let copy_y0 = input_rect.y0.max(resolution_rect.y0).max(region_y0); + let copy_x1 = input_rect.x1.min(resolution_rect.x1).min(region_x1); + let copy_y1 = input_rect.y1.min(resolution_rect.y1).min(region_y1); + + if copy_x0 >= copy_x1 || copy_y0 >= copy_y1 { + return (0, 0, 0, 0, output_region.width, output_region.height, 0, 0); + } + + ( + copy_x0 - input_rect.x0, + copy_y0 - input_rect.y0, + copy_x1 - copy_x0, + copy_y1 - copy_y0, + output_region.width, + output_region.height, + copy_x0 - region_x0, + copy_y0 - region_y0, + ) +} diff --git a/crates/j2k-native/src/j2c/decode/direct_plan/storage/sub_band.rs b/crates/j2k-native/src/j2c/decode/direct_plan/storage/sub_band.rs new file mode 100644 index 00000000..9e2bc193 --- /dev/null +++ b/crates/j2k-native/src/j2c/decode/direct_plan/storage/sub_band.rs @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! HT and classic code-block payload construction for one direct-plan sub-band. + +use super::super::{ + add_roi_shift_to_bitplanes, bail, classic_decode_job_parameters, code_block_required_by_index, + collect_classic_code_block_data, collect_referenced_classic_code_block_data, ht_block_decode, + ht_code_block_has_decodable_passes, sub_band_decode_parameters, ClassicPayloadCollector, + ComponentInfo, DecodeAllocationBudget, DecodingError, DecompositionStorage, Header, + HtCodeBlockPayloadRanges, HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, + J2kClassicCodeBlockPayload, J2kCodestreamRange, J2kDirectBandId, J2kDirectGrayscalePlan, + J2kDirectGrayscaleStep, J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, J2kRect, + PayloadRangeOwner, Result, SubBand, SubBandDecodeParameters, ValidationError, Vec, +}; +use crate::j2c::build::CodeBlock; +use crate::{J2kCodeBlockSegment, J2kCodeBlockStyle, J2kSubBandType}; + +#[expect( + clippy::too_many_arguments, + reason = "the sub-band boundary keeps validated band identity, storage, and payload collectors explicit" +)] +pub(super) fn build_grayscale_sub_band_step( + payload_range_owner: PayloadRangeOwner<'_>, + sub_band: &SubBand, + sub_band_idx: usize, + band_id: J2kDirectBandId, + resolution: u8, + component_info: &ComponentInfo, + storage: &DecompositionStorage<'_>, + header: &Header<'_>, + budget: &mut DecodeAllocationBudget, + ht_payloads: Option<&mut Vec>, + classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, +) -> Result> { + let SubBandDecodeParameters { + dequantization_step, + num_bitplanes, + } = sub_band_decode_parameters(sub_band, resolution, component_info)?; + + if component_info + .coding_style + .parameters + .code_block_style + .uses_high_throughput_block_coding() + { + return build_ht_sub_band_step( + payload_range_owner, + sub_band, + sub_band_idx, + band_id, + component_info, + storage, + header, + budget, + ht_payloads, + dequantization_step, + num_bitplanes, + ) + .map(Some); + } + + build_classic_sub_band_step( + payload_range_owner, + sub_band, + sub_band_idx, + band_id, + component_info, + storage, + header, + budget, + classic_payloads, + dequantization_step, + num_bitplanes, + ) + .map(Some) +} + +#[expect( + clippy::too_many_arguments, + reason = "HT job construction needs the validated band geometry, decode parameters, range owner, and shared budget" +)] +fn build_ht_sub_band_step( + payload_range_owner: PayloadRangeOwner<'_>, + sub_band: &SubBand, + sub_band_idx: usize, + band_id: J2kDirectBandId, + component_info: &ComponentInfo, + storage: &DecompositionStorage<'_>, + header: &Header<'_>, + budget: &mut DecodeAllocationBudget, + mut ht_payloads: Option<&mut Vec>, + dequantization_step: f32, + num_bitplanes: u8, +) -> Result { + let coded_bitplanes = add_roi_shift_to_bitplanes(num_bitplanes, component_info.roi_shift, 31)?; + let stripe_causal = component_info + .coding_style + .parameters + .code_block_style + .vertically_causal_context; + let job_capacity = direct_sub_band_job_capacity(sub_band, storage)?; + let mut jobs = Vec::new(); + budget.reserve_new(&mut jobs, job_capacity)?; + + for precinct in sub_band + .precincts + .clone() + .map(|idx| &storage.precincts[idx]) + { + for code_block in precinct + .code_blocks + .clone() + .map(|idx| &storage.code_blocks[idx]) + { + if !code_block_required_by_index(storage, sub_band_idx, code_block) + || !ht_code_block_has_decodable_passes(code_block, coded_bitplanes, header.strict)? + { + continue; + } + + if let Some(payloads) = ht_payloads.as_deref_mut() { + let segments = ht_block_decode::collect_code_block_segments(code_block, storage)?; + payloads.push(HtCodeBlockPayloadRanges { + cleanup: encoded_input_range(payload_range_owner, segments.cleanup)?, + refinement: (!segments.refinement.is_empty()) + .then(|| encoded_input_range(payload_range_owner, segments.refinement)) + .transpose()?, + }); + } + + let combined = ht_block_decode::collect_code_block_data(code_block, storage, budget)?; + jobs.push(HtOwnedCodeBlockBatchJob { + output_x: code_block.rect.x0 - sub_band.rect.x0, + output_y: code_block.rect.y0 - sub_band.rect.y0, + data: combined.data, + cleanup_length: combined.cleanup_length, + refinement_length: combined.refinement_length, + width: code_block.rect.width(), + height: code_block.rect.height(), + output_stride: sub_band.rect.width() as usize, + missing_bit_planes: code_block.missing_bit_planes, + number_of_coding_passes: code_block.number_of_coding_passes, + num_bitplanes, + roi_shift: component_info.roi_shift, + stripe_causal, + strict: header.strict, + dequantization_step, + }); + } + } + + Ok(J2kDirectGrayscaleStep::HtSubBand(HtOwnedSubBandPlan { + band_id, + rect: J2kRect::from(sub_band.rect), + width: sub_band.rect.width(), + height: sub_band.rect.height(), + jobs, + })) +} + +#[expect( + clippy::too_many_arguments, + reason = "classic job construction needs the validated band geometry, decode parameters, range owner, and shared budget" +)] +fn build_classic_sub_band_step( + payload_range_owner: PayloadRangeOwner<'_>, + sub_band: &SubBand, + sub_band_idx: usize, + band_id: J2kDirectBandId, + component_info: &ComponentInfo, + storage: &DecompositionStorage<'_>, + header: &Header<'_>, + budget: &mut DecodeAllocationBudget, + mut classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, + dequantization_step: f32, + num_bitplanes: u8, +) -> Result { + let (sub_band_type, style) = + classic_decode_job_parameters(sub_band.sub_band_type, component_info); + let job_capacity = direct_sub_band_job_capacity(sub_band, storage)?; + let mut jobs = Vec::new(); + budget.reserve_new(&mut jobs, job_capacity)?; + + for precinct in sub_band + .precincts + .clone() + .map(|idx| &storage.precincts[idx]) + { + for code_block in precinct + .code_blocks + .clone() + .map(|idx| &storage.code_blocks[idx]) + { + if !code_block_required_by_index(storage, sub_band_idx, code_block) { + continue; + } + jobs.push(build_classic_code_block_job( + payload_range_owner, + code_block, + sub_band, + component_info, + storage, + header, + budget, + classic_payloads.as_deref_mut(), + sub_band_type, + style, + dequantization_step, + num_bitplanes, + )?); + } + } + + Ok(J2kDirectGrayscaleStep::ClassicSubBand( + J2kOwnedSubBandPlan { + band_id, + rect: J2kRect::from(sub_band.rect), + width: sub_band.rect.width(), + height: sub_band.rect.height(), + jobs, + }, + )) +} + +#[expect( + clippy::too_many_arguments, + reason = "the classic job record combines codec metadata with its selected owned or referenced payload representation" +)] +fn build_classic_code_block_job( + payload_range_owner: PayloadRangeOwner<'_>, + code_block: &CodeBlock, + sub_band: &SubBand, + component_info: &ComponentInfo, + storage: &DecompositionStorage<'_>, + header: &Header<'_>, + budget: &mut DecodeAllocationBudget, + classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, + sub_band_type: J2kSubBandType, + style: J2kCodeBlockStyle, + dequantization_step: f32, + num_bitplanes: u8, +) -> Result { + let (data, segments) = collect_classic_payload( + payload_range_owner, + code_block, + component_info, + storage, + budget, + classic_payloads, + )?; + Ok(J2kOwnedCodeBlockBatchJob { + output_x: code_block.rect.x0 - sub_band.rect.x0, + output_y: code_block.rect.y0 - sub_band.rect.y0, + data, + segments, + width: code_block.rect.width(), + height: code_block.rect.height(), + output_stride: sub_band.rect.width() as usize, + missing_bit_planes: code_block.missing_bit_planes, + number_of_coding_passes: code_block.number_of_coding_passes, + total_bitplanes: num_bitplanes, + roi_shift: component_info.roi_shift, + sub_band_type, + style, + strict: header.strict, + dequantization_step, + }) +} + +fn collect_classic_payload( + payload_range_owner: PayloadRangeOwner<'_>, + code_block: &CodeBlock, + component_info: &ComponentInfo, + storage: &DecompositionStorage<'_>, + budget: &mut DecodeAllocationBudget, + classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, +) -> Result<(Vec, Vec)> { + let Some(collector) = classic_payloads else { + return collect_classic_code_block_data( + code_block, + &component_info.coding_style.parameters.code_block_style, + storage, + budget, + ); + }; + + let first_range = collector.ranges.len(); + let (range_count, combined_length, segments) = collect_referenced_classic_code_block_data( + code_block, + &component_info.coding_style.parameters.code_block_style, + storage, + budget, + |fragment| collector.push_range(encoded_input_range(payload_range_owner, fragment)?), + )?; + collector.push_payload(J2kClassicCodeBlockPayload { + first_range, + range_count, + combined_length, + })?; + Ok((Vec::new(), segments)) +} + +pub(in crate::j2c::decode::direct_plan) fn strip_grayscale_payload_owners( + plan: &mut J2kDirectGrayscalePlan, +) -> Result { + let mut job_count = 0_usize; + for step in &mut plan.steps { + match step { + J2kDirectGrayscaleStep::HtSubBand(sub_band) => { + job_count = job_count + .checked_add(sub_band.jobs.len()) + .ok_or(ValidationError::ImageTooLarge)?; + for job in &mut sub_band.jobs { + job.data = Vec::new(); + } + } + J2kDirectGrayscaleStep::ClassicSubBand(_) => { + bail!(DecodingError::UnsupportedFeature( + "referenced HTJ2K plan encountered classic code blocks" + )); + } + J2kDirectGrayscaleStep::Idwt(_) | J2kDirectGrayscaleStep::Store(_) => {} + } + } + Ok(job_count) +} + +pub(in crate::j2c::decode::direct_plan) fn strip_classic_payload_owners( + plan: &mut J2kDirectGrayscalePlan, +) -> Result { + let mut job_count = 0usize; + for step in &mut plan.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { + job_count = job_count + .checked_add(sub_band.jobs.len()) + .ok_or(ValidationError::ImageTooLarge)?; + for job in &mut sub_band.jobs { + job.data = Vec::new(); + } + } + J2kDirectGrayscaleStep::HtSubBand(_) => { + bail!(DecodingError::UnsupportedFeature( + "referenced classic plan encountered HT code blocks" + )); + } + J2kDirectGrayscaleStep::Idwt(_) | J2kDirectGrayscaleStep::Store(_) => {} + } + } + Ok(job_count) +} + +fn encoded_input_range(owner: PayloadRangeOwner<'_>, payload: &[u8]) -> Result { + let encoded_input_start = owner.encoded_input.as_ptr() as usize; + let codestream_start = owner.codestream.as_ptr() as usize; + let codestream_offset = codestream_start + .checked_sub(encoded_input_start) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let codestream_end = codestream_offset + .checked_add(owner.codestream.len()) + .ok_or(ValidationError::ImageTooLarge)?; + if codestream_end > owner.encoded_input.len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let payload_start = payload.as_ptr() as usize; + let payload_offset = payload_start + .checked_sub(codestream_start) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let payload_end = payload_offset + .checked_add(payload.len()) + .ok_or(ValidationError::ImageTooLarge)?; + if payload_end > owner.codestream.len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let offset = codestream_offset + .checked_add(payload_offset) + .ok_or(ValidationError::ImageTooLarge)?; + let end = offset + .checked_add(payload.len()) + .ok_or(ValidationError::ImageTooLarge)?; + if end > owner.encoded_input.len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(J2kCodestreamRange { + offset, + length: payload.len(), + }) +} + +fn direct_sub_band_job_capacity( + sub_band: &SubBand, + storage: &DecompositionStorage<'_>, +) -> Result { + sub_band + .precincts + .clone() + .map(|idx| storage.precincts[idx].code_blocks.len()) + .try_fold(0_usize, |total, count| { + total + .checked_add(count) + .ok_or(ValidationError::ImageTooLarge.into()) + }) +} diff --git a/crates/j2k-native/src/j2c/decode/reuse.rs b/crates/j2k-native/src/j2c/decode/reuse.rs index 4a0ac9e8..1b56ad2d 100644 --- a/crates/j2k-native/src/j2c/decode/reuse.rs +++ b/crates/j2k-native/src/j2c/decode/reuse.rs @@ -17,8 +17,9 @@ const CONTEXT_ALLOCATION_WHAT: &str = "native decoder context retained component #[derive(Clone, Copy)] pub(super) struct ReusedDecodeBaseline { - pub(super) parser_bytes: usize, - pub(super) retained_channel_bytes: usize, + pub(super) parser_live: usize, + pub(super) channel_capacity: usize, + pub(super) scratch_capacity: usize, } impl DecoderContext<'_> { @@ -26,10 +27,17 @@ impl DecoderContext<'_> { &mut self, retained_image_bytes: usize, ) -> Result { - // Completed tile graphs and transient scratch are never reused across - // images. Component sample owners are: they dominate repeated packed - // decode cost and their exact capacities can be carried explicitly. - self.tile_decode_context.release_tile_scratch_allocations(); + self.prepare_reused_decode_baseline_with_cap(retained_image_bytes, DEFAULT_MAX_DECODE_BYTES) + } + + fn prepare_reused_decode_baseline_with_cap( + &mut self, + retained_image_bytes: usize, + cap: usize, + ) -> Result { + // Parsed graphs borrow the previous input and are never reused. + // Component, Tier-1, and IDWT owners are lifetime-free and their exact + // capacities can be carried explicitly across unrelated inputs. self.storage.release_all_allocations(); let retained_channel_bytes = match self.tile_decode_context.retained_channel_bytes() { @@ -40,32 +48,52 @@ impl DecoderContext<'_> { } Err(error) => return Err(error), }; - match checked_combined_context_bytes( - retained_image_bytes, - retained_channel_bytes, - DEFAULT_MAX_DECODE_BYTES, - ) { - Ok(parser_bytes) => Ok(ReusedDecodeBaseline { - parser_bytes, - retained_channel_bytes, + let retained_scratch_bytes = self.tile_decode_context.retained_scratch_bytes()?; + let retained_workspace_bytes = + checked_combined_context_bytes(retained_channel_bytes, retained_scratch_bytes, cap)?; + match checked_combined_context_bytes(retained_image_bytes, retained_workspace_bytes, cap) { + Ok(parser_live) => Ok(ReusedDecodeBaseline { + parser_live, + channel_capacity: retained_channel_bytes, + scratch_capacity: retained_scratch_bytes, }), + Err(error) if retained_scratch_bytes != 0 => Err(error), Err(_) if retained_channel_bytes != 0 => { // Reuse is optional. A stale cache must never make a decode // fail when the same request fits with a fresh context. self.tile_decode_context.release_channel_allocations(); Ok(ReusedDecodeBaseline { - parser_bytes: checked_combined_context_bytes( - retained_image_bytes, - 0, - DEFAULT_MAX_DECODE_BYTES, - )?, - retained_channel_bytes: 0, + parser_live: checked_combined_context_bytes(retained_image_bytes, 0, cap)?, + channel_capacity: 0, + scratch_capacity: 0, }) } Err(error) => Err(error), } } + pub(super) fn retry_without_retained_scratch_on_capacity( + &mut self, + retained_scratch_bytes: usize, + first_result: Result, + retry: impl FnOnce(&mut Self) -> Result, + ) -> Result { + if retained_scratch_bytes == 0 { + return first_result; + } + match first_result { + Err(error) if is_capacity_error(&error) => { + // A retained cache is never allowed to turn a request that + // fits a fresh decoder into a resource-cap failure. + self.storage.release_all_allocations(); + self.tile_decode_context.release_tile_scratch_allocations(); + self.record_scratch_capacity_retry(); + retry(self) + } + result => result, + } + } + pub(super) fn discard_reused_channels(&mut self) { self.tile_decode_context.release_channel_allocations(); } @@ -426,6 +454,45 @@ mod tests { )); } + #[test] + fn retained_scratch_exact_cap_and_scratch_free_retry_are_deterministic() { + let mut context = super::DecoderContext::default(); + context + .tile_decode_context + .idwt_scratch_buffer + .try_reserve_exact(8) + .expect("reserve deterministic retained scratch"); + let scratch_bytes = context + .tile_decode_context + .retained_scratch_bytes() + .expect("scratch capacity bytes"); + assert!(scratch_bytes > 0); + + let exact = context + .prepare_reused_decode_baseline_with_cap(0, scratch_bytes) + .expect("retained scratch fits exact cap"); + assert_eq!(exact.parser_live, scratch_bytes); + assert_eq!(exact.scratch_capacity, scratch_bytes); + let first_result = context.prepare_reused_decode_baseline_with_cap(0, scratch_bytes - 1); + assert!(matches!( + first_result, + Err(DecodeError::AllocationTooLarge { .. }) + )); + + let retried = context + .retry_without_retained_scratch_on_capacity(scratch_bytes, first_result, |context| { + context.prepare_reused_decode_baseline_with_cap(0, 0) + }) + .expect("scratch-free retry fits"); + assert_eq!(retried.parser_live, 0); + assert_eq!(retried.scratch_capacity, 0); + assert_eq!( + context.tile_decode_context.idwt_capacity_bytes().unwrap(), + 0 + ); + assert_eq!(context.workspace_stats().scratch_capacity_retries(), 1); + } + #[test] fn context_budget_release_replaces_old_owner_without_double_counting() { let mut budget = ContextCapacityBudget::with_cap(8, 8).expect("full old owner"); diff --git a/crates/j2k-native/src/j2c/decode/tier1.rs b/crates/j2k-native/src/j2c/decode/tier1.rs index 5bcd9edd..c9dc22e2 100644 --- a/crates/j2k-native/src/j2c/decode/tier1.rs +++ b/crates/j2k-native/src/j2c/decode/tier1.rs @@ -12,6 +12,10 @@ use crate::j2c::bitplane::classic_decode_workspace_bytes; use crate::j2c::ht_block_decode::ht_decode_workspace_bytes; use core::mem::size_of; +pub(super) struct Tier1WorkspaceAccounting { + retained_bytes: usize, +} + #[derive(Default)] struct Tier1Requirements { classic_width: u32, @@ -98,9 +102,10 @@ pub(super) fn prepare_tier1_workspace( header: &Header<'_>, tile_ctx: &mut TileDecodeContext, storage: &mut DecompositionStorage<'_>, -) -> Result { +) -> Result { let requirements = collect_requirements(tile, header, storage)?; let planned_bytes = requirements.logical_bytes()?; + let retained_bytes = tile_ctx.tier1_capacity_bytes()?; let mut budget = DecodeAllocationBudget::for_storage(storage)?; budget.include_bytes(planned_bytes)?; @@ -128,7 +133,7 @@ pub(super) fn prepare_tier1_workspace( .structural_workspace_bytes .checked_add(actual_bytes) .ok_or(ValidationError::ImageTooLarge)?; - Ok(actual_bytes) + Ok(Tier1WorkspaceAccounting { retained_bytes }) })(); if prepared.is_err() { @@ -138,14 +143,17 @@ pub(super) fn prepare_tier1_workspace( } pub(super) fn release_tier1_workspace( - tile_ctx: &mut TileDecodeContext, + _tile_ctx: &mut TileDecodeContext, storage: &mut DecompositionStorage<'_>, - accounted_bytes: usize, + accounting: &Tier1WorkspaceAccounting, ) -> Result<()> { - tile_ctx.release_tier1_allocations(); + // The active structural baseline already included the retained owner. + // Preparation temporarily charged the complete new owner so a Vec growth + // also accounts for the replacement peak. Keep the resulting owner live + // and remove only the now-duplicated retained capacity. storage.structural_workspace_bytes = storage .structural_workspace_bytes - .checked_sub(accounted_bytes) + .checked_sub(accounting.retained_bytes) .ok_or(ValidationError::ImageTooLarge)?; Ok(()) } diff --git a/crates/j2k-native/src/j2c/decode/workspace.rs b/crates/j2k-native/src/j2c/decode/workspace.rs new file mode 100644 index 00000000..38d0dc22 --- /dev/null +++ b/crates/j2k-native/src/j2c/decode/workspace.rs @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Reusable decoder workspace ownership, policy, and diagnostics. + +use super::{DecompositionStorage, OutputRegion, TileDecodeContext}; + +/// CPU parallelism policy for native JPEG 2000 decode. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum CpuDecodeParallelism { + /// Allow a single tile decode to use internal code-block parallelism. + #[default] + Auto, + /// Keep code-block decode serial for callers that already parallelize tiles. + Serial, +} + +/// Observable counters and retained ownership for a reusable decoder workspace. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct DecoderWorkspaceStats { + decode_calls: u64, + component_owner_reuses: u64, + tier1_owner_reuses: u64, + idwt_owner_reuses: u64, + scratch_capacity_retries: u64, + retained_component_bytes: usize, + retained_tier1_bytes: usize, + retained_idwt_bytes: usize, +} + +impl DecoderWorkspaceStats { + /// Number of native image decode calls made with this workspace. + #[must_use] + pub const fn decode_calls(self) -> u64 { + self.decode_calls + } + + /// Number of calls that began with reusable decoded-component owners. + #[must_use] + pub const fn component_owner_reuses(self) -> u64 { + self.component_owner_reuses + } + + /// Number of calls that began with reusable Tier-1 allocations. + #[must_use] + pub const fn tier1_owner_reuses(self) -> u64 { + self.tier1_owner_reuses + } + + /// Number of calls that began with reusable IDWT allocations. + #[must_use] + pub const fn idwt_owner_reuses(self) -> u64 { + self.idwt_owner_reuses + } + + /// Number of retained-scratch evictions followed by a fresh retry. + #[must_use] + pub const fn scratch_capacity_retries(self) -> u64 { + self.scratch_capacity_retries + } + + /// Retained component-owner capacity after the most recent completed core decode. + #[must_use] + pub const fn retained_component_bytes(self) -> usize { + self.retained_component_bytes + } + + /// Retained classic and HT Tier-1 capacity after the most recent call. + #[must_use] + pub const fn retained_tier1_bytes(self) -> usize { + self.retained_tier1_bytes + } + + /// Retained floating-point and exact-integer IDWT capacity after the most recent call. + #[must_use] + pub const fn retained_idwt_bytes(self) -> usize { + self.retained_idwt_bytes + } + + /// Total retained lifetime-free decode scratch after the most recent call. + #[must_use] + pub const fn retained_scratch_bytes(self) -> usize { + self.retained_tier1_bytes + .saturating_add(self.retained_idwt_bytes) + } +} + +/// Lifetime-free allocation owner that can be moved between borrowing decoder contexts. +/// +/// Parsed packet and tile graphs remain in [`DecoderContext`] and are always +/// released before this value is recovered. Decoded component, Tier-1, and +/// IDWT allocations are retained for reuse with unrelated encoded inputs. +#[derive(Default)] +pub struct DecoderWorkspace { + tile_decode_context: TileDecodeContext, + pub(super) cpu_decode_parallelism: CpuDecodeParallelism, + stats: DecoderWorkspaceStats, +} + +impl DecoderWorkspace { + /// Return reuse counters and retained allocation sizes. + #[must_use] + pub const fn stats(&self) -> DecoderWorkspaceStats { + self.stats + } +} + +/// A decoder context for decoding JPEG2000 images. +pub struct DecoderContext<'a> { + pub(crate) tile_decode_context: TileDecodeContext, + pub(crate) storage: DecompositionStorage<'a>, + pub(super) cpu_decode_parallelism: CpuDecodeParallelism, + workspace_stats: DecoderWorkspaceStats, +} + +impl Default for DecoderContext<'_> { + fn default() -> Self { + Self { + tile_decode_context: TileDecodeContext::default(), + storage: DecompositionStorage::default(), + cpu_decode_parallelism: CpuDecodeParallelism::Auto, + workspace_stats: DecoderWorkspaceStats::default(), + } + } +} + +impl DecoderContext<'_> { + /// Create a borrowing decoder context from a lifetime-free reusable workspace. + #[must_use] + pub fn from_workspace(workspace: DecoderWorkspace) -> Self { + Self { + tile_decode_context: workspace.tile_decode_context, + storage: DecompositionStorage::default(), + cpu_decode_parallelism: workspace.cpu_decode_parallelism, + workspace_stats: workspace.stats, + } + } + + /// Release input-borrowing graph owners and recover the reusable workspace. + #[must_use] + pub fn into_workspace(mut self) -> DecoderWorkspace { + self.storage.release_all_allocations(); + DecoderWorkspace { + tile_decode_context: self.tile_decode_context, + cpu_decode_parallelism: self.cpu_decode_parallelism, + stats: self.workspace_stats, + } + } + + /// Return reuse counters for this context's lifetime-free workspace state. + #[must_use] + pub const fn workspace_stats(&self) -> DecoderWorkspaceStats { + self.workspace_stats + } + + pub(super) fn record_decode_start( + &mut self, + retained_component_bytes: usize, + retained_tier1_bytes: usize, + retained_idwt_bytes: usize, + ) { + self.workspace_stats.decode_calls = self.workspace_stats.decode_calls.saturating_add(1); + if retained_component_bytes != 0 { + self.workspace_stats.component_owner_reuses = self + .workspace_stats + .component_owner_reuses + .saturating_add(1); + } + if retained_tier1_bytes != 0 { + self.workspace_stats.tier1_owner_reuses = + self.workspace_stats.tier1_owner_reuses.saturating_add(1); + } + if retained_idwt_bytes != 0 { + self.workspace_stats.idwt_owner_reuses = + self.workspace_stats.idwt_owner_reuses.saturating_add(1); + } + } + + pub(super) fn record_scratch_capacity_retry(&mut self) { + self.workspace_stats.scratch_capacity_retries = self + .workspace_stats + .scratch_capacity_retries + .saturating_add(1); + } + + pub(super) fn record_decode_complete( + &mut self, + retained_component_bytes: usize, + retained_tier1_bytes: usize, + retained_idwt_bytes: usize, + ) { + self.workspace_stats.retained_component_bytes = retained_component_bytes; + self.workspace_stats.retained_tier1_bytes = retained_tier1_bytes; + self.workspace_stats.retained_idwt_bytes = retained_idwt_bytes; + } + + pub(crate) fn release_reusable_allocations(&mut self) { + self.tile_decode_context.release_all_allocations(); + self.storage.release_all_allocations(); + } + + pub(crate) fn set_output_region(&mut self, output_region: Option<(u32, u32, u32, u32)>) { + self.tile_decode_context.output_region = output_region.map(OutputRegion::from_tuple); + } + + /// Return the native CPU decode parallelism policy. + #[must_use] + pub fn cpu_decode_parallelism(&self) -> CpuDecodeParallelism { + self.cpu_decode_parallelism + } + + /// Set the native CPU decode parallelism policy. + pub fn set_cpu_decode_parallelism(&mut self, parallelism: CpuDecodeParallelism) { + self.cpu_decode_parallelism = parallelism; + } +} diff --git a/crates/j2k-native/src/j2c/ht_block_decode/readers.rs b/crates/j2k-native/src/j2c/ht_block_decode/readers.rs index a8e32533..fa84706c 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/readers.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/readers.rs @@ -127,9 +127,12 @@ impl<'a, const PAD: u8> ForwardBitReader<'a, PAD> { PAD }; + let valid_bits = 8 - u32::from(self.unstuff); + let next_unstuff = byte == 0xFF; + let byte = if self.unstuff { byte & 0x7F } else { byte }; self.tmp |= u64::from(byte) << self.bits; - self.bits += 8 - u32::from(self.unstuff); - self.unstuff = byte == 0xFF; + self.bits += valid_bits; + self.unstuff = next_unstuff; } } @@ -199,10 +202,13 @@ impl<'a> ReverseBitReader<'a> { 0 }; - let d_bits = 8 - u32::from(self.unstuff && (byte & 0x7F) == 0x7F); + let stuffed = self.unstuff && (byte & 0x7F) == 0x7F; + let d_bits = 8 - u32::from(stuffed); + let next_unstuff = byte > 0x8F; + let byte = if stuffed { byte & 0x7F } else { byte }; self.tmp |= u64::from(byte) << self.bits; self.bits += d_bits; - self.unstuff = byte > 0x8F; + self.unstuff = next_unstuff; } } @@ -245,7 +251,7 @@ mod tests { assert_eq!(forward.fetch(), 0xFFFF_C03F); assert_eq!( (forward.pos, forward.bits, forward.tmp, forward.unstuff), - (5, 37, 0x0000_003F_FFFF_C03F, true) + (5, 37, 0x0000_001F_FFFF_C03F, true) ); let mut reverse = ReverseBitReader::new_mrp(&data); diff --git a/crates/j2k-native/src/j2c/ht_block_decode/significance.rs b/crates/j2k-native/src/j2c/ht_block_decode/significance.rs index 22162448..70822261 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/significance.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/significance.rs @@ -177,11 +177,8 @@ pub(super) fn apply_significance_propagation_phase( sigprop.advance(cnt); } - let combined_sig = new_sig | cs; + let combined_sig = new_sig | (cs & 0xFFFF); prev_row_sig[idx] = combined_sig as u16; - if idx + 1 < prev_row_sig.len() { - prev_row_sig[idx + 1] = (combined_sig >> 16) as u16; - } let t = combined_sig; let mut next_prev = combined_sig; diff --git a/crates/j2k-native/src/j2c/ht_block_decode/state.rs b/crates/j2k-native/src/j2c/ht_block_decode/state.rs index 0773de15..e2ea6ff9 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/state.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/state.rs @@ -50,9 +50,14 @@ impl HtBlockDecodeContext { pub(crate) fn coefficient_rows(&self) -> impl Iterator { self.coefficients.chunks_exact(self.width as usize) } + + #[cfg(test)] + pub(crate) fn coefficient_owner_for_test(&self) -> (*const u32, usize) { + (self.coefficients.as_ptr(), self.coefficients.capacity()) + } } -#[derive(Default)] +#[derive(Debug, Default)] pub(crate) struct HtBlockDecodeScratch { pub(super) cleanup: Vec, pub(super) v_n: Vec, @@ -61,6 +66,15 @@ pub(crate) struct HtBlockDecodeScratch { } impl HtBlockDecodeScratch { + pub(crate) const fn empty() -> Self { + Self { + cleanup: Vec::new(), + v_n: Vec::new(), + sigma: Vec::new(), + prev_row_sig: Vec::new(), + } + } + pub(crate) fn prepare(&mut self, width: u32, height: u32) -> Result<()> { prepare_scratch(self, width, height) } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/tests.rs b/crates/j2k-native/src/j2c/ht_block_decode/tests.rs index 139b37f0..075e9cc7 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/tests.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/tests.rs @@ -8,6 +8,7 @@ use super::cleanup::{ use super::facade::coefficient_to_i32; use super::magnitude::decode_magnitude_sign_phase; use super::pipeline::{decode_impl, prepare_scratch, PHASE_LIMIT_MAGREF}; +use super::readers::{ForwardBitReader, ReverseBitReader}; use super::refinement::apply_magnitude_refinement_phase; use super::segments::{CombinedCodeBlockData, HtCodeBlockSegments}; use super::significance::{ @@ -28,6 +29,20 @@ use super::{ use crate::error::{DecodeError, DecodingError}; use crate::j2c::ht_block_encode::encode_code_block; +#[test] +fn sigprop_reader_discards_stuffed_msb_even_when_overlap_sets_it() { + let mut reader = ForwardBitReader::<0>::new(&[0xFF, 0x80, 0x00, 0x00, 0x00]); + + assert_eq!(reader.fetch(), 0x0000_00FF); +} + +#[test] +fn magref_reader_discards_stuffed_msb_in_shared_refinement_byte() { + let mut reader = ReverseBitReader::new_mrp(&[0x00, 0x00, 0x00, 0x00, 0xFF]); + + assert_eq!(reader.fetch(), 0x0000_007F); +} + #[test] fn test_coefficient_to_i32_shifted_alignment() { let aligned = 3u32 << (31 - 5); diff --git a/crates/j2k-native/src/j2c/mod.rs b/crates/j2k-native/src/j2c/mod.rs index c0225dcf..b9d75e47 100644 --- a/crates/j2k-native/src/j2c/mod.rs +++ b/crates/j2k-native/src/j2c/mod.rs @@ -33,6 +33,7 @@ use alloc::vec::Vec; use super::jp2::colr::{ColorSpace, ColorSpecificationBox, EnumeratedColorspace}; use super::jp2::ImageBoxes; use crate::error::{bail, FormatError, MarkerError, Result}; +use crate::image::ImageSource; use crate::j2c::codestream::markers; use crate::reader::BitReader; use crate::{resolve_alpha_and_color_space, DecodeSettings, Image}; @@ -41,8 +42,13 @@ use crate::math::{SimdBuffer, SIMD_WIDTH}; pub(crate) use codestream::Header; #[cfg(test)] pub(crate) use decode::should_decode_classic_sub_band_in_parallel; -pub(crate) use decode::{build_direct_color_plan, build_direct_grayscale_plan, decode}; -pub use decode::{CpuDecodeParallelism, DecoderContext}; +pub(crate) use decode::{ + build_direct_color_plan, build_direct_grayscale_plan, build_referenced_classic_color_plan, + build_referenced_classic_grayscale_plan, build_referenced_classic_rgba_plan, + build_referenced_htj2k_color_plan, build_referenced_htj2k_grayscale_plan, + build_referenced_htj2k_rgba_plan, decode_with_capacity_retry as decode, +}; +pub use decode::{CpuDecodeParallelism, DecoderContext, DecoderWorkspace, DecoderWorkspaceStats}; pub use recode::Reversible53CoefficientImage; pub(crate) use segment::MAX_BITPLANE_COUNT; @@ -103,7 +109,7 @@ pub(crate) fn parse_with_retained_baseline<'a>( )?; if retained_baseline_bytes == 0 { Image::from_parsed_parts( - parsed_codestream.data, + ImageSource::new(stream, parsed_codestream.data), parsed_codestream.header, boxes, *settings, @@ -112,7 +118,7 @@ pub(crate) fn parse_with_retained_baseline<'a>( ) } else { Image::from_parsed_parts_with_retained_baseline( - parsed_codestream.data, + ImageSource::new(stream, parsed_codestream.data), parsed_codestream.header, boxes, *settings, diff --git a/crates/j2k-native/src/jp2/container.rs b/crates/j2k-native/src/jp2/container.rs index 0f281deb..14427caf 100644 --- a/crates/j2k-native/src/jp2/container.rs +++ b/crates/j2k-native/src/jp2/container.rs @@ -3,6 +3,7 @@ //! JP2/JPH container traversal and native decode parse orchestration. use crate::error::{bail, FormatError, Result}; +use crate::image::ImageSource; use crate::reader::BitReader; use crate::{resolve_alpha_and_color_space, DecodeSettings, Image}; @@ -178,7 +179,7 @@ pub(crate) fn parse_with_retained_baseline( )?; if retained_baseline_bytes == 0 { Image::from_parsed_parts( - parsed_codestream.data, + ImageSource::new(data, parsed_codestream.data), parsed_codestream.header, image_boxes, settings, @@ -187,7 +188,7 @@ pub(crate) fn parse_with_retained_baseline( ) } else { Image::from_parsed_parts_with_retained_baseline( - parsed_codestream.data, + ImageSource::new(data, parsed_codestream.data), parsed_codestream.header, image_boxes, settings, diff --git a/crates/j2k-native/src/lib.rs b/crates/j2k-native/src/lib.rs index 23b676a1..570382e8 100644 --- a/crates/j2k-native/src/lib.rs +++ b/crates/j2k-native/src/lib.rs @@ -179,13 +179,26 @@ pub use color::{ pub use color::{ComponentPlaneParts, NativeComponentPlaneParts}; #[doc(hidden)] pub use direct_cpu::{ - execute_direct_color_plan_rgb8_into, execute_direct_color_plan_rgba8_into, J2kDirectCpuScratch, + execute_direct_color_plan_rgb8_into, execute_direct_color_plan_rgba8_into, + execute_referenced_classic_entropy_job, execute_referenced_classic_plan, + execute_referenced_classic_plan_from_payloads, execute_referenced_htj2k_entropy_job, + execute_referenced_htj2k_plan, execute_referenced_htj2k_plan_from_payloads, + finish_referenced_classic_staged, finish_referenced_classic_tile_staged, + finish_referenced_htj2k_staged, finish_referenced_htj2k_tile_staged, + prepare_referenced_classic_entropy_workspace, prepare_referenced_classic_staged, + prepare_referenced_classic_tile_staged, prepare_referenced_htj2k_entropy_workspace, + prepare_referenced_htj2k_staged, prepare_referenced_htj2k_tile_staged, J2kDirectCodeBlockIndex, + J2kDirectCpuEntropyWorkspace, J2kDirectCpuScratch, J2kDirectDecodedComponents, + J2kDirectDecodedPlane, }; #[doc(hidden)] pub use direct_plan::{ - HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, J2kDirectBandId, J2kDirectColorPlan, - J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, - J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, + HtCodeBlockPayloadRanges, HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, + J2kClassicCodeBlockPayload, J2kCodestreamRange, J2kDirectBandId, J2kDirectColorPlan, + J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectRgbaPlan, + J2kDirectStoreStep, J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, J2kReferencedClassicPlan, + J2kReferencedHtj2kPlan, J2kReferencedPayloadRecordSpan, J2kReferencedTileGeometry, + J2kReferencedTilePlan, }; #[doc(hidden)] pub use direct_roi::{ @@ -240,7 +253,10 @@ pub use j2c::encode::{ irreversible_quantization_step_for_subband, EncodeComponentPlane, EncodeOptions, EncodeProgressionOrder, EncodeRoiRegion, EncodeTypedComponentPlane, ResidentHtj2kEncodeError, }; -pub use j2c::{CpuDecodeParallelism, DecoderContext, Reversible53CoefficientImage}; +pub use j2c::{ + CpuDecodeParallelism, DecoderContext, DecoderWorkspace, DecoderWorkspaceStats, + Reversible53CoefficientImage, +}; #[doc(hidden)] pub use j2k_types::{ sort_packet_descriptors_for_progression, CpuOnlyJ2kEncodeStageAccelerator, diff --git a/crates/j2k-native/src/scalar/classic_decode.rs b/crates/j2k-native/src/scalar/classic_decode.rs index 720a16e6..db0d73d2 100644 --- a/crates/j2k-native/src/scalar/classic_decode.rs +++ b/crates/j2k-native/src/scalar/classic_decode.rs @@ -23,7 +23,21 @@ pub struct J2kCodeBlockDecodeWorkspace { bit_plane_decode_context: j2c::bitplane::BitPlaneDecodeContext, } +impl core::fmt::Debug for J2kCodeBlockDecodeWorkspace { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("J2kCodeBlockDecodeWorkspace") + .field("allocated_bytes", &self.allocated_bytes().ok()) + .finish_non_exhaustive() + } +} + impl J2kCodeBlockDecodeWorkspace { + pub(crate) const fn empty() -> Self { + Self { + bit_plane_decode_context: j2c::bitplane::BitPlaneDecodeContext::empty(), + } + } + pub(crate) fn prepare(&mut self, width: u32, height: u32) -> Result<()> { self.bit_plane_decode_context.prepare(width, height) } diff --git a/crates/j2k-native/src/scalar/ht_decode.rs b/crates/j2k-native/src/scalar/ht_decode.rs index 7732496f..0f7d7cb7 100644 --- a/crates/j2k-native/src/scalar/ht_decode.rs +++ b/crates/j2k-native/src/scalar/ht_decode.rs @@ -42,7 +42,7 @@ pub fn decode_ht_code_block_scalar_until_phase( } /// Adapter reusable scalar HTJ2K decode workspace for backend experimentation. -#[derive(Default)] +#[derive(Debug, Default)] #[doc(hidden)] pub struct HtCodeBlockDecodeWorkspace { coefficients: Vec, @@ -50,6 +50,13 @@ pub struct HtCodeBlockDecodeWorkspace { } impl HtCodeBlockDecodeWorkspace { + pub(crate) const fn empty() -> Self { + Self { + coefficients: Vec::new(), + scratch: j2c::ht_block_decode::HtBlockDecodeScratch::empty(), + } + } + /// Current coefficient buffer capacity retained by this workspace. #[must_use] pub fn coefficient_capacity(&self) -> usize { diff --git a/crates/j2k-native/src/tests.rs b/crates/j2k-native/src/tests.rs index 06283fe6..7b3ac5c1 100644 --- a/crates/j2k-native/src/tests.rs +++ b/crates/j2k-native/src/tests.rs @@ -3,6 +3,8 @@ use super::*; use crate::j2c::ComponentData; +mod workspace_reuse; + #[test] fn direct_grayscale_plan_rejects_rgb_image_with_typed_reason() { let pixels = vec![0, 16, 32, 64, 80, 96, 128, 144, 160, 192, 208, 224]; @@ -22,7 +24,6 @@ fn direct_grayscale_plan_rejects_rgb_image_with_typed_reason() { )) ); } - #[test] fn ht_uvlc_encode_table_bytes_match_entry_packing_order() { let entries = ht_uvlc_encode_table(); diff --git a/crates/j2k-native/src/tests/workspace_reuse.rs b/crates/j2k-native/src/tests/workspace_reuse.rs new file mode 100644 index 00000000..c253d708 --- /dev/null +++ b/crates/j2k-native/src/tests/workspace_reuse.rs @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::{fixture_gray, fixture_ht_gray}; +use crate::{ + encode, encode_typed_component_planes_53, DecodeSettings, DecoderContext, EncodeOptions, + EncodeTypedComponentPlane, Image, +}; + +#[test] +fn decoder_workspace_reuses_component_owners_across_distinct_input_lifetimes() { + let mut workspace = crate::DecoderWorkspace::default(); + let first_pixels = { + let bytes = fixture_gray(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("first image"); + let mut context = DecoderContext::from_workspace(workspace); + let decoded = image + .decode_with_context(&mut context) + .expect("first workspace decode"); + let pixels = decoded.data.clone(); + drop(decoded); + workspace = context.into_workspace(); + pixels + }; + + let second_pixels = { + let bytes = fixture_gray(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("second image"); + let mut context = DecoderContext::from_workspace(workspace); + let decoded = image + .decode_with_context(&mut context) + .expect("second workspace decode"); + let pixels = decoded.data.clone(); + drop(decoded); + workspace = context.into_workspace(); + pixels + }; + + assert_eq!(second_pixels, first_pixels); + assert_eq!(workspace.stats().decode_calls(), 2); + assert_eq!(workspace.stats().component_owner_reuses(), 1); + assert!(workspace.stats().retained_component_bytes() > 0); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ScratchCapacitySnapshot { + classic_coefficients: (*const crate::j2c::bitplane::Coefficient, usize), + classic_payload: (*const u8, usize), + ht_coefficients: (*const u32, usize), + idwt_output: (*const f32, usize), + idwt_scratch: (*const f32, usize), + idwt_output_i64: (*const i64, usize), + idwt_scratch_i64: (*const i64, usize), +} + +fn scratch_capacity_snapshot(context: &DecoderContext<'_>) -> ScratchCapacitySnapshot { + let tile = &context.tile_decode_context; + ScratchCapacitySnapshot { + classic_coefficients: ( + tile.bit_plane_decode_context.coefficient_ptr_for_test(), + tile.bit_plane_decode_context + .coefficient_capacity_for_test(), + ), + classic_payload: tile + .bit_plane_decode_buffers + .combined_layers_owner_for_test(), + ht_coefficients: tile.ht_block_decode_context.coefficient_owner_for_test(), + idwt_output: ( + tile.idwt_output.coefficients.as_ptr(), + tile.idwt_output.coefficients.capacity(), + ), + idwt_scratch: ( + tile.idwt_scratch_buffer.as_ptr(), + tile.idwt_scratch_buffer.capacity(), + ), + idwt_output_i64: ( + tile.idwt_output.coefficients_i64.as_ptr(), + tile.idwt_output.coefficients_i64.capacity(), + ), + idwt_scratch_i64: ( + tile.idwt_scratch_buffer_i64.as_ptr(), + tile.idwt_scratch_buffer_i64.capacity(), + ), + } +} + +#[test] +fn decoder_workspace_reuses_classic_tier1_and_idwt_owners_across_input_lifetimes() { + let mut workspace = crate::DecoderWorkspace::default(); + let first = { + let bytes = fixture_gray(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("first image"); + let mut context = DecoderContext::from_workspace(workspace); + let decoded = image + .decode_with_context(&mut context) + .expect("first classic decode"); + drop(decoded); + let scratch = scratch_capacity_snapshot(&context); + workspace = context.into_workspace(); + scratch + }; + let second = { + let bytes = fixture_gray(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("second image"); + let mut context = DecoderContext::from_workspace(workspace); + let decoded = image + .decode_with_context(&mut context) + .expect("second classic decode"); + drop(decoded); + let scratch = scratch_capacity_snapshot(&context); + workspace = context.into_workspace(); + scratch + }; + + assert!(first.classic_coefficients.1 > 0); + assert!(first.classic_payload.1 > 0); + assert!(first.idwt_output.1 > 0); + assert_eq!(second, first); + assert_eq!(workspace.stats().tier1_owner_reuses(), 1); + assert_eq!(workspace.stats().idwt_owner_reuses(), 1); + assert!(workspace.stats().retained_tier1_bytes() > 0); + assert!(workspace.stats().retained_idwt_bytes() > 0); +} + +#[test] +fn decoder_workspace_reuses_ht_tier1_owner_across_input_lifetimes() { + let mut workspace = crate::DecoderWorkspace::default(); + let first = { + let bytes = fixture_ht_gray(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("first HT image"); + let mut context = DecoderContext::from_workspace(workspace); + let decoded = image + .decode_with_context(&mut context) + .expect("first HT decode"); + drop(decoded); + let scratch = scratch_capacity_snapshot(&context); + workspace = context.into_workspace(); + scratch + }; + let second = { + let bytes = fixture_ht_gray(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("second HT image"); + let mut context = DecoderContext::from_workspace(workspace); + let decoded = image + .decode_with_context(&mut context) + .expect("second HT decode"); + drop(decoded); + let scratch = scratch_capacity_snapshot(&context); + workspace = context.into_workspace(); + scratch + }; + + assert!(first.ht_coefficients.1 > 0); + assert!(first.idwt_output.1 > 0); + assert_eq!(second, first); + assert_eq!(workspace.stats().tier1_owner_reuses(), 1); + assert_eq!(workspace.stats().idwt_owner_reuses(), 1); +} + +#[test] +fn decoder_workspace_reuses_scratch_across_alternating_shapes_and_precision() { + let large_pixels = (0..16_u16 * 16) + .map(|sample| (sample & 0xff) as u8) + .collect::>(); + let large_bytes = encode( + &large_pixels, + 16, + 16, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }, + ) + .expect("encode large classic fixture"); + let exact_samples = [0_u32, 1, (1_u32 << 28) + 7, (1_u32 << 29) - 1]; + let exact_pixels = exact_samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + let exact_planes = [EncodeTypedComponentPlane { + data: &exact_pixels, + x_rsiz: 1, + y_rsiz: 1, + bit_depth: 29, + signed: false, + }]; + let exact_bytes = encode_typed_component_planes_53( + &exact_planes, + 2, + 2, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + use_mct: false, + ..EncodeOptions::default() + }, + ) + .expect("encode exact fixture"); + + let mut workspace = crate::DecoderWorkspace::default(); + let large_scratch = { + let image = Image::new(&large_bytes, &DecodeSettings::default()).expect("large image"); + let mut context = DecoderContext::from_workspace(workspace); + let decoded = image + .decode_native_with_context(&mut context) + .expect("large decode"); + assert_eq!(decoded.data, large_pixels); + drop(decoded); + let scratch = scratch_capacity_snapshot(&context); + workspace = context.into_workspace(); + scratch + }; + let exact_scratch = { + let image = Image::new(&exact_bytes, &DecodeSettings::default()).expect("exact image"); + let mut context = DecoderContext::from_workspace(workspace); + let decoded = image + .decode_native_with_context(&mut context) + .expect("exact decode"); + assert_eq!(decoded.data, exact_pixels); + drop(decoded); + let scratch = scratch_capacity_snapshot(&context); + workspace = context.into_workspace(); + scratch + }; + let final_scratch = { + let bytes = fixture_gray(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("small image"); + let mut context = DecoderContext::from_workspace(workspace); + let decoded = image + .decode_with_context(&mut context) + .expect("small decode"); + assert_eq!(decoded.data, (0_u8..16).collect::>()); + drop(decoded); + let scratch = scratch_capacity_snapshot(&context); + workspace = context.into_workspace(); + scratch + }; + + assert_eq!( + final_scratch.classic_coefficients, + large_scratch.classic_coefficients + ); + assert_eq!(final_scratch.classic_payload, large_scratch.classic_payload); + assert_eq!(final_scratch.idwt_output, large_scratch.idwt_output); + assert_eq!(final_scratch.idwt_scratch, large_scratch.idwt_scratch); + assert!(exact_scratch.idwt_output_i64.1 > 0); + assert_eq!(final_scratch.idwt_output_i64, exact_scratch.idwt_output_i64); + assert_eq!(workspace.stats().decode_calls(), 3); + assert_eq!(workspace.stats().tier1_owner_reuses(), 2); + assert_eq!(workspace.stats().idwt_owner_reuses(), 2); + assert_eq!(workspace.stats().scratch_capacity_retries(), 0); +} diff --git a/crates/j2k-native/tests/referenced_multitile.rs b/crates/j2k-native/tests/referenced_multitile.rs new file mode 100644 index 00000000..92c8868d --- /dev/null +++ b/crates/j2k-native/tests/referenced_multitile.rs @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::collections::HashSet; + +use j2k_native::{ + execute_referenced_htj2k_plan, prepare_referenced_htj2k_staged, DecodeSettings, DecoderContext, + Image, J2kDirectCpuEntropyWorkspace, J2kDirectCpuScratch, J2kDirectGrayscaleStep, J2kRect, +}; + +const OPENJPH_GRAY_U12_53: &[u8] = include_bytes!("../fixtures/htj2k/gray_u12_53.j2c"); +const OPENJPH_GRAY_U12_53_ORACLE: &[u8] = + include_bytes!("../fixtures/htj2k/gray_u12_53.oracle.raw"); +const OPENJPH_RGB_U12_53: &[u8] = include_bytes!("../fixtures/htj2k/rgb_u12_53.j2c"); +const OPENJPH_RGB_U12_53_ORACLE: &[u8] = include_bytes!("../fixtures/htj2k/rgb_u12_53.oracle.raw"); + +#[test] +fn referenced_htj2k_grayscale_executes_all_tiles_bit_exactly() { + let image = Image::new(OPENJPH_GRAY_U12_53, &DecodeSettings::strict()) + .expect("parse independent OpenJPH fixture"); + let mut context = DecoderContext::default(); + let plan = image + .build_referenced_htj2k_plan_region_with_context(&mut context, (0, 0, 19, 13)) + .expect("build referenced multi-tile HTJ2K plan"); + let mut scratch = J2kDirectCpuScratch::new(); + let decoded = execute_referenced_htj2k_plan(&plan, OPENJPH_GRAY_U12_53, false, &mut scratch) + .expect("execute every referenced tile"); + + assert_eq!(decoded.dimensions(), (19, 13)); + assert_eq!(decoded.component_count(), 1); + let actual = decoded.plane(0).expect("grayscale plane").samples(); + let expected = OPENJPH_GRAY_U12_53_ORACLE + .chunks_exact(2) + .map(|bytes| u16::from_le_bytes([bytes[0], bytes[1]])); + assert_eq!(actual.len(), 19 * 13); + for (index, (actual, expected)) in actual.iter().copied().zip(expected).enumerate() { + assert_eq!( + actual.to_bits(), + f32::from(expected).to_bits(), + "sample {index}" + ); + } +} + +#[test] +fn referenced_htj2k_rgb_executes_all_tiles_bit_exactly() { + let image = Image::new(OPENJPH_RGB_U12_53, &DecodeSettings::strict()) + .expect("parse independent OpenJPH fixture"); + let mut context = DecoderContext::default(); + let plan = image + .build_referenced_htj2k_plan_region_with_context(&mut context, (0, 0, 19, 13)) + .expect("build referenced multi-tile RGB HTJ2K plan"); + let mut scratch = J2kDirectCpuScratch::new(); + let decoded = execute_referenced_htj2k_plan(&plan, OPENJPH_RGB_U12_53, false, &mut scratch) + .expect("execute every referenced RGB tile"); + + assert_eq!(decoded.dimensions(), (19, 13)); + assert_eq!(decoded.component_count(), 3); + let planes = [ + decoded.plane(0).expect("red plane"), + decoded.plane(1).expect("green plane"), + decoded.plane(2).expect("blue plane"), + ]; + let expected: Vec<_> = OPENJPH_RGB_U12_53_ORACLE + .chunks_exact(2) + .map(|bytes| u16::from_le_bytes([bytes[0], bytes[1]])) + .collect(); + for pixel in 0..19 * 13 { + for component in 0..3 { + assert_eq!( + planes[component].samples()[pixel].to_bits(), + f32::from(expected[pixel * 3 + component]).to_bits(), + "pixel {pixel}, component {component}" + ); + } + } +} + +#[test] +#[expect( + clippy::too_many_lines, + reason = "the structural regression validates tile order, payload spans, globally unique bands, stores, and bounded staged scratch together" +)] +fn referenced_htj2k_grayscale_plan_preserves_all_odd_edge_tiles() { + let image = Image::new(OPENJPH_GRAY_U12_53, &DecodeSettings::strict()) + .expect("parse independent OpenJPH fixture"); + let mut context = DecoderContext::default(); + let plan = image + .build_referenced_htj2k_plan_region_with_context(&mut context, (0, 0, 19, 13)) + .expect("build referenced multi-tile HTJ2K plan"); + + assert_eq!(plan.full_dimensions(), (19, 13)); + assert_eq!( + plan.output_rect(), + J2kRect { + x0: 0, + y0: 0, + x1: 19, + y1: 13, + } + ); + assert!( + plan.grayscale_geometry().is_none(), + "the legacy single-tile accessor must fail closed for multi-tile plans" + ); + + let tiles = plan.tiles(); + assert_eq!(tiles.len(), 4); + let expected_rects = [ + J2kRect { + x0: 0, + y0: 0, + x1: 11, + y1: 7, + }, + J2kRect { + x0: 11, + y0: 0, + x1: 19, + y1: 7, + }, + J2kRect { + x0: 0, + y0: 7, + x1: 11, + y1: 13, + }, + J2kRect { + x0: 11, + y0: 7, + x1: 19, + y1: 13, + }, + ]; + + let mut produced_band_ids = HashSet::new(); + let mut next_payload_record = 0; + let mut maximum_live_band_owners = 0usize; + let mut all_tile_band_owners = 0usize; + for (expected_index, (tile, expected_rect)) in tiles.iter().zip(expected_rects).enumerate() { + assert_eq!(tile.tile_index(), expected_index); + assert_eq!(tile.decoded_rect(), expected_rect); + assert_eq!(tile.destination_rect(), expected_rect); + + let span = tile.payload_records(); + assert_eq!(span.first_record, next_payload_record); + next_payload_record = span.end_record().expect("payload span does not overflow"); + + let geometry = tile + .grayscale_geometry() + .expect("grayscale tile has grayscale geometry"); + assert_eq!(geometry.dimensions, (19, 13)); + let mut store_count = 0; + let mut job_count = 0; + let mut tile_band_owners = 0usize; + for step in &geometry.steps { + match step { + J2kDirectGrayscaleStep::HtSubBand(sub_band) => { + tile_band_owners += 1; + assert!( + produced_band_ids.insert(sub_band.band_id), + "sub-band IDs must be globally unique across tiles" + ); + job_count += sub_band.jobs.len(); + } + J2kDirectGrayscaleStep::Idwt(idwt) => { + tile_band_owners += 1; + assert!( + produced_band_ids.insert(idwt.output_band_id), + "IDWT output IDs must be globally unique across tiles" + ); + } + J2kDirectGrayscaleStep::Store(store) => { + store_count += 1; + assert_eq!((store.output_width, store.output_height), (19, 13)); + assert_eq!( + (store.output_x, store.output_y), + (expected_rect.x0, expected_rect.y0) + ); + assert_eq!( + (store.copy_width, store.copy_height), + (expected_rect.width(), expected_rect.height()) + ); + } + J2kDirectGrayscaleStep::ClassicSubBand(_) => { + panic!("HT plan contains classic entropy work") + } + } + } + assert_eq!(store_count, 1); + assert_eq!(job_count, span.record_count); + maximum_live_band_owners = maximum_live_band_owners.max(tile_band_owners); + all_tile_band_owners += tile_band_owners; + } + + assert_eq!(next_payload_record, plan.payloads().len()); + let mut scratch = J2kDirectCpuScratch::new(); + prepare_referenced_htj2k_staged( + &plan, + &mut scratch, + &mut J2kDirectCpuEntropyWorkspace::default(), + ) + .expect("prepare bounded staged multi-tile scratch"); + assert_eq!( + scratch.retained_band_owner_count(), + maximum_live_band_owners, + "live coefficient owners must be bounded by one tile" + ); + assert!( + scratch.retained_band_owner_count() < all_tile_band_owners, + "staged scratch must not retain one band set per tile" + ); +} + +#[test] +fn referenced_htj2k_reduced_roi_stores_are_disjoint_and_cover_the_destination() { + let settings = DecodeSettings { + target_resolution: Some((10, 7)), + ..DecodeSettings::strict() + }; + let image = Image::new(OPENJPH_GRAY_U12_53, &settings) + .expect("parse reduced independent OpenJPH fixture"); + assert_eq!((image.width(), image.height()), (10, 7)); + let mut context = DecoderContext::default(); + let plan = image + .build_referenced_htj2k_plan_region_with_context(&mut context, (2, 1, 7, 5)) + .expect("build reduced ROI multi-tile plan"); + + assert_eq!(plan.full_dimensions(), (10, 7)); + assert_eq!( + plan.output_rect(), + J2kRect { + x0: 2, + y0: 1, + x1: 9, + y1: 6, + } + ); + let mut coverage = [false; 7 * 5]; + let mut next_payload_record = 0; + for tile in plan.tiles() { + let span = tile.payload_records(); + assert_eq!(span.first_record, next_payload_record); + next_payload_record = span.end_record().expect("payload span does not overflow"); + + let destination = tile.destination_rect(); + let decoded = tile.decoded_rect(); + assert_eq!(decoded.x0, destination.x0 + 2); + assert_eq!(decoded.y0, destination.y0 + 1); + assert_eq!(decoded.x1, destination.x1 + 2); + assert_eq!(decoded.y1, destination.y1 + 1); + let geometry = tile.grayscale_geometry().expect("grayscale tile geometry"); + let store = geometry + .steps + .iter() + .find_map(|step| match step { + J2kDirectGrayscaleStep::Store(store) => Some(store), + _ => None, + }) + .expect("one tile store"); + assert_eq!((store.output_width, store.output_height), (7, 5)); + assert_eq!( + (store.output_x, store.output_y), + (destination.x0, destination.y0) + ); + assert_eq!( + (store.copy_width, store.copy_height), + (destination.width(), destination.height()) + ); + + for y in destination.y0..destination.y1 { + for x in destination.x0..destination.x1 { + let index = y as usize * 7 + x as usize; + assert!(!coverage[index], "tile stores overlap at ({x}, {y})"); + coverage[index] = true; + } + } + } + assert_eq!(next_payload_record, plan.payloads().len()); + assert!(coverage.into_iter().all(|covered| covered)); +} diff --git a/crates/j2k-test-support/fixtures/htj2k/README.md b/crates/j2k-test-support/fixtures/htj2k/README.md index 58c03d78..31ccece9 100644 --- a/crates/j2k-test-support/fixtures/htj2k/README.md +++ b/crates/j2k-test-support/fixtures/htj2k/README.md @@ -10,7 +10,48 @@ These fixtures mirror the tiny OpenHTJ2K-derived HTONLY codestream fixtures in `openhtj2k_ds0_ht_09_b11.j2k` is copied from `ds0_ht_09_b11.j2k`, blob `d4f2031359c32eb24825d00dde05a92cf3ae451e`. +`openhtj2k_hifi_ht1_02.j2k` is copied from `hifi_ht1_02.j2k`, blob +`61ced26eac84e240e28bc14e38928581b0601c01`. It is a 128 x 128 unsigned +RGB12 conformance codestream whose code blocks include exactly two coding +passes (`Cleanup` plus `SigProp`) as well as three-pass refinement. Its SHA-256 +is `8ecb1ddcd469e4b3fda6df01c919e831e16f7c3bc19a0b3c41d72038f2b44e53`. + The paired `.gray` files contain expected 8-bit grayscale samples in row-major order from decoding the checked-in codestreams with OpenJPH. +`openhtj2k_hifi_ht1_02.oracle.raw` contains top-down interleaved little-endian +RGB12 samples decoded with OpenJPH 0.27.0. Its SHA-256 is +`5ae8c1c25d8bffc4701c2b72d8000e34cf8b4e96e665d4177dc33dd0f6244d8d`; +the `ojph_expand` executable SHA-256 was +`4b420506bd2a44439cf472d956bc1552c8be72ff6dffe2c10042e0e36b8de843`. + +`openhtj2k_sigprop_refinement_overlap.j2k` is copied from +`tests/data/sigprop_refinement_overlap.j2k` at OpenHTJ2K commit +`3a1f96f63492ffff167ed8a764c3e362d2491bd9`, blob +`f31e936d6e1903593a33d6b611345db49efe2b4f`. It is a 512 x 64 unsigned +RGB8 stream with cleanup-only, two-pass SigProp, and three-pass MagRef blocks, +including a refinement byte whose set stuffed position overlaps the reverse +MagRef stream. Its SHA-256 is +`dec18535d0d6b9e113c0ea23a319e0cc77cd4e5ec9e37fe1e827b9489dca615d`. +The paired `.openht.oracle.ppm` is top-down interleaved RGB8 decoded with +`open_htj2k_dec` built from that same OpenHTJ2K commit. Its SHA-256 is +`d592eea6dc7d5d28693d0c5eb92cfcb7d51372e4acd8ffb1664f69b7dce7f3da`; +the decoder executable SHA-256 was +`483c92fa604823f98d899642367f68a1ae62a91276600aeb68cdfbaddbcd0fa4`. + +OpenJPH 0.27.0 produces a materially different result for this deliberately +overlapping refinement stream: 305 of its output bytes differ from OpenHTJ2K +by more than one LSB. That output is retained as +`.openjph.oracle.raw` (SHA-256 +`dddeda5af064c87abfce577ea1e6c2d714c203cef87ebb5bd3bf2507efe052b0`) +as cross-decoder evidence, but is not used as the correctness oracle for this +OpenHTJ2K-specific edge case. The independent `hifi_ht1_02` fixture above and +the `openjph_batch/` corpus continue to provide OpenJPH parity coverage. + The OpenHTJ2K BSD 3-Clause license is retained in `LICENSE.OpenHTJ2K`. + +The `openjph_batch/` directory is a separate OpenJPH 0.27.0 corpus for the +owned batch API. It covers native signed and unsigned Gray/RGB sample types, +reversible and irreversible transforms, raw Part 15 and JPH input, and odd +multi-tile geometry. Its reproduction commands, decoded raw-oracle format, and +OpenJPH license are documented in `openjph_batch/README.md`. diff --git a/crates/j2k-test-support/fixtures/htj2k/openhtj2k_hifi_ht1_02.j2k b/crates/j2k-test-support/fixtures/htj2k/openhtj2k_hifi_ht1_02.j2k new file mode 100644 index 0000000000000000000000000000000000000000..61ced26eac84e240e28bc14e38928581b0601c01 GIT binary patch literal 30665 zcmXtfcUV(T(C$eggb*MJ0YVR<_aY!o4WWh5Lg-bhRFxtKh)O7-W9YpX0THPRDoXDn zASfUx9aL1XV*7dhzI*SyXU>_O-Dl6U&$EBb?7aKe>aR7ROa%T(-2c=6vHwTn{-6Gj z4FC{!5a|Ctf2jZ*0RMAI{iOliATR&|!Jtqga5?3F<*zFsOQ`dzJNXZovXlRS$us>2 z)r^#XC`%Iihs@Y-|4{tb2S5WLGtVGTp9sZBEu|Ams{dTB|6Kpt=AO~MVTz^~&Rx{O z2Ve9%t9HfEH~5mLqM=N9XoP2&uaBZ{SlGodS>^xhHka{=zT_J&dsPAd|4js5@r0_9 zwgMjiZ(?706?Md4CV=>##Y8Rxpuf5Oz>eq3TClK}*$f!e1t9;m^;3_Z+|HN=(YpXx zKWGyCZE&C&80Z4n{`HB)fhG+ARvEaD&aV@>Ky4lXMXhAhr9lSj0NkB&^&H@x_rC){ zagaP1z^N6N`W-O`c;x{|Yju7DU`QQ+HWaM@|9%DO2aVwXtt|kOg|D-%0mT1}S>@pX z{>{X}+D-;YT>#Jzx@!P%2DAKADEf|j+upW{0pX=7q_CD)MIiuirM>)oKi|o{ zuHQqBb_Ceo`DAbsh}7zb6mkMWv;XuVkxwjP(pNR%)ybiG-2wmxf;ioI%0^;^Y+e10 zoLV790>Bj=^E6H4>_@_n|HB@9p+o{Qb^WZTv*Q(V4l-l zTX&d?2#JFOhA&Qlnw-D4@S-k#VcnY8vQIKs>mBM-mWpQ;?Q95W_M4sDWK+!Y7U%vk z9`ifEU`kjgQU6&%i-s<;uJic&xKuLvkQ3{_{U>CLDh}--LS+C@YbQuY81;Lk`h1Nd zq$w}ad|0?~G?)anWAR>MaTQ5B;K&hpo6V!uA_J3)tMvS+T3eH9$*z_t+N_XVtA%0rsjAiIDPsri^?KTre%erO3SC1y?<> zUy>~aqW_NF&GRn@HtRmsJcNVS)CCMW_yj@;#Qj?^yem;hf1~a~HV7o$x23rj9n8h3 z(s#>}F_ebe5cefEREC@4=Mj)QUM`q&6?O3}OQeMO+pV4g_SbVh_qs-9Gl_tT@7zP; z52b3(-74AtYRE5*)j6phxY`(Gyku;Ed9_ReDQPA)~wn!Pz z?)BZ%c6mkfBl$f;n*Qag$2GJ;Aw%9ZXUxfIv$BS+Yj$?c@?YkXCswmpGe**%ngYhX zLm|D*g_y0yHVq${Z4$kqcQTQp=u$n8(bEc_Lxd#D2n0+na<-=)6bYror52Ghocy5_R0M-(-AmnQDgN6chm8Y{lgD`(NqQgYxfW3&Z#(64A_huT;^kmT``Qfo{^Ts z&0)xcPlcsxhB$mZgmK&vGh}=cm=InZf29wE3`~;q+4gd5#J0wNh|H8}GAT{=I@ z1O?2u1IkgYX|-omsEBR%LhmMv6JNhoX`M>Ry{!kNE=u0R$7tI$D=iGDl%KqZm$r$F zZe6Y0{dNBXtMKU9Ay8zhoGp4945Ic3y87rTT6>2zp1~lJY0EJ=!4vyP+8bm5xT9H zJr${~!|?Gx(VDr$uSh}A@3y~Q`d--#wmDw+xX4vAFSDtF-GW}2Sh&rM&b0Jz;TygwDtKHzrlP#QQR!DUL+hDXliH`x(N?dnwRUI$i!&45N)!huh}{zh$+ssD@}Z z^z^cCqgr?Pof_C>`!>;No4kWeXc^twxkB0oXXrmTkU*{r(vHyrYAkzRkOW=xSA_f@ zLR7NMG?iElY*6m=vnu;wL)+6Q-wz=`H)zFgA5-d@ceV1Lb#I~;TveM3Nn<}`5S^2& z7s{L0v%EwmIv3&xO+Ph(3L*BzFl5I?595v;L8X=B)0f8c(?0OLzB_;moc($JRfnUr zbwBIrNz}jB6sjNga~qXF@%&-~HcCkh$~_7-hfebW0`1)8K3XvSMv;KC16FWua@8q^ zdA+hH594QX{#PrU$~lxgKF5L9`YM*_IDMu0AQIOLZ1qc(ZYj z^V%|T9yV@`bkuP*yFpDJ6UrLjCe__m-mbzuaTl(@&g z?UZIY@W|R zJlyMBveQlP?JkHSv{n5_FFS%_!y4Qdfx2~qI>6qM`r`Sz}u6lV^ z%jSN#9Y>54I8p3v84rP*lCCWKr=E)k0g$-*eeC=LO350?9}b0q4a4{an8X~r|HKI6 z#*B!4X|$$Z9!Y|y=t(p8T9Qd}B*a3{S#JY?Pu{A?5(zBm(4rhSyTQQ>nLxB(6G*#R zNk7I2QR3dqrKAJ*$`5hv360Xfm*0n==Bwt>u`fy?lbvkAT?CXtAYc7-J}cI#>#T~? zm{xM>4V3{IT?e)ZKmt>VFBlg&PAUf ziDKS}hw1T3u_k57oO}@SfX*)vHNz2%z!L@`HSk+fBwFSz+I9Rv%A zp+H=E`x5OSJk6N7w2V!#to^pUNyqP58RFV~fCjD>%HjE^e*n~zGJnU-)M_#^+O}0T z2>j8l(vRl4!&ghZAdsJJoFucW9N?P8e!q_vpM7NGVVd|B%(w^XTrQCRw=sRBfRhU# z6zEYF=OJJFgE_f`t*jOmDtsh!0Hr7%!K1#r6^HAL$YX}}J0Z3BSKiZuB7H(BmvTQj zTx9@cU`X3Pc;is+~`+VZPvi+UXp7zQIf~_xbVSN zLG=~+x9@U0LEHL)>GhWEnPnOkzE5gs`jUfA<S z=O6nCxoamb$BnR-K@yYnzAT&xM24zVp$xpqRpq^sf#_*RmrECep;tjlvaOoRcPu{K2$bs&nH!9>962t)=PwgzkSC11&yC?heMVpKU3>n>^(>#DiV3B5q1{tLfYhY?)A9 zhX`FfZcup|x35>i zz_GFH4PIax9w&%(4zdqGxNI9#{@PcYpi|VBMD@*Q_dM_U8rn3Ad`Ho@U*vQa%v%Pj z++ww47eDLSF|rwh))U46VI-a^0yebuCq`Z z9?GnJRD_V$3JR4uZ+ra&JtbC?mlxs(DQGXU7D9`Q)n{iY3B%p;iWqo&cs<1C{Dg@B zygts-43N0LkhREhTT0x8GMhNL?$qYsu2Vcd5oy8g*4-5Fya?|a$i1|v)0h_krtC+9#}x9sRMCBS zr-e@!xS-~-HOa2@L;fyxc)p5FP+kDTIojkp6e1`;*wH}>5x$Yj%MO!4mwy6p3(Le{ zZUk2B-R+66E5f9>-~tWr7Q3oQa=oRs^&{MG=6OE5FK)T-^+c{i zt{4ZJ%7((+%0MRTA(J$Oh*x6g%T7WdIr1v)<(i>M-PuL5$)mvdD}i<1?xm^&sb-7;Yd64wH3Ndt;m_vD#w{oni055X936X&?`u zy1Z<&4J=RHMKi|X54n(ADQz?~@wq&;U{95w_Ul$1KSCurVW2#HYnfAE^9+=fKa7q$ zG``>L;aUgq5SQKD3v5*quo3o(+`-iL)rUR)>|z|GuEeVR5WF#FA#*bW#pe@6ZG5WF z;4F4rAf}Mh*%^^D!4lHqNhIYdL2JZS@7D7yF8hbEDjPfna&0(BJ1xbYO1^#{rJThVYeKld z)=EP7eP8mYKM6wu5LBRRYt9_zU7|vrd$(b#hBqW_kSDB~-NDt=6s%np!3kP|5&G<% zgx%VDWVsrfM0k7%1;IJGCYuE_&NFtNxpDY)g22FIYY~}nnZuAzbxaF3^L6A~j~38# zpm#};xmSM?DWJ_NH{wv6AE@sG4dVFVO&Z>FYXn#Y2{YVVx(KyFpI2lf(CJKm-KkJ( zX&UPT)?^S692vzECPVV@hYamT^lsa&_8fT3JTV>U-jGdTv(cV-Df>*gO-p+_E;hiD zS~L`-ihqFO5b|DRF_kK9Fz0PNON!Lctt-WMU`lUP$<%0VPyGo})f0`6GK3~lWYp_? z@CjNn{y~2^B0G5PtD&{!p*}3WZg2(6C|6=-;ILy$z(r(@T4|+_#l19E^>uz$t7Orn z@TXlV>B4SUt4qN{z5-KC@5&4tVvYkQ4dGv)Ky|w*_EtQ`s=uBx2Clg^l7cYBrt)5U zEfMXSpx(42{0Yj6xrhd{r+Q8&3fAm7uuwI7DG6aF{P2Ppq4#IS8q-6F1gpG4p4!N! zn1oK}ceRX1_qfXR;e_xPDE+B0?ZS zfiSIQyJC*CH$jn8LNkMG&$S9Vp98|AEO~duM^^%4!d~2e_fD(p#sQJEAYm4#My@=A zXWHw!SYNdj_-Ogy<*SiN^1z$l%I>*2jtUI0z&sB#8LhEW9X#L3rD!w2>;7%3rQx%* zCqYL(GtE!EPo!IRUg9(~BTpUQmKN&uvdl)dM`PRC+t?{lOF+^@Th~>SNW8WHNc00* z3we$TcBj7NoDVW2bGi*e@iqHta0{<2l?4Ml6QSxViOe^p_ZeYMwc1YSK6O~g@Z&u2 zKd&>rDP*oVJNg|gYNywkg5PX_gkN7dpY-h&e{MNn^t!`nx?pAr=U#39FFP^-Db#$e zw@oaGU(e!k-M&;c9b)ByyY!Gtd008K$vsQAJ2&Tuh|Cb=F0ajeLdij30;G9(Rlw7Y z%$ewtN7oQPOD{ICvpdcdL^&B@ORL5O1U=9A$L9_3;PdVtVlO4lkNbtVW+on@T5mof zUu>OM^{4DHEpZaMvii8djP&1^PI1*HA;&YIiSW`ZK2b<4# zLMD0i>DjyR3DMR-!NybCrt+a#@Fea`(gvcLMWJozu`R%G4R}gm3Xe#CV?Oy>v0S8! zHF|O3cLL7jj8NE1?4ei>KiV~cy#G7iUOy4R9@40OpV)y>vcE;C$G|U@C&_NjNTXFj ztr$J9O)d*3*@z1X1M3-Qq;u3J%m()hej0dw%{O*`%|`?Qu0AfEV4 zCy^Q1XCjf$I|Nn^cXDx5Z?_+2+1$bG=l?-iTXTvNF;%D%`Jatdyu#BrF+bnth?=Rj z95T{4ezLI(NTZytj)C@ml_lJQs^5ghoq8@J)+uy%ChH_A`oReNi#k^yhPAFJWt_3| zQXue?e3z#odP*KA-h6{MZx2xMt^m+ZBR<${?K}PtL6VT;GF`RSMTdYg`&V)(5 zexam)jiVSqhgSsWws00eW^4jIJx*iw<7Cairj2dSS=8s&=j?k^0 zw&Q*m>1vevIr~=oV}bOM6Mt%&d`*?y{@94V{*silU)?d|CGE&im~0lfb$m`gvLdEe z1ZDv7Yu7nGxcvez`)qk)hidQic!;flJSdVor`Y0Kp|2m4-y--_$>*T)UwEtpWkRl- z>k9#|=gG#;&i^J2$0g)N=gQq>Ira1b-hq$Qi6}5xLJ}9^iU#1MT{#=J``&E#!q0O* z8%;608yBL-F8LGl^1Yn~9}Ld&oT^E43^v?6BgUnly`H*&&lrc?Bxmt=6%&c5Xq6+c z9M&AyEdKQFf-4s@;=z$z=9=7*sdm1#a~lezp0KMzX(eYf^gX;9(h$_RBpKs9PZ@5i zxG(!S4%~ezE!XE)_=Dl(L7C_9bLl1A=6aLs+MN-7_)LaON|N>nl_CbL zuHIZYdm;_K-2!IZC}Pu8VI6NXij**$6J$$})b}X9HHJ>;C;legor(VS>cgbT1>=-% zyc|h8Inj$VvfE}9D^q>bNfzzmk$^xE5{Fyj+=aRNM+7O6cB-&Y21%GEBG=v{!NNwa zVOeBbUr{@|sa4IraU`Kz+HXo91_{^#O@D?vuZ&}>Io(*(KkmM~>}sUq(;d=RtseLb z+uVXZJ$VJ|zawt~H1Y??(UcVDOd_uoD8tzvQ|yi&m<$j^%}<>K$XCDSD&M+ttw}m9 zw=LW7Nkn|ENbnhDr%NXd&!-KGlHEA3vGfEAW*iIq$};?W&F6P3-!DC`7M7G$rrQ6^ zKg(XVxqnqeQcPY!UM?D+f9!8kQsTJEcSdOYzR99MOYecSV~eqdWo9a+{hV>7oiI&| z^LMpgL;F|lOz^8RT_r(m(Wh=lr6U;Uy+#zj-uWDtK$u$nX^xOW-tQt*7(-6#e?9s0zuRp!llGm0ISKzIjiN)|CQ!6Mxb8e8 zju8eyG03BJ3krttE;%-33EgOrS2~B4N~j1#OMA(jS{c&8PY$z* zi*``kBzP(-{XqId5)O7o*`&*o>>kecHU`C2A~rW{sg9XO+I8HZ?<8UUml|Dv@eURn zL$G{JQDB`kTE0;&XfPTK)PeNJVp+jZ&_(Fi2SNWf{5PX_d9u>CvMH88LgP3CudJ?y?dd)G8U)umL>2<9~CElRTLn z-F58;(zw`m$LPZt2i;CorZzk{fI10QCkU#n8qJ3P%lg$yB`+FZz5r-s@ znp0p4`4Z=ChJYobr}H>UXZ@^IpQj}+J)fR+>7A13m6lIa%&i&QOT0O?H;cPj-EOaC zn2PZzB*`}4zFb@j5AgK6YrwkU%LeLpyg2w)K?S9-a3kbYRe{&y(k%Qbb7x(mz zzL*jRE5NCW@#EzT9#M0;AeP#cxAwfm!?tT&0QT9d1XXh4g}u}`AJ7%vg=fJF< zLh!Wg)*f?&(HR>-RS+{L?HgDO#j+2=-#0s9g+>Z~6OLTDEl+p#pJfuElYk}Jipd6V z2kPAOsU5h8O25$~AE`KYb;Q(x6*zN#pE3p2kb!Qz^kw^m|k`!vL2MLB62S-rU3+&5|6&E5BB?8YBg zzEhtYf2NVw&a%DW8F@qd8_E@i>RtFV3O@LmyI1o=ENFKReRxC6ksuIE^#5dV;fajA zrp#{G*Dnr>?N{DXDD_#)<=bWxq{>Lm!W*&qVdtQm>JDycmy_~k=1t9Bn<>lBrrNzG z+N6uR#hw@T)3wNXWTRJ^-f`*^B|682uk`6&nO&NUp^HWH{G1ad=glD5d2@C2Q(9V1 z73bR^-2o*M52wWF(C%OT2Xc(WyZR-Js{z3`jXb*8r>oC$miKe~(ld<{{FYY;oP7Jn z6$`NGva#M#>4D<|Z|&De2hvTvBi7on<6SRDIkPn~zw)99EPb{H&%6+1kN&#M{zDA| z4eDVH4ud)VDVBAx9^@55nHw^Ig6Y5kx@L|qK{^_H+LPGHHh2|_{OxatB{L!!c1|WC zm&bvYLeVu<)fzhx(lxPWWCoj{nqo|jTEn;5yR(>C;MDL{>fFrQ;XuTrGw@ZwmBTC)mQj&-F1EwX-{dP0VD_=$zgWzkh4YQzKbbztg>KjWUYDBp1V4MIBk*Z^ z!6ww`b=Os^6Z@gxpS=z}OI7;dH~*%4Xs*^!&X+68L zm9OX~chyVZ|H+L1SEY~cJr^r}J~+79&4>AE67cup_07kwt#4_&uO|e&UjKNz*ZBC3 z+4mPsGufTt-SwsS@AGjjc|Q01eQf3&BzpD6wZ8w#Twm$D5O7vB&(}|Ly(S`aRwyXYo0j zW~9kE+osKX3|hm?hIx42$OtYP@2;nOwWDcw(;LTXPK1NrGW9u)bCaa^rmJq14(u$6 zZ92X(i{1b08IYv#FAnM&+I_cF^T}Wu=Tzqk?ZpNXkeyQqYcqj~kRzd16}(R`04Q*b z;R;bh9CNJCy8Ie90*yFP#dQO8Sc-&6(KDocd&S3&&HRQ3P``|7Msf!Wp904Jc`ew&m(19f1=(QxXI$$&2cl<8`4Si;`2~(VUo9lMDaT zt1tQeFsjw1wONp}x$MKA$B#>|g;MRl97jvno!hMBSuR}b>C_E4Yxbd5Z@GfO=k6nV zHtFg0%X`0eFFyB^yD67?;pWg_%cDp45~WSvO|I?#jQ`=#Fdk-h`Sh!ob3s?G7#z+& zyJh?9)!CJxse!N~jsK>5u0?#G5TBBA``n={E8CS^YRnXyd1qoH^D>7ByJ8f`~~CbjM#cS=Ewo@O`ah9zipHP==Y+tDWS46N64!ke1e$XygrF*ue`c8f)@}$n7 z4vplcJ!4BKH~Is_XFhD6;D<@}8GyPiW4lgG^rj2+3;y?e9<&a|p&{yl%pw~6t(EOhd%B0oe z{nv0(d1ZbD5;rfY6BSDaE${`OSY&W;zOlnqX>0WcY@CrbT1(6)+uq^S)q^cL1DI`1 zUsaWq+4s524X5$}q<|O#EU1N@8+&49WHAzV);46xjD>7aT$aCR-fWolK~&bl72+Xb zXyRPH(?>wA6d@mcZSbj+=;R8ns!Mmo6ihuCV5C*|uYZ=5w9&4LRB>9kA$NGf%^By` zcR%avqi)zn{;dc}Z-qYeRR7rKSDgJLu7bL`C#rSW7AFPA3*jC<3SrNkR5S|WX-bJ( zKf%HG`g{|ImbA?9>v?cc@R|pfH}8#j`LA$x3kGha#NebO0&!W;CO`NU3@M6fTk~*p z8Wg{Kvthcet3H__j(GopD3=5K{3T85?2qt za_i(6dGM{eQJUkiYF0E4a*wxR8Q0)_S+ug}uf3%4ka#TId5E_FrBSrv5r?$1T$C9b&L8ZNlvyJu7TU|pSe zrKg0N%m#uUA{SMgF*?uLLf>Ii=$pKZ;-F=-;ikK*FrnC&cyF-CIL}#)PlD)yR1kbN z|4m_Y>NU*r2}x2-s#XZM47|hXADKtOU%===k1YB12i^Gr?%%|z@6#FEHC7&jax49Q z7=y)}$hn9x86{b;rDw`o7yk{Mb&V>oUU%DV(EIJ%0S_0PcRchLZ4O$I9Aky)&^)dAq3Z>Ftn=!yO4?y83Gnz@ZVR&ZU5=Ql z`P7o`sblpq#t+qh-7&}$ZSc^{7#OSm%o4JY=h=XW_%{MaN?;}Z{4HHJdMHw=2y^{kJ@#|I{E4Q;YK`?esL7^|}E!vo1> zK9Sk~92aj~a3fK`MdOj_KLVhpVewt2%LlpJ*|8}eo9osvB+Ul3>Z^Ihz~IIvEB6;j zlYui2=BXTl#+f8!P}&9zsF-4G4f~Plopj5~&~Z!eA|lzO)Gu0lI?u zXqvw!0teIJN`<^o6{FdzaZdnU=q7ew-(91r&{ z3Q*jnP-)Jz4=>dHGedj?WXjY@dli4F@61Dbz(2d>+-S$n_Qb;rwmQw3&+ycLxzLcx zri!D=_VUrr3QuBgt!RZboqWq5REN-$ShXmqu*AO1*$v=zzR*4~O(QzQ9w@LRnmmt+ zXkzn1)n;7qtPR$m;)j0VqDA4_;woZa1jN4O&zy&_=t5smPfCC)byMnUQ{R55`|*q} zhc}W`d`)=2u>K27UbfSk9T%-j2`N{$%+N!2C*%!=L&}2s`Rx6-^&6SSZQo6j81z=Gz2W2@VYyB+Q^^W+cR3tU9=0) z*7>sS*~N$3Jeky=Plv6zaO)f+GFGEhQ-yTtA*=jgD<2i}E}vn($Twz}T(}>n^e}=! zrO)|;$RAWcnyzdW)7o(nag%DxH9^x-(|Xc88=%cZoGpRGtB|q!G$Iuv)i#r8Pbj`L)qz`7HjPb?-5BEbh+MVg z`B)_VizXo60Fsr|zB2aoB=e5mDLGu>BR#5; zMV-Q1R2)`-EZ)<`Y8_^2a67$nRm|u-Exc#$d{Hew_2ZH3OI}VTeR+A^)fjX9rdOYb za~tQ?qD{O6$G56_b{hY5ae4s7DhO1gOxVD!Rk=`DLFO>ZwMNc*D>&|rkkC)8OwrK0 z6cB5qFLhm4$So)hlU=P}X)3<2MXODdl&iXQ04^Mq zWP#O}N6%g3ya{!=_8IQySv^=gn=)>4=lNXin7D`Z&9|q0zfMEyNap=vt|3D-dV74G z_fnI;WI?t2YWtrl)v`7$j<67zH(uqfujcZhHjbWlCRqIFUc zvi^^Tjb*(JatGwEk+ew~c=$Zmk)Sbc;b@2W9Sp4?--!G(cxM-%e-L-N4aPdF zGpxiGV|3Rf?H0rcY5}43F0%O@`BCu%4nXRHVsTt-piSx~WCQP?I`P2Kbqy&Qc+EoI zLopk*&TaNB*21PciCE%t_bk@f$0Pn&h~*xRQi$jPt4*H#m}7y9LToVV`!jYQ!`l+)x;G z*6S=6X&8UP`Fm{{?RiVp?Lk|vePqP&fB%T-FR+w4pbqdo=_ycu41l!MqpIljtczdR zb271g&e?P-5~v%ynZqe(@u;nYMNdjgwMp0qDWnb!m741Cz`Kj@yB(2M<#<#O zGM-A*!A4qd(}NsYSHb?@P(c#Q8Oo#`X;93#>Pupag&bQKDWTn?ZK;i6ECk`$@uke( z2~3*gJTh|C?2ZS61CaIMoXN@CkFNRwMA7&Kxlu@_!NXuf=_F;2B=cxHC8<^)BZ2G? zG5L6w>^oP=#R4dnVsI*Y+EAn_w9&fVQx{+#3XxIgn-lDSQQHe{b#U8IDnl+1a@sNQ zcf+Ln=}Df~5;>oS>zUucou8VQyx>C~3DHcJjrfxVS{o1JscBTkE{FPsiO?@kdS!uy znv-(38JlH0u60dr25~7nm}*Lf0C1GOlTbwu9h)KI@6%+( z@+y;#eaQ8)k{I&7!-SnUB9)eeAroJ)fITGYhXb)z1a%^G;;jbNp4J}q70{qaW`=#O zDgI2SU++32NrO=O21fO?J0G<2@Wk4x_UI-}9yl4b0D1^nQ+T5Uvgs@UA!Kh=8t+o< zknLQOvYv33L$?hOrFFc?m9O|Cf_-U&CO-XIBarTQ>qRFb)08~nrE#U};Wcqq(kSVI z#ia3gs+^sHGXT#EmHIWCjm)z1-^Od)&|;tbL4K9w1kddL#F9*e1BvljK``gNwAyAGPp8f3?>vED#U zr?`G}GAP;ANtkA2!i1odQT(%ep2xh-^5UM!zcI8QlhJqb9Q`8V5S6dBiRt4NN9MplVF6xYO z3zDEIaYL&{(_qi-MSB+BklnQV*QYzL$t z`mAtDjV4@0*b*=dM&C&USwn7IcH|g zVl3M0m#mP?<`>=2&rFaZLA{E)l(+G?pfO=2(B-D<7nKVXpe60lN9*I8%r9uNDxe|)f_%_P2x--Y}^*kna zaO;_qUpv2|l`>f!)B!&bLvp?@^5KLeyljcrev;ADWY0!T6C08fzONy^f$Ntb-rfHZ zx2Akj&oW0^yqTQB&F{WU*8xI^631VCmOfk+zh>-d?UzK|?_){UiD;tun@tcq6SDi_^bV zR2;*20A4jggTZ)HTHv}7q=|brLoGxKEgvwz#MwyQIhW{yuoLW^^uDN`9_;G8>Yu4zdRZ?9|6lmCr_5 zR!L<|jsd%{_-UTc1my3djB-IHV#K=NVJN{+)!pw0XYuOZ6LqfC!t4*G0`I0@O?vz9 zq0j9qSqUoyJ`$hR)KTCKO-o6e(qf|3f~&9Sj_E^s zNjyc84Jt$ES73W}RrH)IQ>$|^R`q6bmx&M6a4Ai5Ma7gEWp%Q9_Z4sJsIO+t^>aPN zN0@ggqlxi&J8b}hEw^DKav+68S7iM&oV2gX%E;Vj?wH=^a7BreZk(DA`D7^mA3T|A zuHi@5{4Rpn%NuhtB#*4y;dQ&bGJO1`Xl-o4HV@7dvL(-{5xK|&hO8Xl0|Oyj4jJyx z_MS|g#Kbb#CDn7oEWvjq&2xP2ei%NEm+CNw0q~Sds1d`6X|UZO4^j;It#&=rgQh+# zIa!N#JaRV9LiQq4A1SyPV*+FXF{Mo0&ggT8R?5U@Lyb_#G2Y#BgNgp~i z9$oE{Hl6S!Py&lud;HawdQa=j4<=j@#;W(JY(ra*9Oc`|o+AOgqpsX4wl~6|?F6?m z@4GA>R>|TjRS5@!&#`eV#zDwsP~B#8$8ZW^vq0P71{$i_StBP|$y;P>0~k>Y>qk-; z-Birv2@%6<_47lM3l)TTXw{*CQYJw^IQr5Giz zUoee|zQ;!&%_TobMA13&_Kpx&9Lhkw;NIrtiAr3AV692n`~41E)umlDm*{hAB^o&H zo07ZDU}O^cTOHh($k~1xL=TOFIrhWl|7@Imc%~ZwPS-0l5Lk}t*I;U5994kIpZi9YBOt~i3A~S;XESGHAzzM|7 zXMvm3$XgVw+9cQzJj4YlZ7y@loIWeGUN2~yse2QISArlBIX}x#0Klp!kozJr1YBw> ziP_2)m9^jS4SO1vmqz`Bv4Ye!S-Vk_?0myGL_mI9s^Fd*Pln_RiL^z9SyhUriZW_jZpU*#_~+rt zrT%SZsuMTsQpl=?smvSeDv!N?bz=Y@Qu;`pv8_`9hS4V+Bcc>&J}{z*23tnT8@EB(3o84%c;DF}-5@|ZyIwSEiRbH_3Tto88zMhR^vI%z-c%HAmfEar2J ziS-jJ{Z(Z$fr1QUYJ;nsyDTAdq2l5C*oO9Abf8FMXcn|A49ViMnnJuuUx+YBotwn} zFkP_=!I?-=zSJ`l;;PBhMkizr2;h;66PLvz?|9=F1>yqed`XcV>sg+{9^PvOI(;F3 z@~Q#)&0Df>G0mHjD^L!{d;HkfLt6}FUaIzM1sD4J-|;vH-|8&es?D(qaz5Pf6h2n^3B0)Bls??E% zI_r6vE^it&M)%=W9iD^ZlmmyYhW64@`42|1Y0@UA);IY9ZpXX3* z>k$Xs?7T&Uwi?U5I6%eF2#ZgSs0v zT}*eHgbNPF^swJpCp!I~07W;t$kqdt{->Z&O7%{V@EVaM z6gC7!N@P94yzm{hsukj*q+T3KO2PGF873it5RO`{E%SD+MB4#GM(vJnY+)vZjlo5DV z3|xmYDSibN`!~Hc0Yz65LoL_8=MBuIi0fy4U>BQmfaoXj7j9Lp)D}_$+BRfhg zMJi)QSd`yzVdSJ|9%%t#rb#oq>Zb5C^eX&fIJ_Rn)X=Y_sl@}TfvN5*2AFn@=6+BV zKGE%t1ugABBn+8P2q6)qxx!upltiFeH}N!GnIOKyAzVvURYoFh6r#b$ARsgjg!m@8 zYqbQ35DQ>1o8kgts1A^Vj}M#-dK1wm0GNy<4>KbGo?s~|C#Z^cHzp`@1Un)Q?_RhbYboprao!H8B=_ zC|I^058~gW@T6&-fG*V%4M21HV=~cZla`A$DjU-j`$#;2R-}uj1yNWGk_}M=&(v#$ zO}S&iwpckWgz1R@W=4P{bdJz9JlKRlYx0bwwT?RqXA_EOMYPq7t4t-XU2uV>%!UU1 zLIWY$)N8tdf%ujwNaBiy_$Hi6NFs%4yQ%@G?Q{d;FNBalFyqwbq8UVuktuF!((qJ< z;1EcZvK*&v5^4mXP1Eq9y?9T-lhmP8)S?m^jtE&N_-tt{hi>g46cZL8VA76~A)^hb zka#xGj*AIClSI~T+2Ur0PKPVqc1u}=`4`y_CqxS-1xFQ5L1FOjISBOte}R81ITz9~ z%%V+)LFcsMP~ZSnV4WxrC^U~kKRy?58isWlJ+AL0fNcWO=(Ix1EZ8Qld05m+hbxyU>s%7G{rh!=_UYF0`A0qF$>BEKsDrqV4>Nc4EvH3lQn zDpY(HS#dUIuIC?I5#mnFpYtrm~?91-ZmHIO8L85E$^iHHKYYgB|8{S1ao3Gfmh9$RgKI`vE`a_!3SLn9323k$)B;LFmy z`$|h(EX1wC_N(t{JrR>U_sK{ZkdG4CBZElk!6p=AKmp$)Y4O~G9w($YMQ=r+@e6`L zvux6F^OSNZ7i)kAFxV+%Vzzu+aq$w|Fy<7CfM@Cf2Q3i>dwJdpw1+(U3YoDy3R0LB z&Wb{$nTnwr6@!GKBsB*ahxQx?iasuzS;Ip@+Po3!h$o_XGbEt`U7KG>*SvH3qv%=t z0l-6f6a_)8m8z0et`3=u5D%k9hM{0Z$C|lQkU@m-n<8r99Gi2HAiO*p9nxMoRmLI1 z7crV_Vnqz#HW#miE;3iJS-qrI4d}eHEfWS3hSNQ^X+Nl5uP4G4A{7W)&O?wwHDITT zCkaw{*9&=%2PxX9LIRnYv-F==W=yl2gtd%JyW}sM2eKPVLeQ$QO`3)C9Hwwy5tC4> z%?)aCj>XGO-8VU|A9w)610Yk-_cBQ5 zZ-X$A+2xQ&QeZccM$B8909-AMXh6NS9w6vi(ANr85p(37)k_Q!;7Vc+2I>1GxJpGl zm55l^+-t*OJUpvDmhB`N21DDdi;;}B2m&B;hJZa5HC0#xIvlL7viP)o6bHowPl0v0 zHMcLZ0*i-eeNfE2ljg4C^Fzuw?=&rj{=_y6sM*4{LO_EgK{;8>xU$BBY`&c8jY&WN z*>iz0HFn-6SdiW)G3eMzfN+5QG@4gKC#Z9Ecb7Ri z6C@Or>Md6D#S#b3{R*L(q7%h>QSkXJK;+w_YI~ zZRE4LqvkqH5>^xj@W$IL23dz&G{9C9;z-xAE2N-x=Q0JI6;4D%+#*5OWDY3gaD#WaJQK(c~EukV}rj6>cHtQ}rWY z$mu3jpIVogH*K+Gh}!C?ezt{`GHaH(Z%z6qw~-%M7Cnyr|9uXST*UtfiXt0+cT#gdXsB z4D$rJ3k4MjSoC$*Ujq$X$I1i%5i#c~5X5zOj3^eNbepy1^t1SZW#3%KNm!&bfanO$ z%WBCAvuO#O=U1smrAoyJIyonZWg^J@OMxtUd!XvLAQ+GFt0RdNDh~kF&LZf^)=h|sC@xCp61=P)>hxND$F>$R z)Gruv1_~gFy;CFIjdk5ClZm+pp$s_6&C-&J2TrS8-oG=c=VBhR6g*(0vUv%G@ze@} z=Oyyu5P>jo09icDYEb9ntV9PaXuM-LAyD5SNg))}DS1$ZTo*>_Ix#4RTd&+I8A-sN zPs;mv&|*$f2Y?D*zCyu3>z3+YK&x1>Pp?Y@FCmPqadAL-;KJp38Q~DAD-@vx^~0!@ z3e=-As(L{2M}e*=4obcyRjGRr3Y>JKFV4ww!OgK!gdkMmqlOrWA~mF-XuEcwkiTN- zLiiLl5C(w9Dx|Euz80@JI8o99xbCY{1s7j~gdDEm1qdgP%Suw@Vi1>!T0+a=J5w3s zk*+!BI{tEx0%VOEP7NV^`UPkh;1r{{%ZO&foTNhd|XGOOh2fl96yRQ?W->rBUWXxmvTTLWOyc z#RfuEQNt)DFdBC#z0k5*jOnRWSn1AGg!a<3p zp%OwRanv7^Gn=!{y^!8$^-EfI%f8vWj;vy+!~FOJzO)Zj){QAW)XLv7L1U7fwMhcO2?9NN-`G8;;0r_mh$zUTB8)c6rUO)qzpQSN&QX| zRj`D3W^AjLaW9crTp$mmkrymEQi*LGLNkK}0b?v`L^AF&L-jP>I&g14>CD2An4>%0NNr5AxuZB-jGf(-~(o zz#K~kV2tr`KrBZ=Xd3q^pb1!p;iXt?5>ZvnrJ)o`_@bQvp%);UMALZlm^=^=2qQ$M z#*tL0bfRTLpf4(3*tn^pqKMiKMp6?>GT$In4~9L_2JzcU_-P;kYC(%@fN~x$8bU@S zEKXE_88ic|wFxv+hZsy8a@c51(bY>#s3Mxm9|A^m*|L=ah%o4lNi#7lm7rW=1*;k% zv18t`0gwd|g0n&d#}wR0mJU$5%prtX2*wwo1~rzFomjalgp%pH!Zi{gYiSoyc~oH| zi9&*u5d?^XzzP{MjP9TOgJ_7xK(vH7+HfpHB9{=$Nis{(ndEwE=ZKJQIvT!0Nv0CMGTm-NS{2lNzpLg zpcw$ToUTfT6=eMvW)<()kPM_9L?P53tS}FIjif7?VMle6L5q(>^E{FYQx-ZrzzfB| zh#Y7&qjXr9Oyh<$9^`O9G(hGAf{34$dIGMq5LAt$3pZG4Qxswb;{^^qb0abc564fv zkYv6bZ;g`aI`mR!5KhCKF+d3xtT7773PAkPlzBEcQWgW*9o3+$6T}GQ)tM8S@Fv*| znyT^wFw++UD1nFGFrEU;q$?uI zS!ZC}H8EQeGo|EPiXp3r^m0;RrW$L4VjU zF^hHtZ{e7^p@&J3^~ zC{wDvuY{7V>s7g@6HS9ji&D+?B!cGfzccZZ0h}opfMmKA%aL{ElgHgq#Omu_$t>d5} z(kAJdabHb&$=Np|n(uGkts!QJywoXx(+(Ikz zBWwH}dL6w{D|Auc<_9>g>)aue4wCS$%aht{&QWV$)jyw>!pL-CwAFpT%~BFZUgmlxSs*Rc1E4 zW&7EKp(9S6mr2jmaN(T`UbB{&qrPw8&1k=Q`#h?hQWZx2J$vy;q_P*blVa0E2#Cv| zIKtYQ(W=kC7y1@N{YgD$rt66VpA}yv}w54q6>SjMQuXale{OhWZOYsChX7AP5 zVcP><_ZA`!+3ADosog%!jyvl27Yv)Kl5j(yFjlvrVw+kzjLaTWkwB(@-9be1(i-nL zlke!XZ)vGt{P3MKNo5NO$@8(bw}QG)Wcg;Rs!J$5^sPE`?{@j_PTTIv`GckPe%hN@ zb;*NwJ^Ej3{4S-cCVfk2#mI*6a~$`yH=nDA{~z=(`jpy0X+A+d@qu2ZF)%_=ax`9Zoe=>U>vvpEv+e$xkL&dolMAo(2L>(>$ zWcjFUyu*@62{e-oBWY|yVd-?5I9!;GR%RqQdWhC;!-#jnhAah<4KcfN2_0x@nY8gb zOd!WNo)1tYkbv5|T>BkyyEd%|ED8yvlpJPJ2h*;};wMPeM4UHXKmanpX#s)33Imrl z)?B2(3o|l+ucRf!5Md4%FAq`$5{@7uEGT4XNE~EgLf5ls{Wd^qP*vmFl$r*FI06i! z5UzNnNlOwL27DGCbu@1B8Hk-nK!XS_)=St#FgDspByuDQ#Lx`seNeDl3+bVRDby4y z2~!Ut29!}9OPDd|Jf}_u z*aaXWX5isWGL0cVT!2rHSOcONBcZk8D&SDbKxjG?5&%R+pn&GALy!W`-G#vrG?b1I z2A)Y+LAIJGM51m7Aizj~4K@uyWstBCr2u_oAoByTR)8R5b|ys>8wMB>NKvW`o`^X_ zG&BW*BLGIw6f4L?y#lhX01_`-c8F?U$sX6^(W}P5-4p`Srwl`R2)wdD<$?-Of`KL= zffE=~3^OA(K!ey43dq-?)Bs=*336Ki0p-MGRE{_f2u_cBs9*sb!N46#2p1|Ch_Fb5 zt^t-Pp;{XZF(p7CjPbUJ4ml+jRTZWNfem1xm!W*RM~+t#&2>_sX26UKRrb{E;OdZ_ zkPR(?6NcdpB(%dR*{Ap8#>7e%@k-u51*On;fLVhx^TDPH9wU;CnFD}uqJVJg6eDH$p|z)ZB%>Oli`0Z8132L&0GP=zgg2mngoG!!Bb zqJWkch9zLUU`-{(i(2=hE_I0(DsrxoFLcECkdp-vRM;*-GOO!38G$pHI`bH@#C|V< z4zQOf5pE#Dpo*Y$lM6M%xGaZy+IE`DR7?beVaUj(;fctA#0*PyMH@x|g~mCcLtk4& zRkgOpK!L>Y-~)+}qzDpV0_=pA0Ym~YHsB77VaO&f+mEN*$#tymnNl%{vh?6~M7d(g zF4xuuGB%xRaD++G6B;eVM>xSmKKvmpT*aLwplJxIF`17tTR5U4MlnEw6afPiEKUCi zgR-`#8%3JXSZqmTYUcs4z)Hjf2~r2=e1@TBSwu6<#2nbpdO-ljls_{r39WL>B#o+e zTtT_N0rRnW?`NgF3Safe9`_)nczR0m*K<53EYrKi9$nuHyZv`m!=G*~R#n9veX&{D zw-DBMQMst5h*xaS=S}PT*R=^IKm1o$`+FV|$`|*g+cqZDs(M!H=|1IvsH5Tl#m<=P zqfzxGN=*vIYj=P!%1bF!h6hd^0u>1^B<7c`8xVj(fR2oiYiZC zHkPy4m^N0LjK4v25=-2T#EQVRv)=caGD}Ut4YFuYdh9l}CmP+}4*hK3s`4vTmnxyK z$lj65b2xq{U0SYPVa;J$aboZ#S8e2p=nNw z-%f7b%HCM44nF1{*m5L!eRj+RUQcM$r;nvwEZ>ZEq*Z*f5#HEcX{y!US6;r#d2YXc*$P8n~w;@#Vm>F)E55yShW zeLo*g%yFXmlr#YC@~P`AnoGA$f5l8utWOd5{3x|;Wxl0rI5`4Gl~a?OwjW6hdeV9# zUeeH5&Ff2(nU6vjcyNwLA%5cT@bhy4mU(&sW3kXqKK6u(^+A2f6Ur^n%%X?Dl24@M zs?J2gjV;Abd1KwVQp)tG{C_%IGpb&LWh{AA#UQkj-Qq1TI*a^zH{xc)7mZSm!|;O% zG=pRiU=&993LS`vDCJ4BN=O8=DWFMIYPgn$*XKa^m58M5UV3Q0rc#ow`-8WGGB@{` z>R}}iO~_ZjJy7m_V2qBvh|{gMtF!Zi$RM!Az=}OW+>X#@rBqxdf|Tof)PWS8Y+r5) zED#+@!!cqOmK*r!xYP-PlwEr=CG5RBLT1-7VkO^_Ys~Yl@stvQ5E1ys&Aj?X< zBWi$-XqtWC4KE3uZqgkDgZoQ9z#4+pH`M19^M{3?0sUNVf|}u+osr)zNt+lGB8Z-i za3#*_eGMULNaW~}h1LEM2uyvlFfkZAfWEA#zNV8AiJF56ialBzBfl90m%vTWM_<-p zZ7V%JChP!_qjKr~5zHc<*1*hkN63PBK35baS_Vh^iO7$@hjrLVDG-EAk}?>qtfKN% zX2`@eWrYfes+tJBRnWZ)b_~F@LnYM7L=dQl0GS+Ph2qJ=(qW1>%6l3UyvWet0f6j1 zgBAJ-Tw;z=m0?x}q6}?vKDbT*Pzx-(AQL=5g|3%Cj56dH5T6aF7IzF4_^JppWIYw0 zc8pL>Km}wXOZ>0|WT$Flp1vL-E1*!27!x*o@_W6aD*^%iyep1SZ2~K*^GudLV|)oLa>4z)fwbL7vqU@6U4kg2*MRl0wi=B z1raq47y+Ox<`agfGAX+r)Bvarq97Rr0}1ijahuFv;y4KEMfJR^K)cQ>Kh$+)y67!6e}aaR9Sq0D{JvyR4rD4mcS(MN^8z8uS#x$fn+en-hW|R|F+B zor|UBHZ96mKY8ThAlIP8_84v<)(uc1z^SzlKn7Sr-V6kkaD5qopjkE>#1RHL2AnHk zmA9xN9>g3+&I(IXgMo2CJ9v#~698`jU?P%H0yqKTWSeDfl<}v5@?vsDSO_72y*e=H zK%j?GA}A?LJ^mQs478e}#;QC5HqSu2=>*LN)D+W*#KB5{r;(2MPDEfDY`}wik{Fa3 z8{qp80tAVQ?s7tK0fCCqiem_y-5rMRMCBO>1;W}@Cmke%LKx9OqY{o#VsDNx(as-(p_l%xesg=+NA3iyZY@ELHFXKaJnRo#NZC^?5&;76p@XRM2` zW&tK0sDS9L&`wG|G>Y7_aD-fw#rM2RZwkDl)+u=Y3Y`C zx25Yh-m(UhikBuWi4~nzcG*ju`f%puYI_M4Sl4aFpJuKth3R!&^Gguk*)O)#r8S*? zN!!Wsp*t$)b0bIBvNKerJ#l5zVryY}Ys1XH-a)QRZkXJ7?AO^l?TD+s>10Koi8?Ok z9YZen_pjCcLC_HP%S}K{R5St30m81Tkppr+vrv#RNwffH*pHnKs zC4El%fz5rWdM3|u(LCofOt7v&~Z+jXHHbxibuB`!`$eh~S>-(`RI zU0)8@+1#PM_2B2#b95`dGU&-Kv(eJXl#Es5sQ~!*r9refS zium1ci|%WZVzq6X+JNYgc!&3=F7#x^beOyO+m-Ld*F#CflYaT~ZFa8PzG-mHl{4_! zN9ex?zZSBwY@c%3jY;@$)v1}LU-Q}-q+JC#HB$qR^>YQPRtNFu@ zp9&Y11bYoN%W7A5Mwg_Q;?CV5P^qdE_iw&lwP*M4GUyK6yYy`HmdWP!uHnZ;f9AT| z*~Xvkrk`JxO7gl+{NAZ2{O?M4-O0tPv*OPGa!cW(ZSH57Em4P#wmeHMJ$q5Xb{3v5 zy5<`RJnQp+sZUwOt$EF~?t6V&VO=x&l1(=-#czEw^gQ@?hXb?lgFjvO`eEfy5p}Xb zZ%gj4%(qt88C1+g-8WuZ&su)SDk`rI)=F>xK6GLCVPMipHL<+iotGN3EfVGwb?p+K zUEMxno+5K}zxr4GZ`WGNshey#Hzk00E~@Sd`AKVwZOTcU99k<>jb-e#D}5W}iejl} zISSROA(cMc3(BNy(F47V!bSPSFo%)!P1%a>G*D^U-V1JB;iMGXD5;%@O|-5x70WQ)Kx4Z zsfhGL6S$RA_O$-tDOgF9=bmS!FI+cGiz|I{P^)j-H3#Y_;_-W@`%lMexv0D;AuQ0<;9VAL)6I-hs zeE7zqLJK*6cT!{&%wU_+$3Tllmb*L;RL2x=}~yx2NaiJ_<984JOLZXr-)EHry2mu21*f) zCW2Qh2M_D9yy!#?Qh;#{-Z$xDA&}mo5I&4G&6LVHU8A&l0qO8_h z7i#d(9VKFnk{U9yA|*tu%=PPrXM;><0e)u`1r~WJpd^jfXaGzuV-p^ktLUMwfP(}i zkCpqTP2L>|D#3W5X4<5d7FVEF#UW!*4q~`?mY`}nfa>u>pG$2xYiMm(X%H|2w$gb? zXl``U;`HIkh^u?=g1R_)pS7osbW=jiCCStsDZu9hv4^Z8An=MXMU4i6V-rK$E;zve zX*MPO28*M!EGXP6fX-QvsbXxPLx#r-%wW^?ok(7R}vu`83lsJg^+J>a(9;&zLqcSILf%;<|9# zH+4VCD6~im(PRk+apWpYxDIi%e`@FzaLI4@w2yP@3jdv!*5?apHZ8#vn6uSwRre^)yv5VZ?m?%J9Ui<-!wP+m2P9ng+n^ z^*-w~rixzu{^BYi9}x1#&~-sn;isf1esAgS|23(u-vx~(<{kp9>fe@pa?!WLQ`viV zwy4XMfBO?lS!|o*U-$c@>shok(qO61B8KNI^6zH9(J(`Xw5FM+byW0`M<}3S<4Jx& z&yY3a&%V$99GJaTcNh4)ThW+x@~u%V*^I7y^?%=E%d})EWxkd%kxc_{~34T}|yc!pu(3y?WXNBe(I|f6VT`_x)7IcB+~FzxUle zkmfEtZnV<#ZLz5`{f%dwraoomN3{4+f2X>iAI~_O9q(*}?w^yo$;NKw+|vAa%N{yU zylSg#&6VXdj1|%5Df{iIZPc-_q*dPptfuf8GXYakSE}gj^Pb)>dUtBw@-v(@@mIFI zEZs#YzC>as^J>N{ex0{iHl{2v0k~^mCef5#!l@qsL1jEo??9d)kqVU_Qv(#HwHFMS zOb=%UAscnGyCeR=7|IE{x$V_(vJd@92BdHy2sKu5Gnb!)kh(6V6TL7I*$hZnOBbL* zM83nNiS&m%X17NwkPygcfKiCXf=kr{2<-YTYG9e!WWA0`0)U_B01LlW9ERYNKJpyr z*tBrw?Tx^gjofI)(PYi#non57zI_}Rl<4D;7wV46*>WYmh+!naPL(%te!jl5Sq8>1Du5U(*>X&1Gfn^T`{5CpQDzxFUh+aTW zF!ca?clC@Myu}OxJc&u)$GGM7CAT~XzX*~ws56p~dndJwXc*5WKf2jYY%?mjD5=6B z>^ekMNh{o1=@!$h#kN+cbd)+1ELs&qMPTKw>>yWTZgWK#elbi*8drwmvl`75A8#YMR0CI@6-0F|@JUd1($Gm4l`azmQIK%vA8 zVXln(rn*c?`#x4;(xaPdSs`zR6~xGj#saMwDgaZPiV}=QkS2nQUO9?VPl@sjdV;2~ zNt2uc$Pp0^>!iu@*dvTXh-e#vlY%y?5d!R;MS_N2s~3g>r>}6{3|&>+avmKP9CH&X zKnya|6AC9bUvX_XGekf^AtmJmmTaYot2RsMXXK~~$HE9zBrn4S#s1)@0C@rgHq01t zpx~e+1RO0u5bb|mR4W+s}z;31I(Xud|XZy zNLX`zpO%>ZRw4T({S0o;yZ_Y~UMx=~(h=me_ct!$erncz;To+^DTX$y zW;ECQ?{-_84mpeG>&myMri7Aj@_jh$jVZ-+!nviNY|eJO;&bzT9uZ3=Ei$@A5_|r? z5YW?3YHu4!_JHg>^8L0de}cL!?X-^|f9QAl>QwTy+jnmxSMlz~|2spdXz-F~_EqWD z%qkf^x!Sn#>F=ZoMeZfxEED+-K5DnAWIOd=a}}hU(Pp=XzWeTPnYAkkEr9{G-HHk* z$;Q#@*KXgnpV(`2EiD(I@wI#bJ^tAY7Eyj6`g$w|!1n=Trmwu0!+m_0M+cAS|NodD6a4ryeV{b$|NS{#4!}Byy_4!f~$F zR^O0)Ey&fLlBkN3-2sZcQ9LlC5y@XwV7%nknoV08t?p`Ffp)S=>bB3EdfoHkQ)%0M zZi@<>c8;2T{aBR^cy9PIrk8X_o$8!BR)nwpq+g86NA+#REJ(?@uN56$ci=qOf8;9n za(T&@)0p-|0t*{q@Wu9!nWr&r*xO|1*M<@m>5&;14#6wIB zs>OyxAYvdT$aRe5z3Wl-RJ`I54eC07I6+a9KFCBElE&s_r{`-zgJ>Ytgj&&d#UDTm z79AR6mSINWgaF8S7i|PPhN};u90H^{UjSf*0N{Pu9hy^%L=pJ}MGTk1Wd+L-Au&5Z zo>8KXHCK&&+!SNTq~e_;DvGBl0$_$PAb1l_c^L?g_v;v11jWA$#lL|CEQ&pk{$x-j z$p}y@7lispD*`e_BAjdFBwUCfJaXu+tdFIi_i7GHIXDE7TIcDHFZK~q>RLKz#wcZj zXR^q;o^Ks90`L(9**$;-9Vvz)vcRZd5?Deo21axN!>iW0iZb2@GsyxtM5jh^YXT+- ztBEeTl(~p_*b1W&;{5>IBGA!`TR0#BFmW#JUjZFA4{ zVZoKi-4>IQ^Z7h~fd5fBBLk5GF&cX7c!VB2Q3*u=R_NWrvxMgIz*edc7$Ij5p?<*$ z#F4)`F=5Pnfr(B_k|?dl8IZLj!glsct&9$QXH!rE58^i#<^>4x^#@9>HTMEk%}+ zkXV@@Wm6PKRFpB?g^h74wTD>urX8<~jslgWj>XO8*I7ctw`=(_ixBY>^U^s(Oa;dB zFm~tYvRhRvYbKSJIgsZbZ$D%Z=Gmde#NdlvV_3}S!O7u6$txlX$RHWZFa<24qQYWvU827GqMpmyZ&4*y+&-b)>m6}DM zGzukm+cwZZnqDk;Y8;?4s2t07G7Jza@^Bmz^s$CM@C|eO$Mv??P?#@>+61{CZdUBze!2r zax>T}n(JGv;FGy;q(zb}^tYw^Cp~S=3Iw2ySZ>+{W{ItCT~(aBv6_Yj+_z_Hxl|qN zx9vV;`9xNdDj*(>aifoil6{;^DD(ZEY}Hm3M&jMJrB!D2iIF+8+%dMaadzFe%&uis zZLRp(t&u|?un6Z-*q*V;2Xk0{Cp%P6SMeKb{6kx*@-8f6E7dOH$QVH7eodEbAyHOw zu^b{#hgVvE#R)Vnb5wrj|1{TK>!z)KAxO}%p$k&g zH6>IleE=Z9$WT#AJ=lBi%T6qrbmIIpb&+9qH5CL#`hRKd`$d3+hUD_rYE&Q51-LC+ zD$BSjM^+#aFzK*SovKjFvgQ*hJ0hr5rAWk7R4^m&VK*pe#2cPNgF;@wrxos9Iw1po z58^M0!U5Qi%jKnRFp0E`fRXxA=#Wf9HsHNa{;FQvqb{y%0h?|-?xpjF_7su&|e4W>5&@^KPH zOb>=r%0Kl$Ap(g31SCj7LGUvv4HXSC;vm1nUN36`~5lLag^X0g>NY z5X51VCfm)0R~0h`H)#689R)(_1OaEb`w2u_ZxHR1>@|ah!Wa-JT!UZ@12zIMA+SJ$ z8Wo!K1y@VN0L1)B_HpOD2n3-FC88M2gezh$M+NYZ0xl$gnoA^Ol($BN4v5ScUA`C~ z04lk4sN<8OLO<6&hDHbpI0sVCWGhT1SI9+#000CuNEKe0IT2_X?(x5-NHB6>0R>eM zH4FL_l2|~}2r@i)L?@7iEQN>8+5q6VEH0X1!Zg8-U8vjOYe+$GHX}V;jBLSML+n-t z_)_9?QNjR2rAbQ1s9b%SSONtpUFgGAV4o>oBF2v{IJ#lUn?eI+1ee+kb1E=&{)dIJ zs_JIb#Ozz=1X~pV7cNp+r5M7$rkq;GDLW<39@_=QL9$x&0AZFWz;^BLmeIYq=bZAMEq|Jfk(R*(Py literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openhtj2k_hifi_ht1_02.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openhtj2k_hifi_ht1_02.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..238e260404f6085f84f0b711c15b8cb771b21cc4 GIT binary patch literal 98304 zcmW)o1K1u{7RTqFbMMR>H@59GXd1Jz?WD17+qP}nw(UlZZL9ms?mkbp`}NC#0GO=7db zYwNZ0vf3W{N@Dm8U0-|6*0&?Iti7cPG`roWPBYkv8eWa;G+9kAciLXp^)`-uBKNeX z)|Hu>NmEE+y(QO79=lL0Nr%vz;H7;LW?Gor?taiY$ZuDM3WxUC=%Fz|HGA3RbB85L z(8V91SKQz3AIalScOkv*NA~MzR)2`gs!8n~O(}2fP5VG@>sf6lt8|v;l-AN#%9H zon)HwnHQK@w!AGWb2Yv;HO(Zd9QRaLN)dC!_O*>Aox5e{N?5yFpP4aQMwgq&?y&75 z&FpVBfrOVcX1SNa9kOTTp=;{$=sUlWA8_?n+m~`r4oeU7pKjG~lFnY%s4`!>={hr0 z8|!}4NdA(*c>LVREN!kw%_TV~HOx9)rarG6MqYSJb*Nr9&*dL!Z4yf~`RK*8|7iw^ zWxsJ{?#LNwZU#t0NpJSqjdqR<*Y0}TB-L;_gWvyR?szHXg}LEHmBZ$gcS0&lHIrK& zng6_gyt^^mT_1Nrw((8V%Q)NH{wdRKNBftY;d)k)p00wsB#U&sUNjwCQFlk;`Tx37 z%3aqilAW4ZXW9gbf>gGd&BB$RXxf_o-X1&L&X$r=Si+k%Iz%6tg#KCATR+QX8D&ao zM4fJ?OD#!i5>sshOm+Q3FPNXYP4h`adrnKqX|8j8dF%?iKQwGm$={<>B(~%fByJ+7^Vh!nf zliuZWl{Bfw)|=*r?2;koH~UnJNCoQohWT5{N)ppdOX)t-(EUaq{jihlCTVRG+L}_+ zI-5(%>uVWm^67ul*4)%p)X6w4t=p;U{JP0Z)dITER5lq*6)%!KrJqb)7ljV(COIXn z$)GQ!m8oZ=+SJljbI>*ITskUej}FpXrj8T$Q2uhC>~48(r`ts`!j`sUC3IwZ>H}$KW|+aIpBK|5b{{0Ud(CXQAhV=` z>7muQ0t2MEq%~`0u~atIZBAQVPRc$$RbKAlauY{Ihnnu(^)n`sdOFz@)aW{!PySWL zn+ImP+3A&%=O&8jE%E+;ts?3?D)W|$t+)Nzeg`JxAX`F;y2x&dY;m33L7C}Vx+~I@ z_lhp9T|u`;iu$Gf3EIIGaY7I0*4<{e_R;UAB;Wm;?3Zy;%Pg19Qqq*NVVFNPOj*;} zo1-1|o(W@*YFcJjc8O~a$TYrt$X&3v$Ey3LA#Kd#?Gm-x?I30>{~aM^XVf61lNTYeJ=~d8J}!o8@-74Wunsx102{qwGiX)rzYrsn1H&s-z-yYBS=^3Ur<=Km?COFAa} zfFM~A#+LD4x@H=lD{{~)X=B^?(t|l1*I2t>Uzy>yINN8DbdXG@k|xw?W*HTAkKU}V zXH8FAlO6Ymjc7AUWc|Ukd?t^ir#T}>rIBf?RrI7OE`Lf)^MY;D)2z@j?5P2C*)BF< zMZSA_7s0)dT{?q~9&M_dLEdFE$Sn2hFw1=N-Rv>@z4g3HGP!6s+lJCoQ)wr&L3`;0 z)0;_~L+99*wwx64Kf6Emtvz4|NFn#e?q?^S(+GU>?oz^Ba6?^Db@q^^lHJ;u_wuOr zxH6S4d2VWI3|(zL>Mr)$PMxgZObJOYmT%gV>+;MEV(!fe-uVS=K6k`EmAdw(4wV)f zS)Dv%Vo#NAp-aIt+lRfi+{+be8|<=MZ4UNbWWS&RZ+t(j!0_S4jon<|q20gp#PTmF^okkJmahsr>aW*KCi{Vza80LZ7)j*6bxSoCyRxC2 z)2aH!oTW<&u#dund^)qkH<_Fukn|w16m;%E*UZJxv#yFusImMBuDDL}v-v0V3lnLN zM7O`bQOuyCe@z?tv)oY4ZVzZ|c^SkF(%bJcNxGYJ^iDn7!@kz~nh#u*3fyvCE9f|D zY&%^^oe$EPHk|FF>DdkOG@PqLclHG%wbN?$9-Cq!+o6=kv&;CmA>GZj9AKK5sOBiw zr>mEmO|Z=zk=b0EG)(=d@=%v*duglbH8LCXll@!&;LeVgg8Vr_gnPBeOaOaq=8n(R zqEgLX(tff;^XhMs$c+HcXLpnNvl0BF7FYGPc9ztBODbo(i{?^lJijMsr=h!JKgutB zK3`VJAQ@?_o|65hvs7f})R%(thuO|N|3fBnoeN4{u+S~jm8(};D+ka0i8=&yx<)~e27lT`h_#-q5 z-}iUjM!#p)jDA$ViPkr%Ona}Od^0(?0?Euw{`m|TaJH1=@0Pa>K=qBZD${!|{o7uq zgHm>z==u%Z^-!+z317-ZkWA$N*b?nkMg%KYXj z{_Y-`${jR%9R4}NCbs=#ENJ<@w71q~llQK(E22F^NkU;=1^>Alq3^W4T8R@h@K0-h z`&AQ36u-6$r)O*#+e@;A+69B{R2$azqt3(Ip7h8frdK68C6jix&p?H@Z3(-INmzh- z{M}8lk904*W3R;H|65%IdMQuW+yP7Gu*dZ;eXBF|HW=%X>211r?YvUnJTuv(F{LCa zckR0Dm0e(ub2?J~qJu_DJ@Cdi^Tm#}22Ax`XL0x6>JW+N22kf)ZFaj?zA-EQk_|Sk zT`L!uocly=f3V*QnEM_n?Z( zgWlR&qrod9+n(~>pWyG-62V1(uYUD=`^7a=u*PqtXM-ZaWsM!05WLfr!7jhNmS8&F zW^e9u`{k%k&;e$YH^B3}+ujW?l~++R${TN=IcT1FDQqHJSti@YO!&w4pq(Xyz%~oD zzWWH<-Q?=I2&=)T#LvXH%h#N454%bI`8A+y9wBN61hm&U|2 z*}UU?n~EmBM3OIF2GCRvX=7u6me(-1kC-R$%{OMemIJ3X_KW%_brBrDl(8c6+}mm{ zf?hhnDdL)~a)^pLt9LafX!&>DYgWlvDP|HduikrKbe(3FO)%ND^y78<^aaQ_rqlpE zwv;3;jk_UbsgDz8v*~ZHdLz9OUU;vuDP&f8vCJEe56M|dS?YHMHZ8X$vVk|)pODX zo#8J&|7TvOH9Tm5)Ha=LI$J}W-`^F|*7lP&Wj21%+VU_M6zsLvxsQWvbGXt5DQtIv zD3)*+!fOyVVwhX5PO!lLPiIs4gF!WM>_BadK9kBevR6PfpU^x$NeVdTGga5Z?$?tx zAN>)64?mM)@Zq>>=$WH@=54BanJscw67lo!)=3 z7>Vzm+G{e#7UT6#$ULdeJ{pB$Ge%}ov5DX#KfD!YE^~Da?6$wR!Avw=y@(RX4c=vn zndTh=b$&DD{(tx5xd--~bY)9kfeU<;spf%QWtU3<=Yu`6*~iR*#QgWQmyvm$&zoh| z@M*76kNwRbFqBrZ$o8>QB&5&TmB*R?-OUUpr(uSy));aC>>XalYh8U{GIG~p{jR>N zL3P}a&1RA}#7pkg_6m7nnFFs)QB#5LSZhk6#vC;F^&ITAAc$@Pj3c*vLMJ%|*I0sD zb=|~dHw|PLyrZ5wm^P-nw}QS~@qgBwntIBkOHEH{Bk9ba?7)293)#UmE5o!=GvD+v z)4hE%nVFDW-^wz${25tlirGzCUh+^mmqh&ruC&(oPoN`Bq^?T9+Kw}YiiepR>f|Qb z)V7e^XTtB+xu`ZB^b$LCiH2dih65QSK+n5p^HB3AYz8>-Wz?Gm+VU4h)e80kh%CF^ z3cHGJ=fhlAaQ%l#9(xuYF*<1Kpgf16eAZ(+fSPZ@WDDdR_vNR=u;+D@L`K2uz@+O4 zODzJcDre4sLYv7xZKfsVJ?hXFQ=MI6B$b|&`{svCk&Wgc?~+Zv&|e|?{Qzq0HJEuV zc?=t`A?P$4WLAuIib!bSMOi!Cp+C*QN`{sx_?nN}uz2e^Qpw)(6arjyS zu+U{$0E4crA59w&R2fvG0ZiuY_HWyty_pNd{#b6xEECnOumT@U2*S)m$2ss+5p%(N z1pCYBErJoWF?-EMbK1M9_cRu>ZWZsIT~RL3hi|hBYQv5TqBEaDD|kjH*5cp(sdJfW z32kzzYRjQNztku2f=_Z@>Y23YZv)K}P*W3g0VY%%4u0DFEdK zOrb7sn`u&r-Lc6Qwxgv*P}9$Ad($0j%zAbcEIJtDAJt1isUV&`M{UoA=hg?6+_ZmC z|84aPid0Q7Lr+lIZ2L{G+x&K_p0;W1BpsoVbcC#?$5+UE6sjA{Dels7Lff3Ak?#L9g9qxS5`+_bPkv;wgd?S)=uAQVDUH97b1d~OS*7^>N zkdvOgZ5C2djU_6*-ceJ!S@43>L6jhdeS}K9OCE#yYiKrqv;!0Swfw(zuzP2RNnwAD zdn$eW@9@V^Fv{ASgdx%*Yl1l8UFOmW_AKp<;+la;l28gT8<$g=t@RMhbTLTgi+N>Ec%FPW z_q@cY8R${4iPT(!Z(af~4X?AoY%#0NI&YPm>AGriw3;aF?p*Ab&1g9HO>Db^xqT5e z=BcSkmo<>AXtM=n0ULaj++hb@frD(-a%cg8eD#i*cBo7R&2_J&_s)zno4kDT*r@q} zF5Se0w=*@R9Oe@~H&yvO*Ws-rQ5`3N88%5zYV}{L^au{m8fNlJ?GO;i_jSI3*nPsV%M%?q!Nx6afx3)pmd%t&tl z-IW>s(p);3%xu+hu(@Mkp>sM!KNvxcjEB;-10Ix#shgGlFO9P^9ev;=9?fE^w6p7F zBWrQ=nUY>rf0RqFKTu+?@%am@I9=K-vHtht7IyurMe3Gm!inqTWnQ`FgrvV;BALhrjgZl-n% z=J}s=0+{EV+90bxOl$gcTq1Prh<;5z=LM#8HGhTkVYpr0QR(de=114ubi^oW4#u8h zx=I&Rj(J?;)MlwHm*Q0NTAZ47x&rJm1tmO})WD%zDeGW-ThJAIG522KF|`HTw_}@M z(~WxBl%wBAn2r48kXg!iZzS)SgfpcZli(sKY6p|^z4=MKZ-pC1mVwd&ud0$MXR3Lv zq$~F~oy14QP0bZ;hQ` zzn&@Oz{$Qws(3NXF|Qnd&v;?HU0zEsrU#ob9lUdJ zj$~dL6VBA|{+2Xs#*XZ-IA#a;{fhbJ&*p%CE;C2G{gMvGF~dFt5jBRVxAO9!@Z2!1 z(DZuAPAV;xlz_Ju(I|EVU9g84zMQ%l2U5$5|CJUrep*YSi1vZ8En@?&{=dt%Q=*~> zy{G=Cz~Iv}NiIo6dy5UT7xr)ft?z(+Ex-E*+;Cj59riie;|EZEJJ`hz35Qyn5>Bzs zCeS2qkFBJq&_=$>AeW3?(1tB@9Cm!4_iDq0IVeN?7=9(Y12y$3sA;2iVV`EuBxnJt zrKxEkUrjF4f&OS?nwyMfqjv!xq7qp2KQKfL-7ZljryFexXro}D|Ci0+SI4{Q=O4pI zh>5;>TeA7x+#A{BJHL!J@VB}^1HXg2DQjJRFnu?Qcthrz^So~~c>@lLA;;JPW6+~&g2YzylSm-O z{CXE$G|!y(`reCNNJ9n?GSdAHCve6gfdh>zVfT=&sgYKJ(pc;vF%g%^oi-Xsw4ir33la^+6O{ zOi9@0M;JkUYQFR@C!D#I43&5fI6ozrGL0mrV{@8DUV9hLjRS?Iw10r!r@(0nvq4Ir zJI&C<`a#F4uaD4&KT9tdTzvbh7gA^={mDKo1?rBYne1jZ;Rkf}f5;d#(7AF6v=gGL zXX4@hu(RKZP(gO(|$H4I%gYq)i%|9ZksJeJynLG zH1_wpe0Coi{$3d9F!ZHdb|r4sG~6`qhphyMT>@qc=@HbSi24{LSwc5TW4O-~FlT)8 zYf3EkCVCaU#NKH$hArA2W$!QX{MK%?{=hFArZIw2elzCIPRVQ1`(xb`>5r!wS#izX zJE`CwbYEdlZQT`FO&1)M*KU(j?$A~6^KHGN|Cs{l8w>GbrZeHYGeu9D$NB<2=XY1Y z&BupL?MBEq_tGWP8+c(|NeHyFg{2r*pu2fyx7fb&5`SS7Gvq9osvUf!DC)s?Eh2s3 zELHJ*oAa8x-DDRRJ)P55^zzYBj3lf&pOYo$B>^=;D@rg`1G!6oc!HnM9n@UWI@E2dBZ@WwAa_JX;}G+M^)C=D7+$n2Ux=Vdb6 zPylOq3BBj2WD`8~9NXj|Dpqlmiz$~(N;A{4qDl2+hNCXvZk*yX*T9uKDs`9xBiOdP z%}icrkL}0gJW21I1hHJwsInh7BqtlWoJ24=n7uK<+jY2xJL&7Vavf(XF1pG%n_vD! z|0&Dd9zeRo=srGK5;xs8mkM-HL-P*zP;a%k0x_YHfmAbFZV$|7bG!g zCn1{DcKnT2pq+Hw={8jURaos&e2wP1SOldgE!;IaS8|oh=<4Zoe2|ya=SX`)YPlEG zYIZs_s>u{g^LyC?E(fl}e{idK`Wm+-zNTlIHPf;7C;asXh{f7r?2&ca-L0`J$Y2zu z7pB_)MQ9RSKALW{NiFQl*0Wvo8oMKcoy-2J0XpnOdZjN)#J}*n0ED>~+_IV+!et#G zvu$!}?}cq^yU7(-()}aW@9oaZ8W+>ek!E0|$y(i=N9&vD(&Bkla$($D=?w3l&zy;a z`=5sCaX?bK%04~O zQ|E7(y4BfVf!S$0kV4rDlWi++?FKs%+<%x#`wSO|CO_Re+^EWukI8?~JL$#oR>>Y% z&rofOS9{hCa4&JCwYyYaxQ@v5s`mpEAQ61z!5o#2_PZ+FR}A_Fj`h>SP9> zawNfvyaO)sTnsl&9^qhAM>{G4Q=X49mfW0R0;JPnepdgm4gp=XVa|`(M=^O z?RF-_8CdU2DM>1z62Co5{*Xf`U|<-$>HH`ck#we%$J6U7aryl?8o`d!yAq%jor1;Z zLV=HM_k+4excDxWPIQS}CM`$bh4d=(d8f2;A(w)gwGlNY0*STbs%W;Cbg_%-X6a~D zkqO$CYM%iY@NF0^ho*W(ije1-tbe*rcDhz$v#eI%m4wM=c3t>+OxM=VVi!cR4VbY7 z?L_?!>i!}t=;3X0fO)t~kMrj!{gKm!;$@YE=>t?^(9t15jLaS;Y6vNUHCNtjAefAwx_v z$pn6Wh0-|H47VYs#A~kMVu|B$ig0<(X~W$z&|wxR)6!WyCX)6mNjfHwxWlKbYq!>iQ$l>j6Gn zD(Q*awFU*a4NBx$9m>p4r>CV9TzftGUl==E$I2}f#YHGQ-RvQm;)n5H>rj$2Ikh15 z>@+JVH>7dFSJm)PemS;}$qn9=^1)nxv7LzvJV__uTg_ndi+v@P;RPeLq}@Vt=m`$o zT&d@_z){+{)o_vDU2|&T3ock=YCgB^N~i47thxZE&|OEXJJ+t&BYIz6n=c?eVe^p2NufQ8+{3F!xBGCSEdF9XZ zXW}emv>D}JIZi_31be2F{0&X`geyA4{U5S&HUlRIH-R~N~{n|lYK@_c|7QAx5>4SoO%Jk>%7C}jl zh}(OH-Bb)kdMKYKHFIvOX-(p{g~WruO*Ut7ADWphg71oY69u%~3fMkv9|Y9q<~p=|jv_(>m49C>MC zqhh`DT6hs~`W4+fgM@=?q?h|%R*>O9GXmGfk#k(Mm#m(uy}4>Y}o!CQa7_I9z|7Ks@2_emTCKm7AL92_1~ zD*47whz8z?NflRKp);gMC#)-}nF-aHa!*Kgel$BxS5pMVb+p8%;(JMF6r1XLiR*S& zQsDvrlq}5h{PGIknp_WnBKm^WH`;ILCs*A=9f)J@!EN%8J$p+Ua4jxd4ttdC^T9>3 zwcKr6nj~8|{}21YRdZ!KA{>L7FtPEu0chM;kY$n$p zE+Vdi9Znvj35r)mm`D=+is!yezk!8@!lsMZG5Fn{Ev~<`sgiv6pWwS5?m3FrR`Sf# zY-G|z;WYyaX&f^SpJ*SmqMbWOdbYYdBJ-JwQ=}R{pD0QE({w~%@^w|^l^#OviHBdj zQUpQG zM7N0tnvIDzOX5lXb3ed5O+v*(d+n4k<->Gyb@=Bs)`U(34Q;<5dGM489!qfnP=Ieq zW`89(HazzMKI3^nCQPK+Xca6vvCrRfMLezDR%BcnH!oCTz6mm%>HZ}hSb@8odL>i zt_wiL9rUQ5$}ek62TS~U;La%^qT$qh0-KKMomscKI56M5_87|Qc=wh$Kb^@xiA^!t zw$%+biH)UeVEr-m1JzQU#O!A7VFBuYg;wASuhLwkfOhB(ey5IJfYQ}|m=bR3g&qaO2yd0D*yT=Qn~)2;)P|L2eN z$Lb?Gx{`zmX8EJpDb;O0IR<7vY=(p9Z-^i4@VDtwxO_Dk$0jX_>e$l#E8F~u{sKC# zJ{$I+?aDS!>bBZl;GZ?`xg0ugM`@ zP_2)cL)@<$OylQTMJC|sjAv^<0O8btUspk?A4V_4)e|z^OhhNYYb@J0gG7hRjWtzmm7L1fe-jX+%Yuie^Ma(oMIDS<(npHJ+M40KL~*^P zBU#)I(v?Xvl6#cavD4rbgTcLZbu9YZTC$Ec`OW#V4bJ|<+=0VBLW@`p2WkioX^g6L z+H^E!xSmbqGsw1pJ!A9Q3|hti+r88~{wL=2Wj~tV-RAZOxO$_EUk9KwYxSyk$U;!r=n&NW5 zxW9IppG4(P-vGe3~ouJ3G89AQu|14HR8GUu2EpQfafC%K zsq?f1pZialZ`0e>64NcR+hjD^t)}Fc@{o0T4<@(h3^Vn$xnNr`uL>~}_Mo9HqaKR8 zOR%RF@T0{NR`!uBY(aM7pnL=ub=4xlEjH1^pm`9JtM;0l!&5xaJZz1rS`-)c1!&}l zZUU=((7F8EQuZAujD9609-9cvlq@m}c5u~vCJmTKS}>DOnJ)NcBqEp>Uril;a)Si^ z9`v8>c)=rahfC6rx$p=wn;M*G>gf&Of7g4F^(tAD0s1=`%j{gqp(qgZP%D4!=vSb% z`SuNHb}ZkqIec=dsm6p|W0uQC)R^;Vzis82UzY22rP0oySqO#YB;{+0!if`BELO@Y9C@+QUkF$ zThSWa*vi##2c?^>ZPQ3g=6`&N3AXqw9pSd)@X_X>YHrZdx*LqT1NFJ6PkRt3TCs^yX-ki{hFLJ^v>)+%0q@_+V?| zI9oeDSmp2GM8{&(xp~0@KelVe-r0v|w8eJf6&kw4wg_{6Fw^@SX_OvrFW3GL+_y#e zo14Hg-`tK1^4YVt{UQc-<$DE@LOs&N5#{sSCsg+_A|*$Z>Ir%_<) zq>fMpd&zwEcni%dAvXxNIMBc2x@i~wZcZrywp=R@sPMt;_fbsjc)?!(36pWCG%(}v zFPef|+Q75w`WgK?S_m{%Nh`TjY~hq-OZG}~wj0WxyF<>T1Uo1%`5$WwNGHxuq$-kR<++ zq+D(rUiwmF%X9=4l23o?G_J}ds&|5PaT!SO)F!h*UKnk&z5e5Wbc1cWAhkcl_8?Q8 z*+%dyF!O&WT~JI*@{>e1q3ccxA|KuNKpTY)2O*a#O#aYYTf|QXLi;tZISHvL^#|I; z92c^q=#eS9o!#y06Ot+~P)!dp^*f-jtwz&m37)Blvay$$P|uFl25vE^VYPeD-xvck zN^JMqM4c3M%!qpKX1grrxmnDxxS7&xJ6 zsqj&-n_gn+#qshPU33a;t{SQP5uAx=1RE^K>AxoIi?=9*KjpMB{sLE8LpW%EXl2r~ zx&N;)pZR~fpe-4RO(>ARRIE&N^ zMMzm}M{{|H=GoiC;^(W_Vl&uQzuYh~Jaoq=(}D@oN$%nWjRJ44M;G14g#2kXYYoy_ zSzvy~WJKv$WSktO`!b41bg~gE;Hs@q<2RcYD5Yc;aOC>XHPOgE4J47W-pel@jI|2M z-)wk)*>M5Zk$G4tXIw>BQ4f;xDr(Z94i%RZ;J^*2PwUZ((%5fWTT+24Z)?P0xW64N zoq_rMM{vO3r+LT>{4~eNliZLP!DxRXC}^a=Ob`1N{FX4pmhJ@8dn*X)4)3x^GRJx7l-UaICXD*#YEizYp!n>>u@k+JR8Zi z!$&9ya;U?1o`5^^gDi1alkS&SN7AOCCPkxIfJQOLCgVidYtCaHlYX#+rZO0Q+5v^C zv|Wf+78?`_J0VRvoCI2B6P^V4LHTZ{P<3gT^YQ-w3LWGIIfuKZl7Gpy)qWt)M^cc? zPY%+yiJXNM-qT#theT@=ISQ^DOBVMGm5>%DlSL+?OPw`?L3OP-<1m);oUVgDXHcx=-^w;dea`QewXY- z5q)fbcaL#1PxwD{KH7Lw1IGp3{XsT66UtITw4Hirg~iNE^uzpeh6=*3z=xR%zsNz_ zvx7TIntcQ_tFcT5tHfjieuO(D<-O*ZE1-Iqoxh5m_?c_lRQ}?u&@`FBw*6h&Q(gV! zi(k{POASPI>$sAI;8`rbM;PF0UFzxjP!T$|VBB_mqdf8_qc189eSK9qC55%f&ZAxPkQ zg3yVy8`~WqfwDFyr*zMdD4T~Ow*d{OhrEPA z$2H5~&S>x8l3enM-5Lp2@(MQE2=$;bXS;gY#Hjki@l!5?nV0KQ_YH3$8TAkm&O61n z(Sz*JxOO0#>OroCf4c)O@Z&I-hNW#o!@`qy|CDBE={If_F&Xn2axJ-lV zRaV24n-7A`NcQ+QRJATJW!z3S;9X88wnwEO1Xk||!ubu3Ie?8a5k$S1GgO0c}zCVLLP3FRCBJSNM3CnkUB5COKOUxnz z^V$XS9|o#_XP?Pcm}YUa)}=(lk4z88;vKuNHD1eHwAREZZr#i?&X5*2n@wla!~2Tk zoKiBoW%TS`u+Tu+UGf;O=x-@=OPD6|iU4xVSj#jYB zwBo!^k^iHkiENh0oHQtoZ<|i;n&>76`p;eCbQP@m07|LHR&9q4P{lt-jkPCF9u^Jh zIq&@ywLh2iMTMw|N_?2#eCOBqhw37#bR8IeiwmQ1L+t}jsJMt|(VO8E2fS)vl^D`L z_~LH>5gA`O3Aoemu6Nl2LuDc-@BTB1{q^pd3=F)0lZ4Er?`U+dTw=Z92D=B+kZ%zg zcF>5E1+}T9S+X9Cl-?E#dIl?C9T#<;G=N!b(aPZlg^eFtjhpZ{`P!P=n)B%!;kNf+ z83XjKtK^#K2DAvZIot+&3*G4%Jok|AgNoyWzvr0i=D3;d&F3y`BB9g_*Cqmb&38_J z?`O8J1|PQ3Algim;<9rP!7CS;mO30UVTcH(QV7w7e+l98Lj zod0IeQ>E+d5L=A9T)?fB^?0fE_9!WORVAITEYCt7F?m!Z=0M)dK`G<`#h~4@L z=QlCuGFCGmrjdV}3X1Sx69wrzTn&7#?x59ayv|RqgEcvh)lKx1My0&XlzT$= zrPeuYfvk8jlWh_7f;yZ|9mt>Wy@NUcXLB7H+G!?*-NzmI55@buoMK1!1gq?_DKrnf zH6mv@OS(d6m~WYczdY%drWv)~kMn&`b&{N8zRlvKC+9x#+um_c^3pZSv?g7i0#7G~T?IdSH*GH__EYlGE4_L0HyO)IW*?uS9=y0P7$!Yw(j%rDndh9m{!C^?dL3kD zlk8eg_qD@E`{g>FG!;lct~HIw+Y~p|*dY1L6|x6?OmUuPvL9T%jSecRf9fssJNcDs zFz87j%xI)17I0!=HHq%V0wXoa^#j?A>Fmm=Qpy@zOAZD(gMzk$3+uL0pYuUGB{``? zGK$n>_j?oy}s}ka-Jt@ z*f?R@k_jt>s-MyBz)_7%@8r-fR7eh8=)TxOAdneU>;zawG(P5rS6ces@HJ*Lwd$mv2P^> z+%!BYaF@talHSrt*13ed%~ZAsMrdvMa(k_4r_t*HU>oGicLq`War z(QhVDy|uXlr*$Lus|htW(=-nv2F~s!MY+VgPO7&g=NzN)+>hFlmF!}8PBDx&-EmIO znKkgunr1&twxt=*r_E|gNJh?cRnmCc#Tc~55?tH5UtHr^2h54xH)OWCn9}*HG1G>8V(t z!4#nF#-wrD@Mj^Dm99x`Ld^Wtz>f#;_u<-vZyJezo^KMOMK|VL+f1~bW9)*~unlVp zc=e?cj@#d)*3+BAps$KNNuwvJ?6^1@o8bGob&uJH(|w1t@C#8d--1Z<%gP{4&_(YD ze+3iu4z&@;K4xrmJx(sMhdjgwTr82OoII#uVI3J+uu2lLTa{f?6r}5TVeM^3KL1a< z3uZS}D=>%S*=r;iYlFKI%3>$3q?QVv`QzYGBiZ|5?QodX=g{Y%G#&V!6DxVCmT&Ua z6?4-xKa+5^rUHB4MQiLC5`1m1m6t_U{lWi_E)xT|2 zRD!(roy!7*Cu#^b73Ac%rI{c+WuwpT(0q~mA*{% z^`|fmX0wyjeF|#6ImqXp zIfEzl*84z`|GaFV=6!n+%zpqz_?{iTTkjZ;sovN0L@Q2-^Ysx9G7;sZGSyp$>V@!e zY9=DTvy<1~3Ii(5hC0ReDFv=r=P&a!*`mz13*e$yY>ilK#%QPz-^@p^6!o8s{w=_h zC^pD^GR$*8#oyV#-@ut=as7&GRNT_W+^@PQoR>qnL(T0&(8U}n$GN1Fa?xdTqhu6* zYfU`gxi&F1oYpmy!aRE_D{7eW*~U!N){+wqy(E2ljw|1TC&uJO)2PMyuZZ}nL#3ET zB#Rp!XDcetPs`{hq}uARF~7Miu9%+ZNjm{f=M@yQhopM$;(0~!U%-iaa`#)3L9Hjp zf}eg1TMzfSt0u*Lt|$pfG9;EfV3liTKC@_|xr8HgmMO87&HIVE%83R$1J&gQXVZGh zZYr|{JbgAUPbrYhX=%b|JB&}%4V`cr3S@WjahdDrj-YHX#QtzoU1b<|3AD0FJf)$P zeGn85Pz$t(`e4_(cD-amGhf1dsP8IhRsXZw10(pX%}F)<$&=QWQ_oW*Cua~;w=(mO zN_}wi=|0jWSs*zk; zz)YJ*1^$%IAU#qq+3w{q1<0bcXXf*y4xH9z zl35aQqPYz2)g%+2NwPpAnoC12%GwYiH+x&A0G5;7l(tFJvaSqCVsOVdU!lI;6@#eOIQ}2 zs-4hDn0_>Slan1;bR=%e23ZP| z9fZ1D)P6D(Q4yb*mFQHNq$*RQAIkPcG621)nZYvFzwhd4Jw8tcRHhHS*GJN==Xv%; zA(v8@J9VFBF3QqJ9JG)2C9hc>T(QN^;BQfd6AiS@gPXkj<4~1Q2G^EPxB`vnp8X<2 zLg7Q7?K=Mte;FB}%_uzY?P63N%N2OeS^h)tl-RJ3d~y!YB`jK7Lfco)1|5TEq!tpQ znXBL4jnL@$Vm{eXo(;w)j7CqrC0`TH$RPunT8Hs@m)Pn&Q}6>i%zjSL zw)J@O0nX7NkVbnFZ{<+eTXJ_I!Ww77wdY8^Ur%w=C{$)OO{6W%M&{=)uG+sm8qOlIN*=L>wV!hC-gdr!N=}|E6bU;L8PJ1kbGWbzR4|i#9pp8Sqn~5 zm*v$<*du6JYti}3z;U{I3(RHC9nT=qf5OWlIYCg9sM2`MiY1(CjEXuv(M&=E{~)_~ zZbMF6Gjy7C$9;I#I&`5TX#P2ZVLs1lv{CGR^8s!ile@WD7nwrj$uF5JBti#DJosrk zw7Wt!wyXj@E--mPJ8w)al(+Wg8ZOFAuM=A05zcU3fPYQL@%;zq`Uz-lCVy5lOEnEY zxef*%$d1nE3ZSo^Vk!=TQ#_Z~Y=IA&HR#|MBa@c_Pv7`SVdcZn`l`qqp8F9F-jT+> zGTC7b=XtdpY?)r@Ia8!LuP~G8*A91ShE2ls3xYEK9bJiDv`R0+AX4EbTx4!^LCL>` zgR_?Wc~#!MpKb{%2CePTQ0mY-o6W!MDrpf;c@Nh=!8r?bGfBu4I>ArrFVJYgX8#K( zyz=^!NZEDw&+1Xq7a2*9{|*B5VDV?11)wh#$0yjn` zae9h8Qhe8$&PvE{e#I*ZLvL0h)iXnm;?!J{o&GnMk7sZV0JVN*N0U~?(|KYJQ#s*y zjzI)Bo>%*oON@!0Hdm_R0wkapzJb2N^IU@B`c?PpzXp7RJF^eW(}c;N2VG^iEW)i$ z$C<0K-a!&Xmn0@9-6DV`?wP0FV{pz5T&=#K*|uEa{{LrlE6rQo$~C_Tv-|a&nS12Q zj*_o9U>YMMJ6FF`M@Kx!%k&adZcGsnw9=Ie;(=g;iO?d+k@bQ1L)_) z?Njfom(jauV$xp)yjq-t2_q?CCF9Hp&bAC^6LmL_@UsSSerf?ovMh>6H}mVc9DTGW ze(rKT=g0F?Xi!Q5*4j@6&xxb3RoI&{vn$|Uc0Vt0;f5qyP0U4Wp%W!M49=X73G|0{^1h34ecSbk>X1u10LkU~kRQ^1zFQ0FG$-(_Y;?_pDv zk^8t~xp^8=FISgma@O+29wAwGo_e@%hsY*UF){z|LphwIN-&iCY?+0on}3aJIfNGR z9uK7`ibMu9^B>aHCx=V^f2wQ7Nt2s4l7^wW*5f7R;2mp`=3S!m(9e@-7=IiZ`|=2dEkoGUrKDUipT3Uj#xlVcq3HI}8_dr^5)J0N zMzU~~7U05U^*ie)_SAJe%J%+v9JdE}1S$M>?w-torw`@qz)CpR@gPBP8nw9t6TTgt zovy)UoLYce@~nTUPSMK|g= za%dx^2;S4bJdH%88Qxqh8HsB=7;JeGz32?O@(ju>m7v*$!tw6gSl&Mc=FOFw(532u@5ic zNmets=3nh8*?{V@O)7J>i%Jf>wm*2j$9cLSE4hNjl7bp8Y|?{OewdCV1X8gnW|KbJ z%f>m&dEpYAPQ9m_;5*$y6NCG4{bo zj#(O=Z9WlI@IL2=kCSO>?HXM4_VDdbKgXEP{JY`}BI#MT_NEY*FUir^mc4&TV zgeRzTsa-7UOp-GUlXJK`kKR=h=Cz3Y&q6exo+PEOF{SH(f8N6`^Kdfa2V1^87^wqw zo0Xc0q^(RJ+Xc11Iy@q~9M=O({;H@JGwG&n;MFl`i6PE4#PxY%sJ%|wza412l?+CY zN^WwXGyEx&@V;-EiTq}Gt%B-uh_kM{Z4ddE(^#*~<^OSX7SL5J%@&@n?wL8k-Q6x4 z+?}95?(PJ42m~isu;6Y%0>PaS91`5!U4py&{~gv_k98jp!Z|b5RlD}yRjtF&ly9ng zV4j<}D(+ivtaz%DHC=6Ep6v+V{S~=-o}NJ@{={i<5H+wj{mUqwiL+b>?kfx3TuXKs zKCwIK3b;TYp#HO$Q71$9VfzXwwP__8pW* zM+bt_X5g?b4f7mZ)`Jy;C9PWt*71W=!tt0VVBiJx{F%L84)RhNSw!y2FWKCdP8a>s zE`jg#D-M=S_Hy*q>U-=+2P(aW7#m4LBDkToWuHrQ__@5CmPaQUNGgHmEVH9bd+<@R?L=vz!&;tp|M^PC&=R9E22+sB{u!RZ%|y@*qs zB|5It$vz2E1Y?$y3e^D~J~lI<$z5ZTvhksT^-*r1A63SA`waj6ZaPI@5PJy~AKyYc zbzaSfvGl-njg%I2a4wOy85XD(y+A*-l{%9a51*=@$4m4_T-f zlZFxa8=sN+8&TPd$u&CM_jr)D(i=Y4cXTXt@Lss=I@19)kjc&z=K`*z#W1=JsN)UT zV0H!0N};l8RLE1PTCI3`kHouidVyMJX5wLZ%V~Oy!s|U#A|Kh8yAln(57YZ0YB7B8 zUn`gOpZvl-+LWlsW;~wJ?m)X0S-qe>ix%lC0Xj3MqPBM7QbtfWOR{?{2LI}Lc(c0T zUgjQjXSbj`-UQ;JTF%UUkfp3Z9L!#Za~oRxI$cGFtHLM_US^;Y|IId8HgD92o0<*- z-;K}4Y_K^<2Giu^wqWm++hIe=85m|Gm)`V#hzd%^K5jl=!fnBx4Zf3 zPIUX5qfS;QNMCn~Ig$KY(%A|_`)ynM-s)+-Jc?L)H2ZpC7KDsC>yWxAa<(a3I?I8asl-3ORZ1=MlsHWb;lZn8! zzLX~u20!NA!M@i{ulEwk7vPa<~1FEIZG_{LoH zt?B6F9?}0cV?)bQ-IhF4QIg6v>k%Cy9Ts`)InJEpL}L^B!I^XiyNL7G`V}1bPV|UJ zC|22EeiWSxy9#v_QP&!_u&SgdcjcE{XhXTod3z6tGzoou4p|0DxrG*1of)xIoHZhf zuDK3QhKAJH4q&;~(h5}hpLFAMPUN$7C5CINJapejPz08!QvA;_>o4;Gap8w{a~58x zP`cw<^f)=x4E=}MtdVwCdzhYJcAA)fzRkH*+adN0J%G2C%1rkh_JH-B^-eoem$!3~ zij-K@<|ibFd3#Pz-CUiJqbP57h=Hnrl)Q3en5VanCwIrYWZ{ z9vi7nn#Yn;VsLIB#4}%&T2>ESIs~-yONCPJG|rhsRMOjaV_17&8lF zKyK~vgv4Vi`GmBgN0y#C%SieNWfrT>vk+sLcsk7#a5VkMXPKrO{d1N!##wqnAK>?E>GD+Sp+xCD z^;Wa_R9rfcXI5Fx#cs9I*#(}wV_&f=>Sz;&Mw89NrV6xzC%LLJqQX~}b#@EB?S7=r%s}d?wrPeU|t%b#D3W@oD3XioHGdcb;`G=a>FuR&qM-I6Ls> z?WS}2pw=sXmX#FW%M|6^4&=k>e!^~?#nhH8_?)Za^$n(qb(Vf07B-8)pKR6*+?h^o zlLl{pL3U8j=dC!7za2}bG+Q{i)6j;}>-^M|eJZg%m0i14$fen33A|Zbd@S(#c+OAo zy|#d9o6BRe?rhx3xKru6ICcc-}j{CaY7&P6cBhmzq12tW`=9(@ont zJu@}wOftCQOY)S@5GD8QmUcAUEbcSO-8= zV*06%dIeTVrdyfsy1cx^Id4gI`k-o37Z>zwn3J+_L=K;U&D@^DcwC>6kp|GCrEyB> zZg{bF>K!QQ5mL(CMLkkDa{s4Ofj0`m{nBCJ0c*~?h;0tj>ot)qZ1bMv_HkG0`x1e= zngNxlo^=atXn^XfnYX4FXb(evUM*BH7W?F2cuK<5h2l+$z_+RFwVc*)ROV&s9Gw4J zGTuUcPOZRku~Cgd@2DcNVVfPa3GX)Yowz2>!k@>bQvVPACBJ;uCv{x7(6|2-XMf&! zHN2+r)p}XYoYxuq7Aj!_c#NSi^XZ)vxF1iNohZCZar-x9db-#|UJ)wCj4r2iL29w^O;@^-gy% zgm+U%0egtatXnuXjC5Cd)7gT1qu~W#no)Yd66xAHOmuuvi zMye62Yg#-5JLsY6(NT`jKZuZ`63yB9t6UiJG_ssG&RrF-Po+_j#-lr2;x?Z^G@W;7 z6UEripGhZzQAsI9oM=>))9f2ese8akC#8dkFp=_*I+9(x?l9*kF~8Y8Z>BJZ-q8$p zZMUOo>+{*u%AV)mb1oA_rR_g(5oyuyx`A?2f=?{;h3~}q4{G!zP*4ewd>=S8Z%?-s zr72Wq!}QKWUpqk8*3ezy3}b6S2&~-!yATs|MZpGZRVa?2Zukj5IQ>A<1F7t7@#mgU zqfIH)w;AwY-SL?($N5$rh3+=&Wji7v2|QRhjH@sOvxKN<&z(EtP~}#@R(VVb+nxRc`75QwxyXj@WEYnL5Fkt zM!<g8dW0wRgcSnz=z)*g4{zrwI>+6rFO@o{`~c_t zM9&?E7LeSqqffen?iNcWYS}>E@=o+7m%TLWK=p;F1-FUn9WbKljUTE|4Q3G9|MOZH z^gdU6*^)|94e!A@IN=XCOh(X6e^TLmmkr#xki`BZ;jp_Ixvyc4ebhT?!zPmsoZq?1 z)hFp{52AfsQgzXWeyGGm_y={#?qnC%L#WqLOt{>#PvQYyj_#9>Zt!HVOT#>zcgf22}js zQlGxyqeP%bEY}sOIMsDFy5KbWwx_h`tEuL>>TWb^Tt7FhY3+Ui%RRSe;&4#*VR)*t zeCyAW+7&mpxo#HHGk#}ZN4R_k^ACV^>_Ob71AktVe7N+Fspj|uimEmy7oAHuuGIO` zK&OOhsU>m9Y^m9=^8rR>E4+$FF5_a3#_9BpesdXUzcF=cq*{V**i?FOeoM#^7?vxl z6ESv6g594^A@s-es!IxXT*u+oqU%;7{uK+a3+6t{?A{yMjpMP2l*EghK)(ZNr3C-XSA&^s&q&35 zAmdTN_o;(c6YWdJc;duy3YtFT=Zhe1*F6Pt3+1%tc1Ga+_Hvkoio=Eqb8R9PCEGW|&wzjoS09N0~=UjqP79Z}Y ziF45&gu*i3%oNM%2OG1SeASdaFvrMmWAUENlORrCQk}pZKxT$TvSXwT740kh@Jilq z9ws-Rv)jjUGV1kq4Z4Gj<_X+FSrec0ehq%6E77}wS>PlvMy_f7l3s9f%5?r#*bB;csb*2Lgd7aNm$mCghgCXdlg~W>oeyQhdJMT<& zSju_X1U^@swDfp#@-oljQNyGrlS)fDO)=`42}Ol0gUD7){UROOBcg+xbU7 z#3A2;je?yG`#_!gPG7wlPCl>Jpp+W$nRoD0ePs6exD01APi)*lWx(f`=)}>>z`&J3 z!rW$nZeD{*RKj^<&(R5Z3wQA+%*T)Po_8@;@8M~8*5{-bxHkrGZ73Vwy0MopFD^Ky z+TmBPGH3A@$80y}Git&GI}8QznZ262T#!y@m>x~_ua4U2^_x9K$$H5=EUq+_l$u)r zJ~K6))Dg9bJBD_eM))@-fX&K)9>Vba45Dv+hqhG{{%x!DhU?DAB;7~r3sE!~CucNU z<38i-e`*!i&%jHoh`PJ#BU;s5CVhNS`eOLEx~o9+-%+xIyn51_DQ(z!GDH_agO1|8 zBxP>>usUZ=CR-LqYj}hPZ8HZ_l}%9P__U|+HMLZE`F3~oXy&%cm}Oj5K(*n-C)>>Al=N_X7|=5 zKr&5r4?4>4avVm2y{;&ai)Az%MG6%8$L13=+K*xWv(i)4r^o+}8n~5{)ll|QCF{d% z9|ZM9$^oiCEaj+*Rwt>!iT=Y3#5lHTwP2!oJe~e?wGre{T=#Xh+kePQ1Hf#jz}`m< z93PR;*?gn=XEclPu=RDrV0HY!56|U=Q_#5}ADjYIfnfKX2{~xEdE$)IOk{G978#PdMRt>DACYHGmO00{H`AgkpSK%7f zO|`XJqiTIc=}Tz;lzq6=g2?m3a2iZNv)-h>(VzO_*}m>*GXmA=ApPtj+!_s?EzBTn z!sm9FzWxaPdpqja5ZQ^>E}hioYzFHwXcll35%nPvy~g}GLuwRU8E|TN%D|+lEVB)@h9tX275{-{ShZ~ z9-^y_{FEc0h~&iEELD+D^+?s>_osoOr?96qr5#NdQx~qOuza>x@Q%E9(-e_fa67S3 z;`-ye3)7wOqYmc`G7DmdGnIVZ1o4z|o164;p`7c>DjeV4BPj=VYN+Ed3swdM?)62d z2VrK?YfQ-|=lMp==#2UlD9 zbzboQRXGH2R++B90?v(3=B3Ps?<}t$gDXnHqW`s-2y`DigP4gO$C>Spk1Mk*buZ$q z&g}Lf&tIU+i^lO1&p9ep-C0f&wDUmiPq(EETr{2>Hudt^s^XUz1^z2e4CmHEh|fT& zr?cyh$``)6r}EAeN2woVtT5}d+=KDBZZ?8mZmLLB&T^7KE?F6vjw@_sBd&7FA}2E* zo|^Eqxy=t}ELbu6Dv#<~beyLv04H2Pqb4RHxYe5bP@jFJ4OJ|~|d{d9@zIIRj1y^tedLn~H zyqHrv-?Ai<+3=@&3R$rib=;O^a3fcV?W}n1f;c1ZB$gWqb9Do*kBK~3vLmVhS+2Xj zii3R!oa$ec<@vg?-b*|`mSEKeo_wo1FQJ@*x}X>IUo}rHvl>ymx2SylD+|N)q?Ih6JE+ZOy*9ZQy!?wOME7^UPmYP?gAX#bBAdyWG2j-!@Vg z252K~nGw6IR_f+xb~Bg&eZp42jm|giy?ye6obb%?j4{96wQfB#8$Zld8Q^|(9_uO0 z5xD9ye(WNu1x~`EQl6h5Uy?KDm=2X?BtJ8ua#eBbF0=jtWYvpw5L3~0bEwh0i{rS& zqjYw0^fy_dK9QL((Hk};x~jwTd`Az;>lB7dY~e1ZA|!Deple4POV;7_S|q;i20vxlzG+8ptU`q*(jCuU+1#N`E3`_IqBJ3;N+i0-i@O038kMGKIeH7*3?I=C>Y;rk|fZ&nI{W%X<7uWSn$p?*&x>s36Zcck=be_t zx0x2k<3BNS)5^preE07}lu%QpF()e$kJ&rx3T)~b^@LtykgUQlJOU(8hsl)QQi9Gm zf><7+-je56t1og%Rw)O*sHP;RQg6epe3U!R>PRXkIJlFIX_xo1pOfgX zkzd9;>zbaPB^A{?(yS6M7cmG#v>)A^`CB6DB#eDYG>h?cK+oyukKGv-8=drLhxk4Rm`14oGBKXDt zdL?cNlj8OvTp9Vvt8M6a_K_I|gDoeiWz?yy>N@>eS$edBcy|*M;SJO*HuGq;Scbwm zuQkU^b8+$4t!I8VhcjP4FiXuGsVu44R$84({mUv3`+5w{csh9LC~mYL(iKjoC2Z|c zFnLpuWD0!-hjzn%b1|FPsPNA0$9wjS{PrETDw*z$d&eKw^giOS85>a+soLhY9>V7j zqqFlfTeY7=;46-YGxa=m=l?##-PX#RC{Fw_fsb)-%eJ?WgmLb zZM+%xRT)lGUHO9R=9VOYr!6OybP95^KhJF|l{7KaE>ZLa|KsTm$F=y8_qofO!L(Nc z_#D(7bblta@K4O-Hi-;qXyu9919FApn~pdg zm1~9sg0jL<3O;$S|fqMyqT*?RWGVzu*w6pxo_MBS4Qu z@O%fT@vvk!=$hZc5}dHE;y#K(eanD4b`(~mHTr2+cY@jGK5#bc@6=L= z#B(~hyE-V0Ltw+g^mH5=@5y*Y)oI@K3EmeQzEmh!*hi(HcYdJ`fjaK8C!smdtpdu| zLD}t8aK?il>cN~x;wKNW%8_9@FcXqWRj{s_N-$A_;7X6-ss0~0r;~HSh7o6;OA+S_ z>0!6QlMkc^IjW1vE&OIz+2?kUem0X?4d#D^l9C^FqcjfEy*PoJtB1^?oU<;W$%c_t zDv(cx!q}w1S(@5Dq9?-U9itAeH!bx%JB__tzhPc4t)6UB(Gm9{TP9UW%w|{=FB{nw z1k#zF$YR^rbv2H^n_c>AZjeDg!vAR6Z3m~EdbgWS-01sPYb}jbjn{JWW`18w1JgKDe zNn$v+gR&Gp@FJ1Fly}h`R>wi9tV4%eo3ptObTmvogS}r&|NEV1J&+EiylRSjVXQS3 zzWRcih|bjrm3F&XDnEHskJK?Dyt&jSTjtigKooPOFp~viRY4~Tu6{hd^Jd)Ci$VTx zV14JnBdvv1K4%BhuX#D}^XdZqay|7)?!e5?!g8!&bCA_8U6N6Xb*f_BpzM zM4mNx19~u@9+yp<+fiXQqe0}5TqY$Bw5#+xUWdaJ&Qy$g0h2I|=Tm_HI>J3`MetkH zv*xQH+&D{_0IjGVlBu4m`6$kZrM%nD8KP&IS2~$=p+<+Z6|;@gU59#pyA91d9QfR` zgvwJ9Cu$qV(O0SUv*3y&n3BkjzS96-ZW{AiHK*2pP={fz3c#Qywns<;au+iYWQI!O z-JF?UPRJbi*px6Bdn_BiqNZwqRx*^ij~?>VIs#(KtXiAocuIp|o14mdTx8u**jvLO z?J(ucD0~BJnGIQjpWL##;E>E_4d#7KVQXbY&fNwy{cu&vgpr*B={l5-#)s(M%mam|K6fzc8bG4cS%K?gzjZDCu#+Qz*-+O+81{y2eS!+xxi#dKaj zR4%HztyCL%oBfzZnGUvbbu_2o7YglVRY;P6+jE*PdXvP45B#OL(+=M60f=-v zc`^godkCNKm72tPd8yKJ66dMc#Q$jJN7r^(4T3W%p}yffx^1=QbaVeA>Q<0?V@8|4 zaujDvV>!p!?72?Qw`5kW%K8{!jjbcO0<61F~5g*vqn1(r@O0 zd;?7d@zeL>bvnT|h9Eia3G-|=@7<$r8qWV}KEp}&{JhYWJe%C|<{jH1-u%-g6Ut0F zz8TEn7Po_#RBB@4N`4UF8EYa_hi6nvzE=b}b}lD$m{bJ64N{HNB<=@VOw5dfff;FK z)MI3!nochep-RJYJO)SAqK-_(*HJ>6aR+W&Gt2YD?ZYna*Y;o0ZdE6peg?d2;ki|U;ZKKpl|Wwx!N!q0 zOx0bL9rV%OEq2+d?S)hUn84F)JU9!YE=nG6Mjy9}n@w7(#Bu>=OM1TPWpMZsxC?(` ztOjakY8-I^=s0&(K673N$V>9=dUcIj`N_&GF4;994v<&4AU?q^`7Oe-@^4{u?FQo?E%?A+9TvY5VvIS3XLvU0>@pQBT+Mx z(?iA47v-?Zf|KH@5}waxaM5WT)Zam#Q>7E#L>AD$ACqO>RAcrTwpD&~^gZZwp1|H~ zylP%WI9wclexNEx-w~!JQyI^xi+UA^Zm4ddcc}?*u=`X~vduhokqs4{z<5W@3f|~^ zx{0?=kQ0sfA`R}BCUi#S@P^eP=ga`}|G*)+8jR|#IH@F$HhK;{dpY{(5>)wzk_vy9 zuYKDc=;q)#u3_)Q6u8#Bc+|Vl=bnXI4O3mH@`K>#PZJN9nVQ)pi{Ug<;zm0}v{xn$ zKdbq46D7$t>-ov&)H?1a{;KY%J61K`TspPdo7Zkhcbnd73NzbvmE8WHcwO|_*n;&Rj}!vtNNLcJlUn()%ywX=l^{Ak3t?iRwn49IQNSJ1MKSq7fl7c&vh9L8HN_vBAJ2=^vM6?($6!Fejdf zNT>`4yPElW+uQ)#L=s(dVD^slW*gw0Xi29xAODQQ9<=<{J+*{;MW?`W)KT~0f-ha80Bx}W}T67?~Q$)+RV!Frk6D7J%j7$ zxxA;AfwYS(PPv|waQYk*s^L~}LAghU>(+l_|LO>b=bSs{)W1i^PQUL8c zwd8~Ibj6ox!I8L(R-i$gB=h%@gT#Laa@`>lCd^6muQwxN*|v@P5T#}lEe)g*uC@vi zXiA!@sAkV`tTm;(PpuNO$L4_E3Em51S5;zE-SfDAFX_U}zX!-ja@{wz1CGL17lTp1 zu9nhmg~0V?MrnQNG{djrL36w(Ugz6twbD#O>Ha~FRt&~0rS7fP(ocZgq!JFRuL{n> zI>0k%pn8!3vZ&H9#+q4xn|iUt*GG6_Q8*3mOFSJ;u3JNv)^OAv;a(D&AaYA4P;^^4 zMNjIdU(+c=Jvx2iY;QBOd|E0noA1#@QNfbwj?N`JpKb{wG*(WtQ?Z74H`nc^Pw|KA zSk0%Zs17h;F+z%?LH9=oya_h=48KiS1A8PI|vi%4?NrbFn%J!s{+uEu|p%Af2PNHF`&cc7sV+JcHEOFExdUfO#X`9jHv=VTnd`bETf1rWRUR=cBgtW!bS zYr$#dm|=144s=`R=;K~VY5Ra#O*YHT*-T2`H<1l3_rd3W=C7`f{yQ1G^ol=3hwOxGuZN|B2CSxWS7PaTW}}O5K$;?4)W(#v1^tEG-jxwomaLd&uqEQ~;bw zAkQ!{Y=$4~>uUI})4c1;RMG)zJRMmPl^d30w_KuMyNksZG6K1D36ul1$=N7)+G{#lGS6YU*MU^%xV>covx_k@wb+Uch|}ia&Ndb0ez>i zKF=J7veP(|nEObIQqPVY-pd}z0@v4vdGM5A>_6bE+)U%PvbW0zHXH1hO-?$ef-VI+ z^bK{s6^^oE(R=0)k!O~YODN`kI5{)u z224M(JKOw6Z@x#@*J-30`7Qzes#a<)o#r;WlFFRQJ4`|Dvg7LzCkA%02m03zot4>; z3HpXphw~V~F0!rYM45PQcg-X+c4xHI`nnW7`#fC7pHbvfnjf&QGwnIJ0)%eeA1-bb z`!q7sBX~QzG<3>E;Yb^*Yq})evBgw_WmVx6e6~J1dz=+?==(tn`*7QCCIX`P2~X*6 z7FZX^I!BdI&(>Ix;Fbf{VY0+Q-4&lhPn`?qYCnw1X7IDZ#dKH(gJuIzU0=Zp=7$+u zjlPqBUaTq}o6NAphrkTYh^s$pDhl{Sv(Mw_$%t~W);4UkZtc`F$@B=hqWbH1Qi{6z z%&KiQr3MvMog^D?xj0Ht4w*!CD2PTgQ{RS@c?fPPMRgmb{P?vYdwZ&J3VOj2u`dU` zBeoo-Ze9f|PU5pwVz)+39gq17_Ed=Ge#5Ie0_0E{-EAKph~`u}HU!8%^8YJ2g_n39 z=W`Y{YaFQUpjFrUYMr*)TZ`xyk~0fAik$Y4vz(KOMep=VkPe_~%tu$eMaJ`b#-7P5 zenJ5-WGkyK%40e5b{$gC!Obz zDS}oKo9WS7_6D9t9(yq^ug3OzJR1x2O7#h@`;aWd>$Mz~;0V~eF}pOIp~attu^9<6 zNhBA@oXzAIfBw39Os{PVNR(3$fV(4wV#bVK01pHnqzvXDI_3ku#TdN?lzsr7?Gv%) zRXP7dv&yT2tT`|MzG@9~bF;v@eV8XIMEvxUk8WZ2j~>Pxy00GVx#12oyE%#FBq5sJ z4~dVn!sv78q0R6QdrV^K0nXV>cbo$cr%P<4fhD$x1n+%kW0>af2haT0U-9A|hb5bj zGWoy?um*A$Y)Wg3-G$0h&CTyVU<+1J`C*YMabpI$#Lzw<~AilYCZ9s4s6- zTryz`I67QBI+1y;^H7na%|0m%+YpAEVh(IcC#vlw^+halqX*_Hnd*tMP+A>8vAV{t z$Umro=o@^mIeIv#rT`P=15_lOXG`Y0%cJU!aZWOy)r5a537g6iI{vUiE$FA>&>PMd zro^c=3+YeW3iDBh9g^&J*l0F_Z5HUuPJS|HRZobU*I1s~u1l;e<#e|;-{||wxPw8| zZ&8ktpiRG|Px%ga*H3@9huG0{0sHX$+=bZ)kXqETNR@#=)w>*KIWV1(B zTry98l#Ur-kC~u_9ICAphjl-RvO{k|pK_d=c{iI(^yo)b5fuy*7=-ik)aZ4$)TJ92(^{HPCMvC_LIouzC(zeR}Cx~*YS=B+NxzJ>Kn zaSo-U*$=xOD5GJP+A;@G*3<@X+oqw6W=beA-?yC1RX^D&!8DmY)HDPg6lc5UMs{!w zwAavOhshWBi8I&uIbEn?cZh)+Qr}!c-wbD_W(6MC)b@Phc^v3Bp|EAmN&)j$!m5Lc z7XmsT%kww-0-ePnQ1>+=>J@y;OX^@^pVG&M#V=#R1&lsFrG zHdJ+h#~uOh8oA(y{71)RsbUg=fLCk_<%uU~P zgj~@IlwO9rga(t-EPIEsh^afOtoq5HJV{=P%XfTamVz1j@f|P0+I<1zJ%+cxp%g6W z0A^j|I@Q@vRmlCOdozK)P2YA)F^4h(C%}1jyG4V&U$U{yGF{Q3Lg`^gfFVy9?>#=L zz~0$VizbnUI>T6QS2!d1W=jq&SL9RkB=L+z6ZK#4G7>s9oISjT_U~Bx9?|GI?btn%R=uxomgR zr$Ch@bro}8H$l_6p$Fo{$-?~XIDGF7^dffF-Jm01W%aZ|@SM*u6ZsYi@O+N88*`@m zf_2M))p1$j0{zMTgQx8n;a)WJRqcQwOe;M>IIYzya&;j!kgPsj0^s~h$pSETQMC|O zs_Z|k-jDsueyW|7N8cr?rwcRc%>3R$$4Miot6as7`YJj$41|SprcQ?x0w& zrQS7!J6$HhDut8P{*Roh=`jz$dL6NjQ%?g`33HZHum^RZK8)you+})XdWt=e8NFld z0I7wCVI9u5j?|Qr@ZJS*S5DA7a1Ev9*>0fvH}f2HYn$14fHP_D)ZS(|B?qQ4H5%>? z=Jh_qo^2p-`xqG8%pM?3zlA?ES_pVRR>-+9R0BcIANDeqZi8n#~7iy&@bPi z(y=Q|Qt-@kqP~rz$Gi(_t;jY-Hn}oOdyU+FQ7@r3tz%-M5dY$L7!~b|1Lrh%kAbv3 zp89S|(;p{B2|bXEU0Tm@pQAJ}$%dC~EkEZDNZZofla69lo7uhAa8|x#bze5jd?i*} zgGREl#qllpJOLRyJHJy3?rR(s{Et*38jIt!@y;UsX0khtBsKd`A$P&RYpD(ga*%0) zUFN5>f@hAW4?8&>+e~u4+fV70_S*CHAZFPjbaEm&g9)cEE2LLbn+oEBD@-@h#f~rq zz>7XSheS>xJB)oPvrQb3%1F@lcNmY$@(O%gLN8$!V}mpzuU=tSQX^{mKK{SiID-k{ z-&)|ada3T<$e01H%d9()Nxs6`ou-3Mpq^2=EV#o_ID9U_t4$b_Vt1)k6&oF8biih1SjxNB_=yvBqL>H zhC!><%<;>01hD(_Fz0wWV_WJR7Sfy6?_YQ!BJU7gwKUh z&jq9V{U2kdVHX2o2~tw^o}lk^MjdX!md@e2C?3=aJo&Fo)qTb{yH2|BZdMRcaU`xe ztQ$&va{}jiE@C*R{vdm((LvPrV^WH`vp{|_&CyYAI5n6N>j9_TLg%NBY>{s63Fj*l zj%RVS_J{w8kk|Gw`ztshqgVUjTTe%K=tf5nK~+7fR#5%&;a)3aN8zIE!1u~P$2tNB zX@7crhsu~xcJis7t5CApaO&x9s`&-@C0JGRc76KC?BxHBI9fL{cQ(sBll^u@yRClB z7Mjc^mfOQAXYMmopN)B+q|O`t2Zvi9eTiM}&2?3KsmaU)%^A~}4y26kN_UU~9dL^M zgd1&YnrK~K9KD4Zy(~^2vcWNW{9EuIjZNu)=P^D1dVWssZqBub9Qy=C=?ThCD)wfl zvPW{d6Opxwpf$Y0d$0-@!B>f-3m8nc&aJ+o_w`iWP(Uuh7d7Xb4uZ$-3>V}}PKyLX zu0v}edr{3x(j(6XyLQ4q?5{hC*IoQUrQ+A4Dh+*vgJ*Rr-~Bupa)ge=efkOCbzZ|g z12Bri1iT9?I||(Okq9XbN)I9aD--|qr~(~uP*his%omV=m#=$|)5d$_Od`BMvs6}{ zfhnQd^n*Fn1$9eZwC;m+@=0H-q~%zvtX|yE{fFD$t1!>LpP0Xe|7ef(j{m-gc9s!e z#3Iw3Z)7|r-GufZHvHr?Pw|Xhlr|v1d}=P$YPR%c`s0~?j{ms^nf#km*3|O!ai^Mo z+;Lf0A3^h4^3SQ#Ls~FN%}w+$W(o(+Pa+{J6HgO_+dlYdzU0FJeA=vZ-2U`C`xQF~ zL4#?8KHnOn8-gWP!}P|INhszYP*2WExY|KQUP-6+5r>W7w9F!R;E|r9ZS@8@d$rwS zQiQ2n<|zNUl6t}9j|E2=do`N)5wnmieH&+QVrMp8{1R$?3pBEEWbA7EetgM@$LTMO z;%+*)%XBX9dDmWs*`Hn`3)?w9kXKuHuHi)5f$sT4Uei^NmzSVc0Sk2m)r>(G9zgE3 zOgl+z){(0#n`gQK>TM(Hdug8Jag>)gdIHY3(e^^N0OfXm{o8Sxz=V-ia40g-QwH}= zHqiT0wg>p??r@LE^d;QgZ1*DGPO*Rfu)D?#a$~#ina6MMazCfug*#;={;1gc997`D zWF|j+!}pkt%%2OEGPRV3;}V&`vmJ$g_W*}R4}NA%oN{S#((WhMR#N?>I@1u_;ODF3 zUT6ynx+%-RqDRQ*c{z7A$vo4U{2C_PiQ6FQk1KDeq%o6puymz*CV=trGWk7GJBO=i zI-pf*J^IK|If3q6QF@T~EOvUNp!P-3>2HF=|ET+l51K+X*~F7f!PG;2etsjo2M1w& zf5DnwQ;+D=L*UYr(W&j#g-~ptYut7`$?}qv&V9BjC5hnD@1r}+Q5~fN9O-3soY)?$ znxLcIur^q}R(XlVw|j%T?4ebhNVDOoYU4MEBagvyi)AY6+Y0x-GtC@#p5YrgLoU58 zRY0xxR0*e@9Y@b(POX)`$*f#^Bh-k@C?b8yv9pMiWNffcWryN72*(eQ+z#dDuOJw+ zvU=>l?N3KQujy4)bi*vfNq!W8TGZl7a9@kbJZ(AeJ5*P?*WIwhVYm)ncv)@wno6*I zS|7#v86JaCELn>{m_i^RYQvdTtP znhMfWU7{kimhEgqimN*inU&eD&;>TH4wFGi>A%`Cp~EB@99<;XrXYQ&g1O4+gxO4s zvSV$OF3yO80y@& zkCWT8Gx4;*lh@PDjAxV8Mm^W_*&S(a!3S0a=Vav9M|4cRB^1=;2fFZ{_qn<*c$@nb zOcQZI1r?}MW9jA3J2RbSu&+-zn+ahu*T`=<4wk^*sbZ`1OqBHGnGYlW&*2fuD=*0I z)6@yhR|UK*{mB*aO@yv4kML8q1EW}=akgyA0`-S?{fey`RqfeiF-{v@)fm{VR=l<1 zx(iH8koar*37q`DbZhwq!(ScTc#e3UE|c&$)gk^5t48GcD`?6;>94!+lj6dW&ZUPc zhi{>;DE)}6xF1Gw7i{xJHWPNG@2;lOOB&p;j>N!AzgAJ|skI8V-Cy}hJf_l?tC8Gi z;MGbNT0hi5Ja{`4_ncW{>1E=8C6ZI4Ycd;ti+?2o?D5HAmXv)yW1JW425gM~=ewKM z^*0gd!spP(2g9Ww;cZ4sZ+D)v0|uickza`T?*?14h6?V2R0omw8*@UF5otya&_8x1 zlXS;{JW4kq>W=A^{9A=!k7qFvaGb3It3V)hDtyN+)Wp(6wRb9isq${nN+>7j@mn%K z^&1Sim|pS{8qFT_glYGMoY09xLKphGg!o_!a;9#o8sxwR^4aO>Eg{Uw6#p~ZVh!&H`9?Qm<`}S$} z*5)9p`{U&sWn$UQaC#hNHleFNO*ionUrG>tVj-sZgW0Jbz!t2IbjKC&+l(<|WIfJ} zn{vs0=EMbW*|1d=aB^k@pA<)*?EqrGB=6ZQmqJ%IA9QxPPB)5r=qn2aiupC0 zvpkerzf;j)G|@5YFFeo^GXGM}LZk%2OLV|9`BY^!t8@sC`fBu+MS0p;btB?%D9pxl zy+U5Y>3asK>eQxb)`6ijS;orR275|v@E8my~xj*{UV zaP>_7t;Q0D zkPlPPA*5Ah`CGY}rkO#^jOVtvXmHFdoaZ~)Zt{(P$$=UEWc<*Mx1b``B47D|7G|?g zqYP-jAvk0=r=XaeC3pP~cf=4lqT0km4t^&9)DhdMebQQ$L-&olz2^Tpv?k^iJKLsfa*y|21T@qOu&My#Y3(G_m&_R5_HV{)GJ;UXCJI zjiqaALkd^|KL75nb_2{_?&4i6n`S6Ot!#zYjgjhf@jv-&qp00G`Gkcyt^Xr;y0r?2cRopfo_ramzuysE zk#H=+cHGnYH|*;bPW?Re+@0!~nPEoDai8sF2Xl%4e&UOw5l3RcW)Hb# zEjKtYlbNGK$ts`qVjS1U%tvX@tjB8f?^2+hWT3CLXg~!#ciCaM*Ii`#fi~;v?KryG z1%{&7!b@_FA&* zCvx?C+0MR{zB&YEwL1H6hq#l>1v=!8f?^dlF$s879TqjQWcHy^eXnoX%Lh&86{p+>|r; zJh!kT`i#AS&iykvvjFTU({Oyp2y5#)UfOK zL6X6)uX{;Pqp#2Zk6UgiQBG0VZ-2VT2WmaNd{&7IQVLZ?K!cI=p>@g39#H5Ws>5M; zxsCkJBvr`{GgU$4IlwUM;6;~%RHw=h)dMe6MRHwHa(){k;SAq*DP2NKnAo5D3C7OB8n36x6)H10_c}5aQtp z%xthxP;g)Ps4zUti}XXbmaR2?VY8>ATa?Cgk_uNxb1NzAb05WyTC(vV)PawZnVL62 z6+oAaEd!m6`)$lVM09W_KNBRiY`Yr9ElZooWMVE(ItMlwrHDqx_R z30r#$>=&VNGXB$;;=n8w)pgJ>95{s-PMg=$GDO`_!_n0v=!jdwuGBI%>Ubud5^jx& z1o@&z>f93Pws9MqbnZ>(rheyaa~|l!ZYQ^dxeHTQ9PVWjwK#%FtL%Cl)8)g&yQOWb z-b$w##lId8w4aYGS5e(Sf!(I|QTM=dXeIkqQ}S?g8GxI5oN)h#StULomgMF>bEAGb zzB|%6rOTo{{lFo4*GXz_JC^$y-lPuay@I%OWMOm%Dd7Qnn7DM`-*5--;dd%ZcD+$f zt4{hhdejOgUs~w}=CQ`BPYg`Kt&sr~bkp|O{dE*Qek81d1AY$SJEk$!*bbeYkxQ9hu@b~O=q@zrV5@$CD1N8}<`8e{6KsDU<>&LrbDd0m5p`)M8L zysIkAxqh^Qt!PekZqVsjsze;KlkWN>?hMNw!F=(5I%bI<2KB>ho@AWKDrIM zMsqr>4a}MiH*0k(^9~f5fr{InX|W~vqqm{spTZxwiCWy19`gb}ZJnNtR#g<%eJHh3 z^eQ-wq0G)l+WtDP=cqf=Ty~~8_jOB{(rLO0^XAR@tn>K{1Bs=E>W^_3p<}R`wvUfMGoh^WS3p&4liH@ zi^33&y z=8nf>JU#GM_^^j_y_1KhvCw$86}7_SQA0BN>}4;|OY z(F%0Bk;r1V8!Bxd^;Bl^u9rv^oHoB?6CBG+x`Xj*Hy*%2@GTjLd-m_bPJH7}Zsm=4 z)7xY=pU~@E?+ku>Xoi{qoz$*Q{ND#{hUo5Wz!=MZ;M;Z?_Ct2GkJ6Wg(&OKyAISv5 zf6YdzB<$IE4lmJ#h)tvKFdcl*j3R>-qNd=C!cou#-%Bv(Vk^C2XEj7^fb}99fI^pg*;&2TV>wp5YApw{GLsbefng zK8rm0oGhLU?j9pfN*gZ{*{m6z%>|VL#_1USb$UZ@N`Ah?uE;C!W8V8ir=vf>uwp%G zVj4R&z4AS@jU<-lhJxbgdIRxEey3VxGBfow=|~ttp=^Rw;fN0NeVI-+D<1a-8MbJVXga!t+S zW?P$+*2&tSo9RLDQG=NcEJ_~xh$C?{88Sbg^U!oI&o>Pr2fF{^#~Jah%?EB3;<0ucrc0H2`1h_t;Zo$2k7%&e>^sXEd~k z9c=p3O;k6PCCUWTn;dY)JK?%FuEIQ0lv_B<(Sui@Bi5!rNaHW48+B4rNgA~j&7~?R zE)c%-E$YA=6_5J34maRMxZHx^#%p*xyOUSFH|Kt_eB_|A!3;;Nkyc0ftVXNU_H-r^ zM&Uynt|xdJx}EKn?sSL#(5VQ=I2bHZLN8;}<2Tr?>!v1^@uhAAzmb5RJ&@_A)A9j# zdI(7D6LI$zhcgaHGv7Q$1D&JGf>b|gP!Iie2K0OH-KI^*>iOV84-?gSB$GJmJ+6S7 zQbn`#MYSX+{Ki3jkk~lO8=S=i)*3h>WtxMgio403o@Sr@kZkqBGtI4QS8@|N1xzhx zs(qR8>&AO2KwRyUl%DZ!E@FM9y$uzk1GqGjR38wp-gjWQ_BNjVri(bX=X=o?t>@Km%0` zCJIxduus>8U<152rfpFtV6)<=?)7FD9aappDiz58O(ZoKav=4+Kkl7}XcNBX z85+S&`m!&usmZBp-uq21tE9w0DDzxvs6P4Z$#frWsB0aVVVP@AQVZIcrIOjXYfpt` zO2D_R7fJfX_;M%ix&7xV`` z=#I+69UpC&-M)z$xB<*ZM%!&8*ST?IryH=rYCxy39sxNV7s}LbK_~h%z$rU(9 zAJNVh^0!Lx$?mJ`bo5~=f=PkN`mDR!Icn}O;}g!tyai5wQ`Qs9eQ&nAE1mf!ohPn) z)7)g2a$J+vQ`voFZhDG%X4{JW4@-K07YGnIgBcG+5iejaRbzbp9p1ZTRoI!4AA zz;sNEoKU^^S6-@-Iv==t2<+KQHXD=ye-<;l$Wbfptf&SZU^_cXOXf#X>k9O?Kjj#h zXA^$58r6(ikWG2H=o6|YtjXz=<9FUQ|u~cK7HJL_KsYkYa2`tby8NK)Xe2hs7IXc zwVnXCr=8V_V{;R{Q;RG*i>jAL&*5}e*J+&8b~kX{Tsw>n1?g~h?ZK6C!%7S?Y(kbB zfoIZtlY6+zVD^JaLWw$$G({ z^Yb#4zJ?pPWwUB$w>0hb1W+0GVSDa6=Zv36+)?7IBx|OGlweCD1GID|Qr@>BTZsnZ+Oi?>#SX*uK_af z=8Y{I@B`Oz{*(u`{$=k(C$jc2=Ar*1J|C$wWXoKlVH+MxAy@+LC5M6hp>E+GjFhwF z&nUwV3zX{+sY)NThu*yk?7?I1vX6ywf6ZDXQ<)1ZEJ-DfvcOogV3^NQRmVy>PE&Rq zO2?=`LwUAAIy*JUq5|TS|6_EEOE=z7 zI?2t{{||MFvs_+x2FdIJQ(5*^-4o{U3o~H}sU{iBP*}U1PC0wO?&a2Vrjq?$*d@#j z&`~4(9M?)ICZ|%N%T#ts*c0^+w$Q)fJ`UJN83%58ulmD@u&08EZKei-d_pAy8Gb9Z ziA^dv*k8cgXtKltb&j5WoVvstoX^xojAWIJoZJ#Jn0nGxF6(HJGyki@;H|#nR6PqP zlO8nE+CHg$d=|Px?P{LC&Rw$vxBPKk-1FRd!2O1W$=-{dWp=18;Kp-~=s>p{{DXHh z$znZ#>ZbHS69XPwMb|vm6yuvuQOBsf?^QCGgwu@^0)sp<`M4|9t@Ifn1JP2aHF zQoCn+rh0tr0k{H|>I`NB+0oyeViLs{f8imkCRqKNB&SM+NIWw^kLLfoPw$n_c&^TI zD_MT0mtj*Cs`qGiaTu9Q*FG0-%ypT`TRV%(&fEPphTk3t&e;x^xKKj5zc?Oj)?plX zuhc#;!(26AjkZQHiQn6LubwcQ*;tB#Cr_CnppKEc0@3wahwBE!r;Go&E*(b<@xZJR zuH9N-q-S^rR;iUlQD3zSjyHpJYx>O@RFI?e z6v6as*U0LBmG}Oy02z&f+y{o?1<%a~Z`oe!98Q?hs+RMCjNO{8>`k>VURL%W;GlN& zDJoq-I?#Rmi+R8j*YO3N1b6K=J;47>?I)DE*WFZm8ek8RmXZ*reg0Bb=n--OQlLtD~u>Uf=sN)UGJ9kdKbU1y_}b*{%-2 zVI~2E2AZ6>oBmJISwKm3by@gbf3I)}!QFy;aCdhJ4uQrUg1fuBOK{g9!9BPHcefwc z0P__yvlf4^PSaG?yZ6Z6`|LA?dv_b>nTmQ6=*Ln0ms@Ag>gKSD>8U32QVTt`TTDs) z0SqCzKFth84HMyV*i!T?3x@qCR666i z7YE@qH<+VpJRIdQHJpeyUMA9q;X@M>CH9GX^j1s6D?J;OrlWbUv#C8Ul^d=X;p%fk zoe5?KwNR?{;%TzsfwR$pZmSq<_E>M6Hv+UL0g9jKpaH3f|FcjabX5c75T4GjOx!#K zOHW0Qb6Z5F9?MU^m!2%Yfxgp|>^`ENS-^8RO;_HU>X8{K?m}07CKGjdHXK1~ga5Cf z!#K!n$Z#=@%I7fpsBPX}v05DRVoA2;c)fWwRm5Ob4veb?HNZWp_jF3o?GGZ8;>yIy z8AiPtS*GHZ?}ob^rSi)O+{ZVd5IMLh#o@mSqnD`0tI9_NE-wctT!BP+kohU%7qY+r z8H@M%MXWU?OUHlr5|S^~2WrAJS=!kg$89>BA|=FChaLp=c%FT_^0H*{@DVEH3A;zvZqWgko-BI?)Rw2YI3$0GK27deq@-bhU#gY z-pkCwBYg*SeHJ--BojTIWL*%017P4?xRsm1raFlvRJ~Qb9Ngt}c%7Zm4T%Pc%ye}a zmL#E$Wz$e&lw*rZN0ZgB>=K(qp$mR*+Xr{?Cb}`mKvuOD#CSUT+e!w_9RA*O^%1|5 zYi5SJQnsnS;-=XpdcPY;FK`~cN&`LLPfdrl!z|+OE+>0*77Jm1H>xHm3I;OMlpCaa zKRk6VRm|V&{?ax4CCy-!@81oJgqZ3mr83d*{IX@bx4AL4BzSsY&G9Xj5-^z6lmV+-JC z58KGP3VUiIF$Dz~`uC%VfSVhIWjI(0C^ z(@d*9GuPB`r6h5FFg(Hwn7F;VkIDnuc3kGykKwrFk1F&Mom76%`Iu;UW`f>)MbG?A`6_}> z=_;9J2hXP`3b(s*3$daRI=7hIrv5T2H=zb6k3CJGd4J0e<~V5Bzi|4M*q`16L+D7Sql9&Kbl;@VK>ig z!XdeYndz5u>C7fq*S6Fx{3~|3UTb>bvwEHyth0#ff3c(34KbSd(*_5s6uK%{ZbQ}H z&fyN$@K?GoS}+9wOU*>s5L|7qnv)?ZdRv=B&A@bf&8d^wU<&HIlL;)t++RxS zH^GM1BcQ0MVIuN^SXLo-9|xsb!wI}WzV1$S(MQ+P)#S&pZV_*y;Z*ylsDT>_A)lj+ z!f}ksrw{+iJ#mi?aSO4py{bwi96_%!n;!ifoMKA+grkBUM`pW9U^=3B&j3bkP?o-9 z8Z@1$s$OwQ?~)OA63J6@7w2;7suJNd>y0X>oI&q!kEzBbdY$~4`Qkj+%wI^af2VHMZ+(=5F-Ul_l4{LeeyRN%{uiiwfpGzotk!!`7`HsCamjw zIz?%k>XK>>YCsRv^d@sR^FXB(UtdDgHbqnUK*XRZ67TjVUwTMdCscL$z zS|XzoFOG_k8bJ-zO9_-VOQ;ztgDm$IpS_m+Mi@?5trW7NSp~bd8ZF8$*%5S(iFIz% zRguDMK%E|kIxw5e2LD?`ypai1KJg0(@Jdw{9xb-nX1kaNs=qB~4ya8*ZvSVSg8m|h zt_aswUa#XP@1pox$a@j~6nk3#W}}*QM92(gscI5L^AD08*7z0eRzH`EW;z8i{ne&Y z=%GK+8h?ddreC6D_*JFVzw)K zOJUU;sdV~2`T79W(|_njYEuQxM(ueBrBD^>-mLU`3sA-H!&fQ-_AnW?x zoo=@(K1)>myz9rrx>xdxY6cG31Fkl=T4k<+U~i@RT_fi+O`lthqqhGOE-#^;!~FF@ z@V%&J2${VdUBE5Q@V~0BTnbw;SvG|G>qIYHhz_(0DvQS;QLDfQR@muuRR?rG^5Z6M z5ID7RiW7v6;v+3C2{g^gwp>lENhrko%q)%yp^7$Z7nr>SvT~f_f6@JA43(Mrsm!j3t*6sCrV4ajo5$=^�q7rtD?ii_M!GOa{G;T|#Ve zrm}x2zrfUVR7vd>`p8{=BA3w&3=;W!O_v~+zs4MMwPC&*yAPnXxAFXmXLp231ov!e zf3IDqx3I^jhTa^M^tYKZF1r=_J@~_4>Tjl&QtDKGT-RAAWCME&u%3X5t{gsu8CBRp z^cgCke^GqZfZelfmWgY3sSj?DTc#T_gFjvE)%D1CgWOC0^`ia~cUpfUW)|gyHWX+! z=!H+nI4}V-h`s&kd_Pm2){#wM6V}VWz#J}+F=xRQPUcyE6a8Qi-l!hjh5OI|PiQfj2p+>su76PRadJy>m$jl+x) znKvNjYx49!xeuJSwQ9rt>V>ZCAL@bdo{p@l5i?4w&{AFkQ5i@4Z>f6>6IaVLi1a*2?h-^=6(~0f%!ke4f%8w8{{GMIx!xU=p(0uu zr1o!`ZlU-7A9e(c@Od4_y)pl)s@#;PDi-&szMjsE`7B;-YI?K<+|6rL#naJjAEHm+ zj?yNQ{f8%b$xKuS{TzO7vkDbfd)>~z;*9Bv2hLiRkM6sJ`p@hH3th=PU`_oCOyUA> zIegr5H5k_WsGLUhyMnLLC>Soq=Uhov24U;XXLtpaV74j@QZY}KCmv3fP2u;ClG)#( z9?!1!p)wr-N?(+Ya*@!?jPw=hLB=MCA=IVk#VFAGYT^($e|PGQ$ZC_wBNMUzWhq@@ zQ}Hv}r;YRo!(hzclC2)d-QW^+)ev0KuHmk8jP4^J$Un}<^zIkMSyhAXAeUZ+3i>Fx z?kNz$jbPDxe#{xAVWuD*S)-#KD&sSmTLzWsR68BSa-4}}a^lR>#PpIq;k*V)rc=lZ>8PJabFvB ziL|Ij$Dvn0MO@vlJD^u7sA5z9r{{gmAUiw+VNXI20y-g=@cA6!^*2Ywy;g0d{wYH} zvscH3uN+Gr$xThJ%}2G!MwqH(*r8;h_VC>;RS`Xm&-Jz{3sWB&VYIi(LN~+sUt(WO^udNMGF>)x`$pDaM#js4NoN7WxbmPxxoM zjEJvhD ztsAl3ES=uQ2;JK_;e*({77qZK|HwKyy_@_*MduoQ1j`+ zx~b{1GdXmL90|KIQ>L-$Q7LUV{{st}3g>%K-i6nTp+4(T`lOtRW5YeQo^6INR5vQd zDRK{dO-J>_%%RRX%{2Ehf=T_L*?dU-{b!;5)0)fnI`h zsme1ts&cx6%;R@A%B+*~by=N7&celgw(LpwG?`3V$z4~~{ju&VTO;S$S|~$ex#6(q z742JD$}P1q&^gc331tXW1WcRS@>k}{D{A&6^2yW^Yh(lt&*3el`NR?8d|}y!E_kR6 z&n>2sH8{i9nU6jWo-@Px_KGZT5otIZtH&1Ynjt$e==HEB!JQg>3#`LnBP4Nlurh3lVx<#)WPbE(v(HW<-V zdO9{;aUW09$u`xo)K#9~LHUq;I)R?oH<#q+A6-vBUYn$zX$jsq3LI>aimgwm((F|^ z#N5r##P-v8dW^ASQIKAbASM%Mqw8M3K?#N-B#$(&;dK@Qzgi0DzcU6t{NAO#d22)&Z z)7y{gZs;i7qdt0{t8c5JG)`eRsWN;fi|C1pQKyygo4IDXbuf+T$X|JdaQS4n&CCcS zw42ozyvmQM%5WK{$X~_H&uW5o#JycSg|BiH_r0x}Wq-F#iDIol^Ad}9Y9UJg+CMaP73uD7 zz;NYb%HSSK^YLOGIk}r!&p9~+PI3|TPA;B!1Njcmu90dfZjhDLE>&M8kVC;kM&U4d z7oGnXv&m#s?YVc6?$8buOZV>tg21NUohtujX56 zuYqSSwUN~rD$GryB2#GzRc>mhvvfcsbaj~)mEbjRDysFLMQ3p1guhTe4m z=X|-I$vN*$UzwcRx-fTPfnKV<(w`<*IZR<)N4){Dzo%+Z^Y51BxCz@)FV|I*Vc-4- z^3+(bMBgz~-=gX)3Bs8{Cy+Kxh&96n}f6GW>udoT&HoK8C`~v1~AwU1G z*3^Mt>?U0a9q(->_#)^%|Ip1r0t24 zs#~cpV3@abhM>QzV}A7q+wuA>EJ-Q4(YAJ)ejeoYUzy^dJ%8%z{uyq>9ls@c;Wa9V z6lNLf=$)o^aLtW3-MI_5(4UvKx7qN!-85BA!9O3X@i1#vf3fN9Qq`B-9Ri8VOmqrS zN1aukKfq1X3(zMo#HH{d710~q)Tf}(`iNGc6m|Y4`PNJ~QR!3PaT=S-!l>~}$UI<- z?2aO?R$zjuD#-I`l}L70@tOLmt)elH)|}4z3M%WhdL(GWV*Ir(pxY7fgIht`+tLf4 zf+atMgG38{TV{Bl@^Jcfz$0=n$9fVK_hGaLIk-pl6;p6@h&`yFw$L3M#2aJ)D9T_| ztX08wTF8b}&R0b(ScAJ_29-}NB7X;FSAVCEi>=}iClA1BmWP?T&ZOr?nL(dY#bhf~ zb~}W!tv~<{)7=zQ4gBUVnHKcUS*Ys&1Jl^xzS{Zf4z3!x)G3$E9Z>(cbZ)u2XNy>Y z=lo>TTmDTB-7Z(*PIj3dGYft`%Rt``h!MQ6sNxP4<3jJfT2JS-6+L_jd~XJst1`UJ zuznvxfh#8~*B|G4-@}qxg%=Uz?V1tgjAqN#m5l%&QH2dS#TX5GbR6+j{ z12W);Q|+*Hu1EFpNsOiaa&k(cHh`sPqH^dWn^E)J7Wr*o(?hMc$*A(T~!d_0Ey&hC|Z;|jWqYK?FG;<~lJ!3FvkH`FFlbC@MePF4KK ztd!mGEx9Y7z`Gt0Y((Jh)RD2}6K|VnjbrvVrj26D=*+Pm^=^Ul92EHUqnRnGq#7uy zqjQTPXP}NKEPI1n_JK+7t!gq0yp^;4OqZcMctnqxM`bkCz%Sd7*RINgTI%>@gaW#r z8VK%lLv{fXdPJruLga7GO_|Hz3ehiQx54J$czng;;{zH=m{cQ^qHJI$O(H+corOa$Pg;nOP z`@=S4|JOL~PHq2x_NtB`M?BMY{6F2F`U=mbpBhac7g-ktwR@?0xZ~DGL4BI;zBXQG z%rK#mJEHD@3r&*;&}hX{4PfxU^Im>4&*TU5r%6OieJSF=7QK)ML^I;fQBe~H98Y-| zlW|@}PJV5%!~SGvsGVRSaZ#Jp2a~K~o)Sr-@nmzzL8>NMfc_hg#Q)R+>hlq*Je}Yc zZc{v2nv8H2t>AHxiWb_yK7>8nJ2B(10-S#U%<_HI+V4Ow!#eYXpmqmTJ}|dA;x_Bw4=^r)U8;qO{%xXIhhfyt+#7k9Pbw__$}v_*va|l^zbgIaPOC zb3#tEH$j0$`y1V_`ktT0ucJ@e1U8+j=}&Mt;=o_@R&7u$?_-PWSTj>5Lt&K?J?#r~ zhLeNGgB-=|{{r%6MzfO{(#mwfdqsWxJ;S@Ihl@Gf<-b4!mXQTIf)T9{O;B8~r8h}~ zM^ih_b4r+qpY>qQAO@d5da1vARb;RYBzk+w7nsz%-_?vkbR9{A_v*P=Nu-L9-{^^6h#GVk8DtN5omB9H zow!Y_=yuzy8Qkl-Di?bAfN0su%vVKSHCv9leVpxq>fnt%#pdo|wwoS=My{x;Lj7Dp zwX+97bt3!e-7EbJP45>Sg`e-HJJ2^x(g#7&Cz$7cfA@#U5<2A9w=aT;ehk|t^vLgF z0Orqq!HEOvYfJt7{J6 zr54q=U?|yPc97QJXg1lyW-R*ESNb%a?@m)mmGFOef9R;Lm|cSc>N85?Xs(65!35U7 z@SC0J^iSbKlG*%E{_Wzstokk}bZ_dw`MQG)@6bFee&=Z>W_ErCO#N)xmuln%J7h0; zTfpaA!G`qKtw7qx6XjO&Uz>=$C)F#lryr2wk( z+j2RG=utU=Kkw3&_ZIt917^kx!7bk8{{Ch%sCcfK{aa?T-LLw27S5-AydGwjX~cr1FxH*vMY66NgWf-=Oa4$CL%7?T1U{Za%}0E zr>fEKhn{gdDe4m7r=OAAme$2f>>rQyRA7zydtb+tdPhgqU#CZ{Vbl zX!by;`PA64n7dmp!@iweK)sHTotw+vcwS|Z4t(b>EOT5?0>v%b$y8A?U0%(HF`SH} z$aHG#MfL%ReFMA6JVblB+q|JZ@8=4d9Ox>$;J4fzMcD~5<{4DSkL+Ikl6-m@ACJ^F zgDy>tbVj!dV*9hr4gOY9(=jB2iS~5xyD#l9u$?1j6&QYC?gZ8Sz;*~8x|t?gXpo=O zP7Z#C?HwLOr=IU02!Dl{0FRbaXAG_I;bwyt{y9@3w8c+uhXe`yX=Zg$nxAYMXn%l- z$M@6Or@<`$iOJ~SwkvgKDyY~xGQIy$aQ+Xd_|l`S%b^aTWJs;cy7BgsO6exrJ7n-o zWc(>OaNd+%P`Xu7DZwJ<$gyO!@-i;>dMeTE3Eeb;PT0w8{BIJJ3Egn#zADzBNf>~d ze6F`ej2BbABrrcwKzMGzp0|KK-z2lr1LhUA75^tA$O8RP3zUUhY)zeWPh`+{R8zDl zQS^M76$E}Jc{~<3E)KD*H2r%k)FzAJ<_@cm^v=FK2NK?xemNb?Q7&qs^@16Drtrc( zMTbR85Yb&sTdW6_DvrwZm}pCU4$#3~(pP0``dfsJh&z_HlJEaXMktGJWH%1RJIrXyv<{<=z~{nSFZqk{T?}%Jh&OeY&p#7R9xW7%Yib# zj0G2VlAru0cQD&n39Ra~_XHJCIl7gdpj%s2emeIFyvMBQS3@ebj6{92h%nTm5&6v_=_rIGcL1$Od1i@fe-oyxv@Qc`9zUOZ1@u4SRIcY+FGT~d* ze*$Z;H>l!2Fh@eAgPpd1aLaWtqk@F~bn^#MeU|C$Cv;nNuTY|(i`^Og;a@jTn0T$N z*9Pt3i{}P4{S79!{|a7sFZ<~lqKke*C02)7__9j5Ms}wHu&Qu(VcwZ@7K54?RZ6gwM9Yu6ERhn zU$+O-OR18%4s-z{LH=h`qjquMm=~_){!^E1Dw~wAD>-y?=onYZm>@QJWoB*l|3q)_ zE`y^Esv`Nfo}_<~&6r$1#EcJYt4Ig~n^sowa(a=yUZAq;#0E7Jh1(onVR(y52XKH? zoXq>)RWj#5oDlk>`q;srZOCq6x5?fHWpbPipDM{cQU7U_RMJ6Ah&*6(F+kr6z*J}Bs)G5JDk z{R;L;5aCC$ll*kEE(tPz-xxA)S=$c=`9thPB{5K4#6;&i#$gD)~JfF&7 zF}I+uxJYly45PkFJ-?c1-@)=A2>cb<4qSAih>qT@h!_AabjMr6nV&2AsvK&(xUD8~ zHe&0u;7)ruMIM@m)6{i&xWCtkemB%`w7cC+OSP3=CJrcZch^qmF{_yNYH5%13Qxf| zbs|0-qvl_S7kwq=zz6E8>Fzq1PZsV%3-uEk-3LmO*>|&Td(Zp_+gYZV{B8$KSvZ)@o=YHagjV(TJ9&YAuegV=w2w6cHx6GGJvVQA*d~_iP5eYz%i?1s0&Z z%A$WGO5K7-TdL-wx50@rc;WU?rSu^(p9_XkT?qHeq}FNtxArT{X$5r1x&8P~n}&WC z$7X}zFV}@SWD;1-0Omahn-_k4m(ry57ui|*Sy0)JZa)W|srE01@&+gDZ^2%8TODeL zBF+zWK-DvpntzKq%uhZr1%s2WiP;|Hr{-CYw&#>Fp^1JhyN+j}P5t1F8(=DvJIdQw z!F5!ln}aFnf6fr|OY015qh6>3*UG+A@o^w5s}i7JVhWRamYec6&YU-;rH^_gu{M=U|ZX28G1f%7ea%C@XLFGo-fjn|J= zJ*tpCGLxK0M_f;))^k)}8Oc@w4W3A6RE-EdiF{GSEY@+=R9MCD@-)ixN96g6==opB zqf9X239gF}&0@kLWrg`Q@;>!^HY$hZ%(2Dc8NAa+URc6kOHm zUJsdMYPU@ax78DrV5N!_l=CO(zW#PsUK^Mr6wY)>PeC~ULlO4@P3$bW-!=pPxr^g^ zZZ^z5C(m{@Kbw;vDwoKSXVoe0<2iL2O=*7e(O|*mMdI@jZ#lCJhq>=h#W_6?{#K-H|3M;M32lI8=h3}=_1eR-EuqmbiI7XC%;nNN7eRPCPRgC z7~~|e`9;qLP09d2_!D*49eCt)tAoVDltjXcID@ri^6Q0HLuouonUOTi)LitAp@42Ftcd_i&*;*Ca8`nIc__PqG@eHF{xnd z*wQq4~x1 zka36@@%3tys4e81ALoCmTEH#ngAQc9m_SC@3UU=+$B{$58l3-gatzA7ktQj%Xi++# zG_o@pVV6iu&HunFE!k@>!dp0th;4GPY%OXM^UKQa?ll?ePtc)DVjuin4wT7L$Si@W zLB$nGj|WMvj;r$lI`Pk{nRL3Zsv%2&lfDz>nS{(kBw2xWy#n)zU&L{h2L`+sDDgAc zhX@e;IQ9j9uQn6z0XuVZ<2rSh^ES;4rM4++M!G!IInC$}(woXm8Wz-1V5~Zu8X)is z%q^FVa~RW&F-KGp^lZy?yr8+OWag6#E}J(&c0aal8H^|9e_`WEKOH$#o$tI1H3%Nt zjiFzHW42~+iVSj-%?PuoJU77(Ck`gLUrhqg&Qx^7---FFf|Tf=@#h4(gO_tAY2T@$v-jF&*?gf0ip`niFrAGQn{P4sa>znpQfNOJ{#V6WWm9u^TjdGZTuF z*cvvOd_7;h#vf_9SAes9lc?Sl&GS%Mg01RzsrE~9HlD&hbm8CnQpHsM!zgr8kBJ3{{hVtE0Sc)2~I?^7$rg#X%MpX*q`ad*s|3A$2yybl`q`Kjca zfZj|D#R;BM!&jkFO31yaZBqGz=njen!(3mJKGf0AXQPIyf#j?WUb?INhD`n!Q!LmI z7FsA+?0TA5sM#LjWUnn01^>euOvvgX6%wbUKr0Bp0Q_+m! zKBTinh#$Y}bzp3BWllMWJbRV7t`#uFYp8NRz`P%WA-x1T9 z_Izp2Mv|yTpJ$0}Q{lS-x^V|A~Zr0dS>KGZYnunx;}O!>_8irLKg@BWV^L*e0I0n-_08K5mYl1_-Df~SGFY_pV$!2t_NMEk zdI$*HTK!b+)p$I~eqdQ|LFS_p&ugp1OiS4*a3cU2nt=wn_=bpX}hJI=cb*`K#ZEBr<1RD>n6nfJ}3w7nPj>SCx3yd9qB5kyL#xHKf%@t{o{|a z9fQ}@?Mp%#f;%=|q_IIYcY?0=nduoE0^2<6r*vsdq@b~@LG_X=jHCMV>=r#VbeqmN zQ)sN8)$a4tx`bvhjM#NuBiKUEJDv(^o*5i`aJ|eDFzU0qJldYCs-YW>hr!^u|Dier z^HK=}U<5I?DYbNBAw*H{8dz>UIhAb`@%10}Hd>lPF0rdhe)(pL>qM~XW7TC4_*!I^ z2=kj>0=_*|UWJPcA&7mPW7IZnxwjI zq!kR#Rdmc1IOn}#h}wD=#BO-fpI~gSpd9E*C#Z>X9?Jh+#Pdqv*w;l;`Iarbw?Gu@ zQ&F$x{MRM^XGArToy`SIqtY|36l3VkC+K%-iL3+qw+4OVS2cxRW-EHX?DWVt@IP_p z7+!O+bbh(m#dGY^L*d^>%b~iTF{wMgjZFEm?O?ZN( zFoV&=rZ8jh2~2D{qMd7_Q!-ibH@~wIN&o|MJq%p*PkQ9_@Pr7pdEQ$n#)&>&Gcx8kF9oOsew%c5t5MdSLdkg)c5fC+gLlN(Ld=W|)?KM; zikKcD5uCD_m`}W>58(|S1BAbjd8@9&8D-UnKnS<$fBo8EFG+(5uA%8poxjTLhqrxg zHo}iTGYNxv^r|&NS7B_Q1`RsM0T4Z#S0B^gM*uHzA5dm0bjc9H*~+z9pqvP zvUsHHWQIMVV!^*QduW(n)^?@lpFr2J0t7WB+U9bmZg7#WqXa8m5*Xgc)X(wZx&F{q zQFy*pk^DTarC#a3a@X~2|Dn5}yMgm}gWu1utH>Ta!8E!g`$j*j#U=zx+t^$KS4ja^ z@(S%oE}cepr7qYde&$oX=N%wpcEl&li5y-S_jnFU*}!CHC($B)&ujXE1gg2-NL=rX zi_|c9mX~Ojqnn0s@x9==;>+fot~PM)XqIt_Ig7q27KmGEF_*hgM&E@gc;M}prO-KK zB6B2E-^oE)WKVofB8jj<_!CHYE+&P;eh9@>3#Rq|W$s`snJBE-`B%n4i*lb`1~2Jq z=jyKXfOqH)X0R=~0PI8f%pdZyQ=jD&PjQ#|Ad9n`6_i#)nBBu!=5z3{qHsJx;Js)Cv_0sKNGu|2E%_f zGHjxfKIrc6d|k=>DHoAd)1!CAdltWx3vj*?R3N{otm>c%$u2X>2i|p1{j}l*(=K^M z6j<0J?5k|-N$)nApeo`zh-gUeS8c*72vv(Im<7b824WT2xw+WGr?=RPE^(~(*k&vb zd8w&gPI(1o0-R_{!Z1D~dTbZ-ytFcsJnG%R6Q;aKBBOC%Mv_e)iZieb`#YA`CbyL7{ez|IgzMCC>h=zP zT==Z){!{ZM=-_9xxj--L*=^_$-V&>$I%Ud*{`TA2wr~oU^p;S}V3TbUI_OWewIUr1 zj<^PqgGh0L0+F6@#)pPt1Z()OTm9BBTw$Hhr=YhV*`7ns9N7#D())u_``&b=V%S68+za#7lbt3h+*sWF62Y8qCnGe{ zM^HJ&f>j?%7qx=v!r17wI-$j##H7+b`A+p^uA~8d$9%R`J~BDzp%a2*|3ii5A-sSO@Jmi~7c%vx{QD{GsAbr@S3y~k46-oUmaR+Od=>`J~Ei9bmJROMe}3%kyI zlhJK2vqKg~vEK+BrWCz!UNEi**@PO#WA1?H0@{;VH5F^=f7b{_KY0rr<_j3+GHUqw zB8Gg!u3?RecDuJp3=>Pd`l6U<<@H95kVEXG({Ds2f1R4G0o7nZFs_nR`M6ShS{#?<}+w#aWgse)?aI9hF;Puyzon)MtF>3?N>7=^o5SL zHcaj?TQ$-|YMkdVT&dl(&Q&n z`rk697mACJo&-kGOTBj5)zIb02ov=Ku!|i`xW7fYv6AkrhR%nAcO~C%Xpc~<9598| zF1?m{*bn+VvoM=ri$03Z^lqbhvRP3?rw#_fAY5dF58oA;(evbHnligi5<2N8vN_l+ zQ-G7#SNBJguud(MePIF)$o?S45o!&c^$>LuWIiF^Po!7S5wtVmXI6vGE{f}9Yw3_J z#?K64YUZuHs+z%y#-&ny;|&F)zAdV&*f@k#LUWZC#bYxxhk;FJ2CHFUEG1PexSn2e z4gEqMyz$nC|CY>$D^%Dqb)0A~Yr+U`q^7>=^<*dEelLZHD>{2gsgoK~6{nLSn1U(% zD}RG);ks!CnJg%vVz?~2Abj5v__CsEB*@82Jx1O&wZJ+5f~VUjZ{Y*;p2{a3jKFT{ zz-Q`-E9G!}VX~~SoXqFZ6-GO`ejyxK*LiUUCDLl~T|VU8MyT3!SOw^OJBZxq4H}AT z_?aAqJx?UBd#z+qG~N-SplI#A^fq~Ez2&gy)4hM-6XMC(`U0wt6!5iK#4}WLV`Nzr zjCWK9Iu!CPGbS(5U~QqdS}0q9kA=IQEutNjt1T8f8T!ByQxUGaq_QNWAgjwQJt+M!XGg2sAn#Lw;!Z8 zUBK-58ct>`e=)u#``}oQ5t$R2_kI#E$=pE`SBF{k)^J!cLKXaIwhP&$n(Y%@a_eDl z54dsWN|4yU0o&EduVt?W$;lu(Xy!7Q)i}9*M&~x#RWk+1COK?-?!q8Ag}GE{cj>Zk zn&|#%?&(-Jz-HAy2mkW7cH=McQjcIRcR}sulh98Nw4)R9f=9YCLnkT^bb#Xl}}aw_4n{7i9?9(F8; zEXMunC|2pluvcm9NmOt#%o;fE{+!Y!BD{OFI&KLk^n4}($CLZUaibT>;(U5WE%Gn9 z#?+j|H8{Z{=CoV_Qf1upDi8G@l=lCb>(o4H&165Ld!&C34WSx2 z1G{?Odr`VsxnH(p_>XLe?tO-(aLEx84rrv|nzBHx9yv&)I;oqf zr_2s?my1AVzw#ML+iGOrvt1#BU>4 ziH~YOw=y%4GrL-hcDFTE`d^@hsll^;mYMW3__D3!mz!cHdy^y7CO-jPzzH&GWSxW9 zGE1eieW-#Sf&!hE>)2K>Pu@bOSNeySV-9X~Ui}R8I5YLnA9&eKCvKmm^Ic}fo4v9b zw``zz2p9YhO8EKwE7$2^cToXG!0|O9mn9Z#K@r`(GThtdDUj)o;2RQaG#Bm=$;taC$l&$lSvbw!GKs_0Tc7otf;#RDs-bzfj?+#e)My{v9McB{ z!VD7~+$XR3pWhm+W)ZjGmCh9`0pV{z#nK$D8oOROfy?2}KA3-~b-^(WJ5=!SIZnl1 zgpQ*IopL)6p;Gj-pZsC2n0d&lenm$d%inA|gg*Iw>{2v}CGe;E%uG!`^m3ET`(O-b zG8CGNUg8aIZ}DK3C%D+=C3hh{Q!synFI}S-z=_F4aee7|TTmyT^>_0#hyA7Grbj{I zAf>Geqqm0*dcG;HwovUHGm-t5FcDvwW7?~Z1jG5x@lg5TPy0CN;@>hqg*N-buCRN| zQdKJ`M3=DxE`FXE2K(Dr+)zW*Z!+wok^+V+EqGnn!R)w5MDA`S6Ej(~PMo77C@3>B z5&Ro}o@L&)H`(E)D8y;sCeotF+ANmI`cyKnh?br3&RvQQJsCRP2l%T5RKjIIR1@N) zS_&S2hAKuim4TE05-jm&m5OKk8xxTQm`0x9-BFX6w#z})kVh=VHFN@#X%pGD6OBGG zx#$5VP(da`5$(%uYBIe~ZdibLawIkUM`k_JnTKrOW+Mj^*=KE37{kmk;OE#%-(L-; zw_>}TiD&+i*YHF=&uqd*)KD|Q6)lKmKb=%AA%kS*E}YgiaAPB>X>2 zTuoG==U~Z)ksZ#eD(bH2$xP5uyiGEhnrah>P-pm#$3*oLpdWoWu}wHXtgkO8OVfjvxnmv*qYKr<>7%@+X^4%rj=vtLT^t;(2!h+-`(EuiD8>IMy!F zi-^n_^hgk%k?7(g5#f^uGr@M22O;YGGoiG>4todPLPGPHx}~#j0?T|*k3sV`#XR&! zapvE0+S8(lUgvU}n@nMc=iE!XD`c7a?yyeINuCNq>-jTGC0vHsFp8QkJvZW=ZLK4~ zWS%qWwnP0cGxBQdnBJjM!8E%g80_CO;ZA3ke#(tFW9o&z`Bm&YwiG2Xw?p%SXLfN= zA5Bpw^aCx3k4sbyrc1)x3PymG9+L%hG(8Wms$*dn1*)Bq;;NYkV$+*yQ_y9%q4#Xh zS-mZ?F#SCPedaeje)~|9ml5-rIlV3aHiikHm8ftdsh7C46<51)(s)bl(azjZ-@umo z!xeuNl?2*nD*qDrM7@XoIV#eSjhZtN|D4+YC;H6}@cw7@H2pUnqtCXHGh!8!!u3%C zj1=J=tZC#87~0Wtryi->lUYtt11BU>Z3YQ1jQ(L3QSlEtmocWJ+HT|9x#}@4ZEs|J zs>GFQEt~7>soSWX|6e7WMm;xu&13oB5A^0w-H96!_DXsSF0>QPZ4VUtIni}D!}(zX z6JEJ-wcSg;>mq-bz8H>*?-y9Azt|9wR%xrJ^(_;6hlMJdihcJ}8+*djU`dX)&K+veN4O!p*qU-Up6MLy|{zCq92 zo0C7(JR|bQH(5dTtI`W3@JE}@e7~bf5SomtX9K)dDILw9%OtP0p21ZYRAjwe?*>t0 zRKe#V#DCqu-0minRJe2m8(k;kg~E90{!oHom5s#L&&)yU@JObQU%{Qm=eadikfP&G zi~en=%WUF;9~CvX{b{%+E}}+y%uK;Z>a-2arQ&iYm*$=dg&v^C_#S*liIEHzBep%u)Yuj9rT=m73j5D&C!GQgtwO0X6C)CejuNtb*ay3d2K78mb-+jObDd(rOZwp zv=jsbXF)jx)Cf!-ho)y@h*_!sP&YNaoob_ZqoIptN|W)Ao3G5hZnjH}(Dz|)*Mr@B zLq*-6T4%3!Q}(0kDa73SB<~e5Gp#5mV-P>zdKbKAKf2uvq6}48Gr3qcrV?n4R+CNx zmFXL?UynuUv<-b`d-+Z6!*1lRq^c2fCqzs#hT;FWj!$6dSYHAEd$N)6^aSMa_V2RAnXPNY85m+zRRT235T zhP&4vYN;rJifIV7`XYLSDL=Td-Bd4CMP=AMU!>%{c%l}{x59J^E5r$}h1ZB4b~XrS zV|cXjxSC&}Zj6VIQxm3&;-Nb^g7Pi7UI)MP0KNYhZu(&9!_(xH;r+k=GGAF6ef}dj zuj4YlZ9!eyA4al^tKe!j;%8+Uz9C|vF({?x0 zIDr|h)V~C)z*I|tDb+S!&;v}#;ce0jJ|dwzfosG?J4BzvBe4eak5TL!HHW)!8{JV9 zH&Z7>3zXYjp{hyAYrE+xn3r%~k@X;#$yQe{Z5#7Mj)e`p=G7J-yv$x~DwVZf9FW_k z`i$EQ4igb79+X0BSKaSn2GQ^IG%0x3h16+u?_2ai>anpV6O(Sg>r1+#F2b(%O!U+n zIn|!-V78dCYK5Lod>+83x$u0Jw8_MgJL2OrCxYM49(7F!jR;lV+Z}YpQc9I-y@0qKcV=`igvSE1?GLWM1>`w%}1b zK`kf#Pf_1tk)p!?_A;qdBkc6IeiHCd?I^^O%m;!7>!Dyy2u6cnTWfPR;Gd%Y$V2b z-j5!6IemXi-bFo?QI1Btl@(WqHu@8aqp%O&ZaT8p;#cZSJeJ6A)JrIQo(mt(-#gw_ zu?*kTlCaD-S2_JwT4A}YG|8wfeBv)N!!DT_}we4{*wp9 z4f_!^rim*BB2fjS~rQEhJ2>?e_dev0O{6dmgfxl9iLUtL2Tf8O$xvq0jmf-YFyO3$?{0Fz@)j=JTIO-pMP+;TRlAdnhM&(bX8D{X&@q z&E9z2Fh;X6vkj>1SZ@r>*8$0Ha`8glR^iE%>@ueKik{)VcTDWUNAx^h@^g^RC~Qo-4naOb+<_4zhlm^G*zUw-XdCs_JT^YIphf6(Tr!iyxuk8uisr;H|KIia(PT~7*9Igv*sg4r)k-+n_qOxAAH!{z17>uYb z$mAV=68@hz_{?)N!}x{s9QF;EBwOq8%vJrPvZxjGl^@YQMc_liRxHs(4De<%m-+_% zb5Txpq2MF6(=Hh8WNeLj3ue_FWFsEl(H-3}eV@tJIC=sINh6#Mezk+}tnOiQ;(Ig5 zjnGrvB{Ze+bzGG5Q<-x5Q+-hXsy28(-cSSO7j;v0mA~nndKP!#8eLBTR2Ea^19=|T z_e$_?bLBPo<^EzI9<393erMopv!cHJPq3>K6--{eP%Xee?YlllUmb-$_bi#B2ff1! znI9Z;rzna(w7X2VRsB z7f1YuR2(&5tOq|Yi5hkhYSr&l`w7)GX2ZY8c(%HU0meIpywHUk)K=Z*zVwtA0;h!{>3+uaiTc!vxGh zFX7WQUGgf)LQ;vAvKjd$2OV)ou~w(h=!b9`zb6lyPn@WBMD-HH_TzK}ubB;usw%S! zERFhXlh`3lM6Tp4&9p~MJV*AbE1I0q==a{E z`pAqg?sI07AD~=q=NESo`aM`j9#nI2>>^@*YfvEPGSlnLN4?NkrS=QB);b9Y?J&Io zpM>(dp{Z{w!U5naD~6+jJ_hfZPA``$)F_oi7Npa>kCK0|W*>{|;Kpl*f5=_g1Kj3~ z_@@7)K4d1Gh?JOf`Kwq=6q+w;h3kH@!#)`c2Du>K^&#e(E^;I0<4m7c_XhD0_{gLX z`{^I&$O$+x!lS|^MV3|2wn=c6S$ZQ?_d4`5T~RXMSNXZAO{o6|upw?ZSm-wD=RWww z^$9QjLrR6-+_&3;T0Epnq8_cj4`j#LR#ZKBx>X5Uo^cZf;NX;XSCIzoXDd zBeUp9ywjJa1d52YAj55$7(GVK+@DF|8)TEwChSR=4NdY5I^SV(ugz_@pt`wA^;-%S z60NZv0)tb@MCE3!&@1rOd<9?DmiRtIjQ14P-79YcdZz)L@)lsZ=%P?}3;@g82D|+a zZbAe2|LEQ!@PNx+4YFhcQ38f*lbDD)HyvEVN)>=BZGsWVD(`XT^Gk&~q7YT06b@{y z4tngm=&`$TQ|1Y&mN8phgU=WB7c&US(Ci%{KBO{3>6Oq^nx1O2No8Kje&#-(^Al5m z`#u<7%{nSE6Y*P6*JS*^lFl-`s$}WHyQ{nRK1qN8gTvq++%4!}K?ZkscXxt&2<{%- z-QC^Y2X}Y)-gCeEW1hJTgmd=pu3EL$sw(?R9-*twB`(dS2V8-IBH{XKp@7#QqnB}( z;G4~+hq&5JY;L>7-703d``kt|>FqKm4t1k@;-{WW@9ks#9w*)v<`71p12+(Z+Z6fv3kievcq3}DwdIxTh3N&YV7 zvX3NSl5aaR7_0cQ*xMkViRu9Q;~e67Th7KR>dJTWo_M=gwV)p+7U;A+H)W~Y%Dt^e za^Gj^v_xR=4Z6}Bl>HO1(Is(+Hd4`K3%>NEu%)%&h{Dl4TN=NUI*Wip4>?7x(H85tdts33+4Sh| zQQVPlwhFcOF!-)}^p52t+AJ^wI5WY_Cn+Ppf|tQ&8d#VaKg0Hhlx2cIG=~z9p;XiVNx)GJem7T@A0k1`EwAYR0YTNUbm82<}R|6 z^aPZW$7jh?*{v&c}P^6gt{vvkwfD zoT;aAnaft+bYf#gFYdx;I|8QeJy|1``NAzYASrDWTvizzCS&yz+k$huf$8y=^hO(k zll--p&C5%rPz2`lAKps|?)V?uNv|SnRADmJ0odUF?nC}m5jGX>~Q{6d@{lie?B8tG^K7!f@U#|r+39LI+dz)65%Qk1TNa`tM}wE1shYm!3ThzRW^n7}RBPaDlSqG)6Sbm;j;Z^o)YK8@B`XthKI$3t zHB^NkUS+c~GiMW#tq*+HN>E%R%EDv2lWy55bo3umC8-2s%Pzj(3nnDo!>F*WF9Z*52VYIyMZ@Nz5ZAK8V1ACr!) z8+29w#&drJE-nb&ZmrHvwRO>LVS3^YjW7XBYdMd?0e_}S(-Bs|+(O5RZ7%WdVsdW3 z+xa@aZOi(YD!T-s;e&CAf zx*_%KJiqS+)Hi{-3JoPWKEn{$n{e68uk4a%b}PC57}zf(C}#js>l7;I1T{(0(2=o# z%D1!X$P~GC&M0{AWT3bnx{c(ro#?>GO>O*vsFaT_;f=s=JJfY|GLd$qSIGONwfzr& zLR#3E^5nZ}e(XUEk4+?}&kc>N2mES3x2Ag^?7atWDvtbuK^J-)qSRQrKAB~T&YKNx zWDlqGnjJ!Qw#``j_ouzkwFs!9r*D`nq$-o3UN;zfEo*GvTl~f6FycQwGf> zV=}^YQ_qZKKU5~r^+g7jmaa~e9kUN~{V!=vGp2GKV}H71^#`M(~DTjJ>@ zYBH|uXKEPCz;rdAs(hB(#jhNoBj!GL@Pdq#v}!#a6B*Qdu!JLdiR~HX0oZ1UEHbU} zik!54K!7XgA$r4vfnRtUkJ(wo-U2XSa8tMo#?n)n&Xkr1`hkuEHyaA)Jd+v1_0et9 z_}bc-fvEyQ-GTvⅇ=lSixIuI@A9?$VLWk52)?N@-)cmFcJ5otcOLppf01lx6r|M z1)U3}aV*Sd>en1|Tn182_cqnMy)a0*agx_&9(A}|lqtJKxlQ5tA4=gGT+cbFjF)VK zlz{`Z5|7F~5ME@T+U$hkZcYvngs=!p8pZkOI*!U&T^w^e1;CZ`c| zGma7QHR@Pa^NwDn$>{j(dgN!8lJ;z(>?fa$e-~t=SJ%r)UTgxk;rqD#nSHdFS+$?- zP5Qa&!=_$R$0F?=Slo_IrjS%81CAzBZ?;{*B-RGghd$ko*EQ{ClTLT_g?pZD zNdH$)I|}zs37gbrC0{qNJwfJ~nY+>j7HBEzZGKMUOlF;pC%tW*N7aE-mMQhy^sF#kXs*yE*}S#aI|ElC%T(T zX|~Y)v-5>U=6c1)_^{4OHq@LoDxMXR!v^Z1M1m}c93n0wjy2$^TLee^Ka0N znUOw;E}o($KNwvHb}z z)l_9S=TVe`-R|(8uhB2oNF14kimQgJ&cK#s?{Bd%^wv9pwzQLA+r*e;f{=>;{P4CZDScrnuK3#b_ zo!M^Lo*JsSM59hh%~r3kXawE7YF;bz2cGQ$QzO!Gsx#2Fw?sBmAAJ)4PVJ7W3h`kr zoLeZEzdEXYfSku&wgLC!PcXm(SkS6^6L%*Em2zVc`yl@I8PeQW9tT~6z_bD7+(FDD zVI801f0s}COdG*zquXx+^r9+?`zyF{- zr(zl-WtpF_NR;oeSIqdo1=C~oPn7!iICeg%nxLRIW{wx(ZKTiouKQbWayu0hVgLVwAVGso^5mmeUtiv>9nef&dfU+h6AUNs*Hj#PgO&E zW4{2L|3o3op@Jp&~4nEZ8DouJz*qi{=A4Ed;@IPGDPQ^R7lCTDadUxcU% zD!Urwtahe5iJcW-h7{6CcLX`#c1QiM|DrM;fLC^n-K6u=)0#+Uf~9#b{vBR7U=Ny6 ziJ!MK;eIc;7hSsIz-FF)r_X>Yo5GRSlJ4NQA5u@}fiY`jO5=KpZGV{IJogiLt~!v5 zv%>8Tp-;20{cMhs1CRbd> zK{JV*wN*Dov1oyoU0$x~p=`!<(SRagn|n~hj-u8}q37@uHt1zG19P)$avF2Ucj}Ih zWUI?)_n+NHp!X1SM0-*fM#9Q;c;HoPD;f4*NoqF130K3L5iDtq59WL+6V4Cl-7uWx z=$fz16o}hQKgdL9*>f)()#Ej&CXBZ{UpIm6pUM2CoFJSqD(>^LiYoGyngN@Al<8;V z>E`&VqM@0t zC%WeHuH!eD0@exvr~OiIs1QG?c6{r{`0aaBOYU&zqr6OD>RLEciRzL!FB9K(!|Z*= z8_*UcoSsVWh}!PdB!jQVL6-(keqKCaQ{k7A>Xj-5D#=9El#ctY$}#(KL8M?d%Lo~6 z#AJc}-bH=)*sbF((_^^Z%#bqsKzCPcLvZ^Wces00*M+;QsRO8$`b!%)kZH1nob^m+ zb?4iskc>UJBbjj|vO~~U$+z4j4|w5AX_vW0$Pvfkd^Uk1_TgJt24jC$y^;GcKR-d{ z3)ML}gBJXTo}L^|FBPuBVQ=EV)BdCL>OacA`@27r9J0XWHbg0CuY$?Ck5oC7KA$u~ zDd~nEYK2qHNssfdwbPoIRzXiB3-pp5=o*#rIn1LQWxAJ_Zo$)F+F!Va&X}+A5uBd| z6*Z>2SkDFLESFW5`A>Af4Ro)Qtuk`*8-SpasK#i@3)D(b(;vBnlGs6>>ht=(8fvO? zmXc9z)mDjdl8glT&%*~Yjk^}2Pf=0+L?1cL-G8a8XwBP?$3(J!K$3k;u7EM@yh!DC zg?UR1`;iKt%ynH#{PWgw5^p`GN=CZhsF%Z-6y%M&djzoP`JY3g0|{`x+02COqjj8GF#f= zoY@WAATkRqI#>?Em1@aGFZpm4N=-c%l`x5(quNvbtyX<-v}(0b|Et%kvmo=jQj2@{ zR*8w=pOfpt zUQuaAxfh6g)9EEYuP4)2F$)c%mKx?fg-MPk&0$VfspG`1E9w${r;F-~K7ivTJ)X~Z z%D*3Wotl6qzDdU6z|4UQ_Pj1EM|ff@iFTnX$|-D`<3(viXYnU<9Q51IEN6N{9y1HZ z>5V)#9hfBJ(5Dux-dt1ximg?MvnG%jWy$HUSFivAJD%`E~FJ$nVn|gSSe&{ZEs@LK^a=kFK zgt<=_*xd6u zL!f}QVB*blu%F*)pHiKb{W4DF0B^Te^+43MR6$U55cvNT@pF#*(l*mmOmtI2!VG)V zaQ8LR&s7)RUO9P8KWP(lmgkusUr-{d;C^7{LeiDmCl|f*z1+3Bxp#oPd=gLf2_}H; zwL!X!FTSshdF!nP(arXXdYO$jtNF<_0zUhS*a+zh+IgX$l3zNjQJk)m)M_#1sB_jy z=;R_pomMHS=-=SyJOfXENyAV}PY_5jU8@st9_AtkCDB2)7FqQdY)Kg%!^=d+>t&h9 z^Zx)ED(g(qsX239Ks%zhz|V{U;V4&x4)#O8AP>JoGo%ioz(Ug4($^*(zTtyO(||(u|+g1dAu_SEWez0{w6b~QX8pm z-aGL?r|nU+)8KGe3I5-}zCvtKHmg_MJ*c%WfJtaS>7f5+zWUC4G3YyUiC0?!w)sZc zNZ#oO9ouaIQ@7F^;#9uP&L`abf1Z5K_K(KI6teh9b-9h$^Y%nNhj*uQo@vHPQzO;VGY#Y z^gM?SeCz7mm*-&GlV-jdFDty3UKvvd^u9o)!{7T3_4yrqYce>%$?8A6K;bG>{$^g* zAo&-B+#7d98D_zi0*AxNyB2TfT)Ko-8o#2E2}HC;eV_yVmpZ1aQBU3EJw}jsexR=n zg~=SOFMI90B6$6?5kodGPi(YK>TcrB{^8eSn3~iqt?4#8?=JWsgIZOJ``&tunA-7y zdMAjgc@p`0GF6Yilk1aw@n!erwE2j=B1`pJ@I!6Yj1BUmWfEP$-<{|xwbNcV1HslZ z|Cp=N6mGa6GahTgIV{7y|F>i?8)1vr>YO?{+nNpnvj_zR*bG$tk*cYJngV$QvgGtYBO;p0Pxp zqh>fb^AT!`l@;*JFVr%;tdGD#slbzL6D0n(Lg{Vn#NbJGf@|#rtCrea(eda>SS3BU zHCJ>fGa26TNZi4$+D8gZL9gM!wCQlePh^egD#lFxS_^ZlI^~fXPB{W;%PBuK7QD1f7f_ zI+PjgAvmi~xfjVt9mzYB?RKi{u)xOnNO#(6=$>84&`%xS2e~f+l|p8@N%p)=wES-3 zk|}b-^_9@)WR{A~_LBna5{(Xj_{jE!6(0^~^&GUOx!tdIIjWux;E=y14t~YOG7sHw z5ZdBzB2Q!LvNtHUr>OnEgFn~FX|nBKas$WR0SVTZQ5mk1FIubf#Q(PHorrFu9=lK6 z)ttb@ZYVl<0e1+>V_e@#Ci}$=Oy(A_rBK0q@X43yv^`?kR%eQU9#7)`$;$8S;2zO! z(Fe1lyfk#P=?z|PZzEMweeW#QViiz*AGoIRl7aX2lv!B&-I{u*FU(iRBw-WfN%`Q; zay!7Cjw3E-zabRuDHFEh1uF1WfBb$4a_Ho0+k@1ydHqG#bhIM} zd7Z-?XJYUR+rge^0%Rn3HJ&@bz5s{MB%*GI_h?CHN)B^Rr6O9cfK@&ZDwu4e*(Fln zJOoMp8<-=&bAN(iveP>})I)h@;%+TwJzer{nhn0?AfGM)P3iWn$G(7)c(5nni7gAi zltgD>veqgcV(EsJoNi2e1~!b{YWO`|Q$Y3+**c&GE+J}p<|z7Cb`btgnL+H`K)&9l z^TVIIFjQM$oW5~8io4U9$~cYsk!|+y4+FpzS2@G$sRuiXfuD?K*PDf)k@(z;F0fC> z)eUr(OgaG*-Ug^zFtpkk0IJD~cC#Bs6zHutao4h<{VkE_xF&Y%r*<(& zeg`*Y8EpS(c&$ISoSVhua*ewVRX71k@p_)2BdOqrH&e&$HYeqa+Z_#Vr>PEmFo?c@ za9tx{Ez?0a!)JzSV?lui)SLSbg=vl zKa)-tQosM8lFF^8(Npss$I)H&0XIb|boan@6(Ar0om173l(N%#qi5q?j81jdM?%>PoQG{k zFE}|nye{qxozuI5zWLGD(woO*Imc_tyi>aIaq!t?b7;tqK)doN6dDWkrI8k=mZa4;)n;M|I0kC%E`JB*GE<1Vh~V`jMN*%}>pkz-7i7O8O`9Ps66Z^Nx}WzM$6^qEdPQ(;gUDDrhx7mIJuvGNRY~Mxj2AL$y0@)%H$yYJ~Z!gDwaoZJF_lQMUolIUBtU2i*90 za=zgq+(%Z5RB_#zaL%1oK{d*`XKvvMDdDy8Iurle(NWl)$$?i?H!5Nr&crlJS9m1s z=tk=Fa_X}RW*%J@DM#&j1^?bkr;dc7Jg&qW`A`++Erjd;a8Q&7=>`$o>qvSz=B$=h zOmuu8!`bzj4UJ=&^ND;C4x)~y{qv3IaQ+YK!MHUxf+!NoW=@Mo-nq?`n4fYGej`e{ zFT?I61f-g;Z7(#of$9H_dn4 z4#`36t5KwL;P}dn|Noe63Y)sk%i$GbZq{p+6fUe3zT9A<84f}{4zv)${kjAiybs59 zg1HCP17>%r*TNmbeysd^ab~_A}e?NK>Sc`|?qI{+* z^~fJQjVI8NbD}vu)RAzr<#cpAimtIgaOrfL;~hF7jc^A=)6Lu+#E?SXAh$azz*u{m zcv8@gVN>cvwZ;%bX)SKuXnVQX89mtYGnzA-qIKvHgc-jQ%)Q=zjwWiHuK6t{CayKexU;~{Qk zSgF=jG+SX2lfV|9f=PQRlR4+X#$lFP9VQsPP*yokbSJ>+sy+0Pk$8QZauXjDyY>(V zzS2z>in3M)R82F}Y9`2@S>M|%XsoEv8nDTU#ienbFd3)VU6@mHR ztJ+X2{ZPZGg}}NZPD4m0u zek^u7Vq^~1q?uazHJrBr^- zCnnC%G~kEjwvmlP?`kNhZ?|nsk46f65#+pqh!n2A;FW%gDBo+7@qof#qr>4v> zKVZLt7T2ppxPs3x2_HYAD#>TK!mOdnJegM#%RO)J>Q?aoD>Y`@g7skK(b6n&U|&3c_(Z|d@mz> zm%S&rfO2vd^3&OU5FYpjG0?wVdXao4zub{$+`*>$hVDa_e5fPX15)39;Pa1$i3w5N z@JD5*x^IJPC^wv56BT4zatBk}7HF?)KqmEYoKCepnU0mu{YGzQ9oWLp_>SW0&tT5> zGLXt+mO8~JEF*qb#%p$H9irYghP_^A?r z1M3mtpU^QKLN8%wx&^0m=Ksh|rYOWUc$(BY&ipW{jH29&GCIC4Pj1Rb)(B#g>3I~} ztmYFu^DGp`y+rIKYBAHE2RbL%>3K&FLjPT+{ab5~ph-ms^>wD_u?xEoKD*;h9ruad zsw*)e^OgC|OlZnI!uO5)fI=3E8y zrrr~o?@L_Sf8N$NzOJpj8Ejad+Bbz_w7Wm4AL z>8{{*)kP^QjCwnPx}b*F)FTzHUZW^-;9>5YE6Z z<`0NIfQk_Um)M^eI~oU00l2IlX1Z9th9`Ca=XN`Mp=aQ_qri<9$>14HO(w#zGeMmO z`?W*m+0ETa3O6!D_K@p$tGWE1;_yN_Q49~FbJd0+k49JT31(!~lct>E2o-MA*(kY6 z_Nm5Pf&(~z{8L=oN<%sXo9Uy{5v8oG^B1>ZsnZMfAqNhkQff9hN7w|B1hw%y&ck-} zFZ3o7esCY4ST$pE&U$?hZqI?ANe=fL2uGI(_IbZlQboa`_nhdGN)9`zz{W%Hb*x~< z(L1$^^PiXrt$~h~LwNICaK$7iEviiuG`<45E6PzfIuRzAMnn}qr*R!mU{*MypTtSr zuSAIritV;kBJ)55lhNH?>JvEUW0=b-31=)Vk#H~Se>u+HW7upsA>wU2J%AiBp5Na} zvP(RbkDJwB6((BNmf_6(yN0J_vD*T4@|CzSpE=J9z?Nlfn9hNFXSiNxR+*ks2fcMS zyn1zTU{~JJag{YX@2Dmo&>f-;`AS^!lg0OJZ{kImzF6&zd^Z8(d=4E zAiYQv)nqVRSyf4=M@i{u3YhvZOSf%BI*>%;|8tM>D+eqI-bB~CKrtJRPJ3*zR?$HZ-eXvecw6e zba4DV>0ec4^!*MXkSbI+uVuGHp=M4Y?)AY{m<68p3;8{hZ-SQ#m&{jgZV#&U{c_)H zjGh|5X>+J` zFx4+m!&AVz=Y;p}fD`E(x8Nmtp$qIv6(Wg?0`B6cngKS;N~IhDcbWsP-~#z#vpQs& zQq5h175NV(Wg7SH0hpvY%+Dcw>}%=Ht}J&<6rXJ#Gim1WuH%~^IVWgcOh9dE3e&S( z-ZVkQdj|^?#l!>hE^O&2XS1`>>57x04*2Mh+NxsG7xTe97G{@_@lSipJ!2oxVNn!) zyfp84fBFdi^ALT~~5Y*@@yI9FoMBWA;pBq&<};^gx8A2Vq3>@~nI%{*I5oI6 z9bjmC;=Aq*1L-lJIi;N@oAF3eL(}n{83)lio`Ap)-@w0=7bk${?x+9mo4TZ)Iep2n z%OpSF>#!Mwvn!N2582SihvC?14%;@vPSVU1vdPIGQ~298nX#{0qlYs^YK{&kMm5Ka z5(j=}E)L9oAhfhM~Uc-s~fMLW*!XKVero1#Ep$q zWKXH}(x?(VrFQC8)L{sspK~d+e>vAU8yZqa~zbBL2f8~ zZyq{-v*~q`57gV#33Gf-2l=MrsB5ubnA(rmyS*xG^R%=s-OA3*I;HgsCL{tAG7HHP%%B_ zT_0AH$=5U0E}W+Y;jccTI^E~jPs0)Y#g4Y^a3jN2S=f?XYCq>W1+J20yo(E*kfi1i z{YnkkhjQ8tGh^tZc})&0s3)q+%+dkF;cmJMqTYpGlZgstzIrLAWQ2OpO&_Fca3d0{ z3RF+ARARPG%~dNvbT!liIG)xjnHH+8bmHP~$%Mi(R~_Mu1?sLaLg#T+O$FcPBQO7} zPsmg<>0p?;UXoB>b)Wf{(csj#E!B0y&&?Q}P6Y;kbITB}aRi@YH3cN23@wG=&srf_*J= zN;lZK3sh(2)joBA-8W}oDuUD$zGEmH(GO<#-KBFk0Uqp_XtGnMs#v+6P^`b8TiI6+@}5XO^n7D_XjK-APec1%8kGAEN8<|Uj)*( zei6>0EsWn0=>r1UC*9Fx!(=kK{h?Y;2iyy8(0XF*1oHA#YHbC-5C$%-Bu%(Ad(dQ$ zp~n6Yczs+t{_UkDh(*~Xx?OG#NhbK2k*YL!IG6YxZI{$&P}UK}O&~%pA+ODau{x(p ztI58$zI$dEtZ7m@^0JV*@@hJdRXvc&T6(*-aMpKHXLVB7h#0S>GSf>o;d9+VoiG5r z5v7;$({4*?Gg}XlrszVY!9QcKfoqW-~O6kC7t%6Z>(C?1kv zx+;$F#)p!N3&Z@}0HMWa7TI60gCqGl?NPd;gO~h1z5Qr>$9a<9`Ggl?XzNK!yOpeS z9v-YCaV8OcU*F*H@+i8~>1Ww3>&etTh?Ae4Bsg~3tHV6oW5k(Z#Ie~t)xByo9@LYZ zrF-Po1L#T(sYKqIRaD2r!5gc{eI?`^v92l^vxxay0?aFNZVg?D+Hj+uif1n!4#`j4 zonEH9SUO5haS|Hhvnt5XIn89?hj=G;;BkeAwQJGD8+tjtDD%yCjX5N(e8%_5Y*qE( z_IEkWV2!G)gsO#;!d=YSc*~s3COVX^=Og+)GcjW6LG*tQ>Fz7Bv~xlYIIep7SOCvG4i_ z7^We(Zh=ZmT%8Eczk&AmkIo2Y+X+hgKvmLUL|e!~(Ff*#?x zy1eg#chNkePN}8uQd>FjPRFU%$DpTdmm6T$i+t-Za4EH6jK`_3#O-k~pYuVVnVD|d zMxDpoKi3II{b}UfRJWPlzg1Sl{FJ0N8BhP$b0++2;zK>P9#mNzCgdCVGa1gGxp06u zv0(X1a(;r?Z_ zo!y0&t=;&7w&NiE&uleEiE=gJSIgk8S+B3V>%arId?S5L>|qtj&I0&FxXrqp|60tM zU*{xsKH-0(!ULG393Vz! zsOY_5nap~-d_`4>3!d+)Q=l&_pq9)I<8ja=)ep>2*t{Ctse7t2J2X?+IaE5)bUEs~ zr_2%@4TfI{?=f9%!rN0)m%%46PIsdhY_vFRg|2~nA<)E;%FF<%sQ>1)Pn0NMEF4qT zY~t_oZ!$8V@hj#$D&8KQMmqlQ{C8ptMwHq`HovZ>Ng=wrUdmCh{ZG#L7UJ7rP{AFP z)2Y;ktLX@K`TY&qE%;EHs{B$w-jSnMqOtygZ5yJxqabGm#of_=OIgm$I<$iq5C1J`-2yoT^n-SM{VaOOG2nWsyDiYiKSygFT-B1Dwu&LSs? zQ{G&`r&J%6@D-Ro3!kJDN?&|>5aOd4R-DZeD;)9_6!7&Z;8ls$uT&?V+Z&Y=)-(b8a!xQOsy*JqBdQfX z)z_TEVD8!|xSDD*5;x07$xb&`xGKQ+jjPtfcCzzGO2OaG$E#3P`8!Wrsdwa&{Nhj} zN2qS}WaL%Di45gb9`e){)l~mOHZM)}5rulYh&=Lzd$&!3(6WPJW@6e$bQR4)sedfv zah{A0SQYSzxgm*dO1>>STI44xNj5&G)iGo+ajbwDhJ*JjeQjx(r*T&%%Tcn@VQ^(D zd|9KNYC1Dco9#G@8kl$V^o_P#*lO_sUUU)Mdl(r%TtA{WGnP(8S5*h62j}@TxO#^f zC?B~QcR)}d+|GK18|MC&!~ti0g-uE-uWRa_R}WpdC~IG!Hunl=cQP4jfEvSWyWMh<-sFRz(QxN1Qyo?~ z<9Oo_)B><#ag|Flqm6!0-*MDD0y`XYPM~Ep1{);iEcqR*Yt>GC1#2ZM$SOi@qK+<) zHX6u@I*T@17ChM<$Nmphz^|MoUksoF6#%a`G#6=spl9zTC$T z>EHL)1U}@aDxiL$_a$e__a;tkR&Gb6jt2+Z4o`0fu=*`d)J;BlC(cYWSe?nJ2Aip% zJHXckXoI7ysVCF#^k$la*MMtTC$t>#Ge3q>;BPS&1`N~ zIMq!?u!4m@InB9R1Hal%=aLTOrt2yrv)ZOGF)zd>q+%aUmGn2xoXxTwEo>J0*CX_- zK6Im$mrwR6T5u80UkfL zpf+B2I>J8fg2(O$LW|G0i@~J0d+2UKW;M>1@x-o!a6}>Cs8gyw4xdK8ioOppGHZBl z+odiX{xdRl4LTb}GtDxtU24kc7DS64M2QSw_z3cy5F>Kx;u4`d=yKAU8<$?jvKjWI zdp$5KGv;@Y0prONKEo!tO=VP(?)7t=#{1Mmp=vhYeJ@C|mH8K?L)oTsfevVc=i#{9 zN^c8T9&pIirX&8lXgr4_=`A{kR-2#xooPCmy9kE-J?wN0iPXtJ>?7d;LR4Zim9t@O z9ctC1sC$ppW8z{;N#ON#)51=+!IM9N={<>wEq>?NU8cqCbvja=rB}tsm0MI7*oh;s zrYDFn86-V8p(Thfsfwrn_VY;OvV~w~1=ILKe<9Zg{fQ_PT^EL5Dkh8g)|XTdw4rLKsLy1&stFoAs$6qc zSCL{MfdgRdCOSYaa_b2k_BzIuTcMoSWN1cy(0a}oAz5kKvm zDl0L0k5_nZvn4rqA)7?gD^Q1T=v?R&CHZ{G*uga0J*x*%Z??d-@Sm!zUXz92=oYBh zQ~7MK@#C&j%~7Q@>kHh-zs*`bTTUkpeAAO0Mv8BAgnR4#8&ZB+s-|;nW=`RdXo?>ln@4UZ3P(9_quXT$| z6wSVX1^Q-BpkgP2H(MZkaYNVUZNdG;%wBV;!&tXpcF{6l13Poe7?mL_?-GU-nYYecR`J@wE39f=D0QVY;@c8DhyP%0F^l*T^m#R zb{n0|Ama5h6jT-yq+N}F4uG*dhJWU+9*J9(eYR}N4M#`L;LfpoVK0-xfrJn_6G=Ow z(`m&lFw;spn{QyJ*V2r;lbzY$nVkjhT02U2WfIXboekf4d&keWUzRc4%1bI(kC(l0 z3zc+tsmuMEBU!;cSyf&BYZ~~LGB6nX0zCGRPYGHWFwb{NE}{GOLi^YUyR`_`;tE+H z62!F-{%|VU|0VPNI^w7av75;bYXYwL9+)CT>FjzaZi)_4giSsB;D3IAv^(MXsj7$L zGYJ8g79rY4hx5%_{PO2e0)vtk*=&Lq^X31E(>(UjiaVc~DR#l!{nX5EH=R?Row^oO1Zc zR>CIzBn$#HPn;?^VcS7CFh=I)8kYE97N-! zm>bQ_Mpd;2ze-$tgKU+CyA!}{)g-VYBXo6iy`|up>gpUEcq(}eI$A@O9YSSynoqk* z{C#1`sM6U+tA-Lil9)csVG987R>r~m%>;mBgQ;uE!e-7_O}QNn;Pe`B6C7ubQwm06 zBL9C8-#0BaatyN<|J4uJnAFk{3^oLW-5B&(lM{GWwdC)82DZ!wgHa1*AS-8h89YNB zG_`EFwSN*}5|Cv!!d=W|zF#A@daXs1>q@>3(ud>-$h;Pv)Is_@^ZDoLk@zlk9}HAqUKGl@=_6p$Q9U?Z0s7IY8z8s?r?LOJ>ZQf+=KrT^T$(bY#}$l zm7b_4chOC7p>Ye+qI6FK1zn=Av)2E*r4EAai>uep3n#wgkrTV2sxX1k36(Gu`F6aE z9V!kySU*)2kMUP0CEk01>*xSGou5fTeH8X9z(}`+`$FKP>J#@w-+! zvDsk*Q8zJ~U}`x4Pumi%z8JW@4Ntg;V1`ptp$x^rTh#5Xm%@7-0-x;S z30%SfwF9moi`xaJuMN*_E?V|xXB1P*#PMrzNjaHAL0o=iEsE~5YkIN_!$m(EO*WbT z{nBN!n+l{9ymxFInvGO9^96TKF>@6Kwh#P&Yq`u@*nr{?VH~Q97I^y)@Mn+8B$b;R z(S^Gc3)I&Nl(+*_Fck-*|9`(=1x50hN19oqF>471UGZ;k0~3@5~mbKAeNA0wfFm`7BgYm7FGc=JS9Vp2PU< zX2L@&IPD_#4EUBwVmw^+(%>Zu?!?;D@g#SHM+Y8++AP-xn7Sw;QZuH}33ykk3ttAf zl_XTQ)Jri5DV3#cEsuKo+h_jY7}H<9y!U_iobR6BIrrS<+EMk;2X32d6*<)dafk9S{UR6_t@(op>xB%i4ZquWLdPfUFCneP|gekO+*gs)Mc^_8F41 z)kWur4^`ic1G&W`n+#UXb&mlzy2TW1WtIlfGb9s>dc8S5dZO!rLMm--iI$gE2?)?! zWb{B}<{^Trv^1Oj(NTK-sr7y{)1yHaAfq!;v`o2qF03tD0UuFO9LIr7zRl*0@vjhHadmGRoQSEa9(ozerMO~^O9l#-K}23x|2V5%UsaCLJPE5$|{k} z%M{yC>3C-@1K+({c~+ckHlkTeVgayZ$=vf#sQWZuFMo*aYm+KCpgMFYb6_}DI~Euu zKIK^6p1)zYWS3tLe0da_Yh3h(L_FJB+66ur0jE{SrwHW?@O!YuZg!GL`9TLX3qUdf zo7*^NiP%yd3z$cts#u;h;tq!SDK*-ADPK+l(i=dijNDj*-@>0-bR;V$xP_)I09ZW* zZ;dZpQA3;bsibiMm@PrRB4O@-EFsx7;hgU*s_(4h7}}c-6Bvvr7Y4&^27}?Di9O4E zB9G;d%#4np7V%bLn?mO=>hV4_^pq)hUiFDE964v+W;~rV|6c#+@JG6Ov|NYO8}}?5 zZvB9|8+qcw;m0F%QyyFrzVYUdOQe#-j*Nua7^o#}9tjNv%Qq^0IB*ev`r6XONYO1% z;5a|C?(i#&nEaYG%CBSUHknfhCW%euI%7L;UKO+RO{(g%821!ofsHcM9oUL!=e?Cb z!(^oS<*xuh?!ZgkS80k4H9rR}rXeiAlR$~H)P}i&aV4}=$0m@O{2IE(1Y%i|;t#|v zn{ZHxAhcm`rg@-SW1ziKv8{HX+ob@TuoRO^Tdtqg52A$g?fwl2)-hJlxsS(^l3`nKo(w1UZ3Y^oTQ)0)+_}7G&?;VGeSXyO ztnf!#ifdA<7~1>G$1)f&ck5*s4tzj_Re0F+#=goXo`>cG@f0o|*g$a7w z@%M=mxv~XZ%x=BI<5B0eYMUA+z-H)T_Zt?e$E_og3g;+CaE<-E6}&25r26dq={7Lq z?dr~&;kT`&Fi5~o!CzntE#QAR!YO~!1GA+UOnjSqv(IIohz^((4X;DK(x~vH6_yN3R=Nnqc5+92%j96gA+0(zxDp`&S z_I+3+@}^AnilOLJA=!t9qJ&jPXA+hlzaN)-JKiX!Z282`I?~pB)GsE39*@nE*odoDfyPuEynYY ziQRW*H(im#g+cP&nZ!l>*Ov78juq!rk^V^+Wge#=5nb{p4* zp6obTsfdXQY#se@*97P`u5yvf72@h(9ErO;#@AY>T3HM1h5I;FGrIW)eKtOG#2hJN zvzM~jv0TAPm%mu+v(p<(S3~X#!t>lIMJs2Dfy@aZqc4^6pwBLX&AKzCh2BHQFrJ}^g=^)x$H{`T%IF;d0n za*Ui*7~G4Oijv}|LQ%ey=gXc>+-L|?$+{@T_h^ZrS zI~t7~t~sC$5RpbiA(-(1qHD*Pb-05dP1^C#_VfqAi~Ht?g~B!|*KJ^_W*fGkt}A3Y zc0^=J;hY$*n5nN`Uh;lUQN?TIR6bPc=>Ek_w62cFVvatQT~=YFpb?1eaz#_pvON@( ze*`?23$r6r^g|q#*Sj0^x*K#r!NE*h_w$&pxXWie!=HYS>x=# zr-}+@M}>>PPXdb20D<=;oJOK(<%vy|nQu|dO%USayZRw?8&r)BbpXx@;M5P>O%5@g z=x1?VaS8>x2R15He_heORwr8Tp8Bg_`IT>Nog#wS=)y@SuB(_ro;0BU99j7-*E32t z=}>!K6gv^E7q_LVE;P5ZB0LkNkVE&O$c#4DqbOh;4BzCM_jo>(=!?$gtzyo3pjYo{ z-LmFTge9xDJ(PO?g~`ck#@Y8pX`k9t9&Ei+anbASYj+$1D9Zlsl7OO%eQw=1jA}}j z9SN}Ud%w|(e!hIpulv;n)UI#(3V!m3i`<1)XjgcFDOd*29Kd46xfk<%u(8 ztv75_u)Px;vY3_HX~oGr794X+P$5CM*Hh%-DR?SO5vJNyBZg7+Zwx|J9Rf0)LMGsm zxNk%b{;xKnqm%0NZDcL@?NGvQWI`$-u1q+!S?u`WU~4OD8eult*LUYOcl2M2tZ7DF zb|Ttt>MiBVRYqV8`J1+Y->1FnLw{J-hnQ|LdmW4*Btn3~lWdrM>rvil77;EZH-zDj2$@Km^it5DG#8)P7c7s=ha_hNuz0 za?wRR2ZUHJv~OPYC^YwJV&lS?SL^NtHm#~D$rFu6L?FBK*(Vws_-}CZ0)U{sRzn<+ z<-|cJAQHeZ!28z@Ds literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openhtj2k_sigprop_refinement_overlap.openht.oracle.ppm b/crates/j2k-test-support/fixtures/htj2k/openhtj2k_sigprop_refinement_overlap.openht.oracle.ppm new file mode 100644 index 00000000..1e454f9f --- /dev/null +++ b/crates/j2k-test-support/fixtures/htj2k/openhtj2k_sigprop_refinement_overlap.openht.oracle.ppm @@ -0,0 +1,3 @@ +P6 512 64 255 +ZZX\\Z^^\[[ZXXWYZX[[YYYXVWUWWUWWUWWVWWVYYXVVTZZXXXVZZYUUS[[Y\\[WWVYZXXXVXYWNNMWWUNNMmmkkkikkijji^^]``^bbaeecfgeffddecccbbb`bb`bbaccaccaccabcabbabb`ab`ab`aa`aa`aa_aa_`a_`a_`a_aa_aa_aa`aa`ab`bb`bb`bb`bb`bb`bb`bb`bb`bb`ab`ab`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`ab`ab`aa`ab`bb`bb`bbabbabcaccabca``^]]\^_]^^]YYXNNLgge775ggeMNLYYW^^\__^__]bbaeedeeddecccbbbabb`ab`ab`bb`ccaddbeecefdeecccbbb`aa_aa_aa`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`ab`ab_ab_aa_aa_aa_ab_ab_ab`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`bb`ab`ab`ab`ab`ab`ab`aa`aa`aa`aa`aa`aa`aa`aa`aa`ab`bb`bcaccaccbcdbcdbddbddbffeeecZZXIIGhhf=><`a_{{y231aa`uus@@>cdbkki887bb`hhfBCArsqssr%&$kkjcdbkki664eed{|zddb𣣡eeccca𥥣FFD칺||zmmkbca瓓fgeijhBB@120332cca옙곳ccaoomeedύ +cca==;AB@+,*bb`UUSͪᬭAB@QQOJJIccbYYWѧ湹__^ffez{yWXVJJIdec~}ccaRRP\\[__]bcauvtWWUNNLjkiLMK!!aa_00.}}{KKI``^hig110()'cdb110~MNLbba}}|eedoom߽YYWrrq嵶VVTט<<:ccbkki㮮??=FFEtus++) aa_ 㲳FFDKLJ빹[\Zbb`##!廻Z[Y\\[ddb$%#󵵳((&LLJRRQ}ddb%&$EFD"" `a_886))']^\&'%%%#aa_==;$$#ccauusjjizzx[[Z775cdbstrnnl~~}糳 !`a_wwuUVTב<=;XXVccazzyUVTbb`IIGIIHUUTNOMFGEYYX"#!ccaFFDFFEUUSTTRKKJhhf221`a_``_KKIUUS󶶴RRPIJH[\Z]][HHFNOMeed[\ZAB@OPNbcaccaab`eeccdbbcabbacdbeecccacdbddbffdbcaddbddbccbaa_aa_aa```^``^^^\_`^``_ddc^^]ddc__^eed]]\ggf[[ZZZXZZXPQOOPNGHFOOMJKIbb`bb`bb`bca]][]][]^\^^]__]__]__]__]^_]__]__]__^__^__^__]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^^]^^]^^]^^]^^]^^]^^]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^\^^\^^\^^\^^]^^]^_]__]__]_`^__^]][ZZXZ[YZZYYZXRRQkki:;9jjhPQOWWUWWVXXWYYW]][``^``__`^__]^^\]]\]][\\[\\Z\\Z\\Z\\[]][]^\^_]_`^``^_`^__]^^\]][]]\^^\^_]__]__]__]^_]^_]^_]^_]^_]__]^_]^_]^_]^_]^_]^_]^_]__]__]__]__]__]__]__]__]__]__]__]__]__]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]__]__]__]__^_`^__^__]__]__]``__`^YYWMMLllkBCAmnlOPN[[Y``_``^]^\]][]][]][]]\]^\]^\^^\^_]^_]^_]^_]^^\]^\]][\\Z[[Z\\[^^\__^``_``_``^__^__]^_]^_]^_]^_]^_]^_]__]__]__]__]__^__^__^__^__^__]__]__]__]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^_]^_]^_]^_]__]^_]^_]^_]^_]^_]^^]^^]^^]^^\^^\^^\^^\^^]__]``^``^]][ZZXZZYZZXYYWRRQkki;;:lljSTR[[Y\\Z\\Z[[Z]^\_`^__^^_]^^\]]\]]\]^\^^\^_]__]__]``^`a_``^__]^^]^^\^^]__]_`^``^_`^^_]^^\]]\]^\^^]__]_`^``^_`^_`^__^__^__^__^__]__]^_]^_]^^]^^]^^]^^]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^_]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^^]^_]^_]^_]^_]^_]^_]^_]^_]^_]^^]^^\^^\^^\^^]__]__^_`^__]^^\]]\]][]][^^\^_]__^aa_`a_ZZYNOMmmkCCAmnlOONZ[Y``_`a_^^]^^\^^\^_]__]__]__]__]ab`aa`bcabcaOONFGEPQOJKI[[YXXW\\ZQQO^_]cdb``^ghf__]cca``_ddc^^]_`^__]__]aa_aa`aa``a_ccbccaccbcca))'kkiddbmnl++*ttrAA@@A?ab`nomCCAccabb`hhf%%$kkjqqo''&}kljSSQZZY997IJHccbjki442decffdAB@~00/bcajjh00.ghf++)ddb11/++*}~|zzxaa_564BBA))($$"ccaWXV󾾽~~|kki331bb`bb`\\Z꿿uutNNLcdbQQOFFDFFDGHFbb`RSQVVUefdCCAoonbcaLLJ==;پ??=͉bbaMMK;;9XXW$%#cca22011/ttr``^KKIhig##"(('ccb00.01//0.&&$bcabbaYYWttsccb""!ccaYYWefdccaGGE^_]jjhTTR%%#cdbIIGﭭ^^\DDC\\Zvwuab`UUS||z)*(׉>>y-0`eCYt8l8#-4k@fD#~cFGye$3>YMYKpBM8$}P28S%hW6cr1_z zHehTVz&08X;K2$YK!~775MTi&DN86JwQ}y9l)BITJQ1+c|*NbyY@co)~#8$Zq3@Yq}6Mx#LspzjftVJ(%N`1OhV|rEuUWNeYUAqFEBSfop$BhxbUh_g3pRn_`t@tq zuU|*4HERSZZI&*Y3^ZW&SqybFnxNn8>`R2(tWrI? zRtuH-#So0Dv0V#jk3PDAJosv+za@eYEn*9$ z$VHNkWjucuJmO?g=MnkNnwK$An*51vb)txqH{WIHF&HP|qmZ*WRB9U3Q zevN?Iur?r~%Bs~XrlvM>_wgqlU9{-%#~*(bTL`(VUTuArZig*YVr@7Ny$vZCbCA#A zXZ$5QlHg5xGT4;F3QaYT5ML5vRbJCH`H?2jQqD0vlv{Jo6w^@G1`z*X&^!tOBHgxK zkXgBE1)9HN#WJ8f>@aL$-HH{AwW&*CscQMm=Xg3xv_wCBOO+uN^MMft%8AB5*YG*eSkNJu8~uLCDUZ2B~Y}wMlP-zCb!yy)N7!V;oL3*}tiLC}&(PmvG2nCC>n8=rfs^g;!}7 zUUvwy@P>8dH*I=y)vA?%34sK=3TDg^jCLt&lm`riyOYl1)KQf<=;b%e55BM9_69O*?h`*+STLiyhS&P3xo5 zL{eya&mM9vB>{3=X{Y{~3N^+MLHHAErn@$1_!?&6r1_K_a!9fT&CB*tk3IIN99N_2 z37pV#$_+?#T?A3=qLNFP1~xJ(CEF=R)R#crJW}9_Ei71o6jtKRl84Ds zX!;GE4+CisPJ+&YNV3Sa29zxfr(9c^`Pvsf>n5iuq z+Ag~jHjQ9%JcnfKC47Pu?k zWe-36AP7J5$iqN-@WBhYKC!+Sp`BQzhRR-0( z;m03;OnaW|*P-($`SRtDP_lgaG9YDo1tQ@b;HT?Kq(F72EEMr3oF!r`NupVnkfO77J)tER~HF}bA@jPJ^mP;~_RH9lbRdoaWq$etu2g`|{Ib}s^z2v zvo48SrF`V#*I&+@T_@RRq^xi z!w=)C{L~a`51wTBV+l+-RVZO;;AFH78UYgRsIU=17i1&gM%2y#k`x+DB$(1DvaMS_ zUgDj*9aihRAjU(kCli=YJn_V1;0)$q$SMpl+2^DL8a1*28=sH~@Fzh4PPEX~Wcv^`m_^gd>uyW)sa}#A6dICd zV_6xWhT(ycw%UZUv>BP{>y}>THxsPZZMsCtnf!?$B_%}-V7Xf7lm8(i1x)M$?6HyX zJgD>&PsnxxD?6VjV_s%vVRgVdA}+h#i)A6x}&jWIj?tsL5gz+<^DNhaLd8gAX}~D|t#N2gcY#E^AUaRD9w^ zmKx9Te9d`_jUj@Xn7rwSRslJdBVg-PDL*@XZRwwl^5h6D#Z={IJLACy^PuO@MOx?Z zel;rn;Dh%Q>g>DkK3vID!iZwEn`0y*)e^P|I_7=j(D27L72FQe)C1|{d-SR6-iaoV6qxzE5M+XA?J5fO;MhCQ zvV{llr=0zN2p)Rq0VV}9+WhzKbXNQ1SlmKe3W>cCy<*y?N)}Dxu(A#dt1YiqJHcw} z+HYjSlHj>}9Z=@B_D)WafX&?Iiiy1>g$6(%3hxa|q3vM);DZl@EwD%Nz=QV_|3gL# zTv^Go;lM%wU{l@ID<69J!ABl>h?QEsMxZVwDUpw~T!qU>%OX9@>YIh0(jz(fC*@8i zjw+pNYN0eWnzBBbhjN7Y{wMjMN^BrGOx&OPR60z7;`_YNXR+{0!p3Tb;tH z*`zHi9$CHu?M;}}nF3UTYdP3H^x#AHKk&ePicuGA!jm8c)Yzsm#U3dV=__+l!e|f? zloDm6Cx#I;EUB;&38;~NW0(znXJ{RL>uGP_I;JgY@mU(|bTpTrgKbJHDisZyJXrB8 zWJIHd))cnz;7KQ)_|QWS687Vf0!#?VAP!_Cb(P0fe$k>Mjyh`5(MK;n_ShxI9e?a` z#~pLbu}hXLIr`|Mk8)bPWJyX#FIfy3g^MFCK5FseBS}X^Ix1yc$SslOMg}se5jQo4 zYzyCNhSp+U0|y5X0IMU8IDFBOM{v%W2afyhyZ7!KO9S3 zbvK4)qoX5)*VWMa8wN1FM}<4P@;c2pkt zCj=II$jDTb!Z6vwF<`#O9&b4QgyUZMs#oC5U;l>P*_uF7I2pV2#kMj#ZQS)yi%5!C z6s#^&{O>1aO*on-qnH+(xi3v{nxh^0j0pZA%Pq`HCfJz#I-^cCKWd6B;YVUVcbB3~ z(>%-!%We-b4_j(HB1F9j_1S-8@+JP~F6}7(_tV38?2@9F6wk0ID2XU^c?34mxaaFC zyvZ15PSP!pt{rXWkRV5!O(UJUX|TCvH#N_~8rkM>bW{dXM36HIY@8xQPZ40O1;jHU z4-b4<+u|Zv4zu~QU}_5dbC9E!36KQ1o`2=!XTb_A9Xrtuy^zGVwVLKh@FzN=A(#MEuXLG8+;2YIYmE zWj)0**|s1L195m55DqI-M;Y*fDzJx8T*X0eUL$jhD%nD3k{Qx-^*)ExmdFIQGpP1M zj5RIiD&{ld|GZ*NI9TO03M13l!U}vo+OGZo1)Bo=M1w7uUIya$KQ{NgQ>`7*_=yC4 z45Qk~EW2v4NtO1d`UFwp>ZVMaS9!~>yL;Bz^6LHURG-Dq@jZ4L6mi~mBcJcG=cWUJ zB%Y7>RN*={^LgNP*IjpT=I3?0??(R4yY67ClX7-MeMD^?#AMi^(`ISeC2p#Bs!aPcx0yE~dlH#9f`W6Msk-a2 zYAKXGPj)qdYt1SiHQs&qosz=qb|Zh!J$LbFJ}_kFB{3{qchcdj^h(_ zG~!KB;a?1)={*~T@A|2M9~=aM(a88G8<__wG`%-Um}J_+zjPOwT5z3Pou8uYcQibr zJhYnL%B-W9AdIWI@BVvn`>crZIpp;$jFaa!flzGDFe4zYVPbBo>vNC@ri5tpfTntJ zK0T~KJu)ZS!OnR#SZU6ve9in&UOl$c06zVSqq>`ia+`*FOiJi8oKioMpwRyRYN;P3 zpn-Bo^evd+Yf;JB8&e&(m@>5R(8^myGf zdz?ARG(lWlgXZ7Vwe`^7d&qGBz~wsppp44>6dp|CoC&(hbM}P?9Y_hVv+aq7vnXIk zT2*}hBp$x2B>fSe`e=|5*0BelW})#oO+izodX4sM@bqyGo!lr%+v89DRKafK3(vq+ zj^%(Gkw0KErBZ&-!3)7Hox*zJF&z_n@^KD7kgVZ?z5H>Mg1bPCSZT*+B%aLYwJP;C zR+KB%r(!}S#M@IJ%;`96>qa@wv~g3MSj*Q<>D&lImUBqz3ZlR#Dfkr7f18|*6WtY19vXlcqI3~Bm2UwDd`_H5Way6mhVnz-|wMVC4%z#7GV=H&6vK z9vMSJIim5l11kj-K4K~#tMglS1tlNFhoDY44Pa+7CCCSMo(8h#0|Aj|9E6}X^ns3|-1;Tu7MXH)Mwz;W ziSm;x{{W?imRRkrL%~EiIHAeFPB;pB7zKNw_$UaN7XVoz@G5}i81&qVLU5)E!^cvf zAs;+rRBPd~Ol#iO#FV_aE7#5&cBTBOpULmo=JyJ#p?Og8@_V_f*Be)|tH!j8 zKbQYsd?ZeuD3mxeTs>%`mgsZH$T}}M0x!!10GC*0XtFi!PWWX#iOCxp(7=Q79|L$B z*ikQU_RG71YqztN?65O_XCtl;FNt%7s{QK?TytM7gA3tnyuP zBH~Umhvu^}xvuV_*qRvyNXnyi=q=EJ2L7KD zK}@O8u-ZZpC?v$_Jjg?--F`k9>Q~^QkDp|?6|(?a7C12kCn>C5O`b(D!nI<{3Nka5 zPF97wo0$w&6Sp0SbbtB_LY;QD^3O(>%0N091`9F<1O&4bP~hc{9Cg%@L^Z&RT<|=^ zIS>$q3eP7VL%rqA+k>> zsi2USN>4PUqPpRcYzvYClram-|6AJ~QfiV-9)mCS`JE;Nhg>iOnk~o+Jq8$RY{0qY z7P4#dTrfB&WT!N4>a6c})hukYdu@eGEobQNP?(%N*lzW~Lh~6z$PCxasFyN8=cQoa zKvV;^Ac_CyoErfrWTEq90F&Oq6In!nglj_iIh~>vE}If5G~yctt3f#Ig^D2pW&~O$ z$x<@g0$=!3m=*?gmz6GN0gPsZYpgbg>h~5Yq-;LZS#}d zsi!u1-AI`xO?ELTZ~z(dK+1B7E2}Nmytodk^H2s`04@S}^cJ1xaR7zPs`gZRJNy3o z@58YecD+YTUsSZeGa2EU z$sAzxKj&rS86ZQGa%P=pg-bj59G!VvxdQ!1=W*{8dNN&4{Le--0V+Q!EQOa?W%v$kunzn`eMuHRVv!jx@Fv}8++%G5MFhj2XhUlCz@IYtJOm9Y z9HvX8Q8{QA{{9L#N)8#|AO&<41x5< zeHO2~nuKaQlr2`rt=NHi1l8G)kF*_JF>5q`mUt>ZP68 zWG1u@k@>)ny@5N*GQxo*TSyTp{(miMY+GRSVJ%OAnQ$!#g{-vThxME8zyDsgNh#+g zrkB6`Wp8-H?$=*`E%YG>3p%E)wAMxnz+z5tO5M=ZLJ9h|bjCy7Ksqsc$&|88lZnO8 zSr}Amr%{*|!*C^nqF~BP^BbMVMdSH-I|I!Z+VVUry~uqG5A)Tpp10Rtd+fUFF1&p4 zv5)=bjW_-Bf(yQ}+itJjW!Km6BE}~_`Hxp#`HMIy!R+v%5B>QIcY481H{FP~Bclv; zoH$K$J#w)QH1k4ea^bLKq{B_p1Tmbt_3Z`fol)g!!|d{p3CG`O_D@cxNW=pZ@eG z=sbVteBtbOzx$o9dCec;mzYQ7vOLSsS7IO-vq}JFDIAj-#=JMcA+lL0^Lz-xyrmIU zYk??;p`87@B=JnSsI|v-X_w7a>pG@Ill7izy4o|no_*P zNg;cCdd!R8_)lRyE$V4|HLi)=RH5&}m4TX{@?t5Yn=PF5GRfzJYK0Wwil9+Czri}s zENsuC=fBI)dF>7%g|p8-^Yqg{$wG_+Z>)2fop;}Tw+k=409&~9TbH2uz>CA@BOagn z)ak$WYySsgqJ5$!VpK#xmtYpbD-lxG=Km1AOHBr|CX~@=biTzUff=~^tM}Wz4^qGPGIFl z6twHEuVE_R`9(Y3amVf8e9bjizxc&FKy$~PORvA-y0^dmt^fLyA5#~%PqW%AHQ>3& zG+rZ|oliohF@eHQNF?mE@IN_{40dwjNhk*ZudsCsc@&Eb}d+!EJ=8I2$@*mDR>r9}&`<}Zn4Q`RY z`s%BQODX>^|KixBs(4 zKlrm>YoH_#Ifid3=rh9TD!6eLnppr(MI^(#5UNY?!*`)Czx)RveC3r_eCf;Qf-P?< zedW9_Q}VU1ozIGxm-ycFrv1oM^3|_?WzRkLfPj5eQg|MFKcVV=!y>A6adD6ScW(o1 zPhB*oY)xD_;J zni<44^B#7K>upJzPd{~?3z5P3=j z%mSW;<5>*yQeYF&Z@%&~I4sQbv*hbc9>~1;lhXL)Ete{zGM%w>$Vx|)D{qK^lwcoOnDXmt1nu zv!3~dLN&uc?BWZ~ekjt^yzC=b^oSHJg1{|VBC-&x$^Ykg)!cSQT7K3;E``pF za{i@!-b7aOFqp3xqxZ0m@X6r{%EB{_W3e?Ue9&Zk6fTH|ELw%F)scQ|gmp0sVP=YL zx7+60Yp=m5zWwb>c~kqo`|cqSCDaM4L5@N=WS!*;!l=4*K9@`SX@)g2akpHX33T4z zs?eBIn=!OrYuW#6j2dQ6Di1R+#vbcg2tZpAPTBwG%mZjM+w!<7ors=7;qmJD6Fw94 zjAuL@4|DBx*Rb=+Mi3&{euwR-e&L1ReBSe(!?zPz6#Uw+{ipZ6?^M1c#AHqT1V60E zkXk}Jv!_wm08x9ThGB6YV>C@xm}12iToB|jmcef6OvPsmXY4hP(UzV@jjn<=U;;QE zJDqvJ^XNa%@?9VuvI6s~uDX&u%K^4MdA3M6^cNrf2s1AjvKPTugOCv+Yb>?4-S*i` z-?WAZLK%<|&foFB`mJzlw55l3Egx#Y1_O>JrL>#iF?w3$rWm30iVs1<_+mMDP? z4-@-8EPnC-JP`vukcR*-1awq<*~?z?%NuW?5RvV;<8#UIN68V34)>0zPG>#mxjV2~ z4L#53ci3S&s^k2T3v!H3v*&575*fuU_z_D=l?il4)caZEU~z{`$6x17SmJyrRa_+< z%fnBaAATvu?z|&X?}hRc|F5_39(f44D3V;mcmJ0&ld=k9=kw-UZUTAspSRzBJ2K~< z`vvR;8f^h(pP#yu#0_<9K4+-26at4qtmX}(3W~Y5S)lHHb0qd8~?Aj@RngO=rHr9 zHj~$`Cg%B*KY90ezVmGg3H1+LcmM|;Zo2tKo(QuL`^6PMru`*Z&nKD(>l66eH@pZ~6e@f&Js>lKk3 zny;`&In~G{p&b(3;5yWq2x<_ zJd4?O+ikR2ro}>detXuO0pregt(<8mH73~qnZp-JB6xuAQ zFpnY+IZb?V3+G_}uhkHgIOoRyH~T-@c}5&e0k|dv#0%Q&|04y$eww}Lq6>cWH-Cd= z?;d-;f!!YVOPQ?sZvvb}-*>-#e)-EAbY#o_(31rPK8<$NsSz8=c0G`mm}-$ZpDEjz zcU)QL=X?rl)DVXFi8(Gb%{K^~O8VGBiNV z)As%L+c%bD5YR|Wuf@-cH=CFy8hbq5%KHzQLxba}SfOd1nk}FXfHVJDH#?(zi-*48 z8s-^SBn?5e3ULJdaaY4v>iYk5{vYqbEuK{=l3dM#OP&YE^1uVkuv5gGWNHH5f0wTbsSi@>PwP77GKfwMH$Nu2`z|N~l zKmYkpDZws~3mLu|ee=yXl3|)=3y4!gY*}C+VN>CwGD~IJm9`h&IvKIrP91{$)94K< z|Cqwro4`nhhxy4)DUUS!tK|PBDf~ZZ11Wj*nFA-xx%Nn#Bn7^zjTDya8#UU0o|6B; zMu44LK#Kq2s@`L~mVW%iDDu2erhw$_lS|D0v;1t$1!Dm8tx{{=GvnE4cgN5?I#}we zQ!@+DpvYSB|7ya;{I-Q=Tbhk!6!<%6;Rw2pK!71FlVP@{3hl7-!=x-Hreo^-14zys zdcV{go_H;Y(R{@`R4NilS~slJpi$h`<>K5pBt>O% z1UKtEZ(%IcTX;VIkJI4gJL$jQ9zqI`q3zT~G{8<3Zb1XQWN`jG0>e$gn>YV;lncy} zSHPZpj(_&lqvmkJd=>%2Z%bjmeh#BOZC{Nzk`-w9jX7j)6D&yzfRp@xybF0Wt1x}@ z#OJT{kqyws!=vyx8I&9VS!W@I(0?AvKo8&~q=1>k+6{5AWq}nS9V`xUP@QHmA0_4U zhJSEak8XM!$s)+L3Su+O%vEG9%rnB~s^%p1Vuf+J{FU^R3EIO49Q*$#DZv)-NOC)m z#`#Yzig7i}q*8LfAV!{t#Xc`2SyL;3Rf8#*iI!V#VXlMCXwRaZ8jXH_(gp+V%*H@F zBG7!=q(=W`-BE?R(s`Y5TZ7Nx)h(P1h4_B}gc+m*o~SO$kDhZbh(d1h{U2xuSpnZG zG8K|&7M11(l8P>n6d7*OPSDUk45X0;pL7zbFq}(7*-BaA+XzgGTSjbvJ{sgYhGy_q zsi;hc+rgh2g2tP+C3QX)oyTJX9X`ZwVe8FPB2R=uA4xtc* zf-1rSZ4v9)L3{-znN@;!B&vgaPFvWP#Vnv3#B;*JfPIj-Cy^L_mQEINBLzl^=9TBa zNI{-QTcG&=O;0ifLl9n_MnIdX8LKhcnioJDi(sbkCZ4{UMMgCI9|s+4_h%m*ILQ#S z=@cn{58ShW^&uH%Qo@NeDoY_l5q3BJI%6fzjkhYcaL#O$!oeyo?C=JpF?KTY|0|b+ zFd6XYEJXN!^c)+BM*vI*Knmu#eXa-yz@trCK#tVKUIcQ?0?8_~+($E}(G~t@lo{XV z-0zNJCbwKe>QRK}PcGLuctUJEfY3-f@AE%w_3#f*xl#zIX_wNkD}pvsU}KZ#yn4c7 z$ECWCc8s?C3^5Xdpd1MTK6y%nr(<#mxP_p}L}5m|e$O6)^|{Z#XVu{#c!np*tpG?1-J8SV2gFa{iJFy-)>1a6bHh^HAP(r4q$7vht4? zP=!Vf+T{7U3u}@VhSugDY4QKyBL7cF1Cq?b;r~GoKTk=3Cacj(O7k`JAz3gF+%OzK z4xF>llf)qypC)o3L#U2#c3$p|;;r9!D35f$gwxJgmy)VPt-vszw&O0+`5%3GETgT@ zKJyhg)K?!c5#UI{J8Y>1yt-669_4#AN3?IJZ2~`V*E1LCKi!FvyIBxvP@|Nm4RwE{ zvnUw_7?$a zkQ=SWvA3k)^Z(4k#q(tUBM%SGfp9cp-Ii~sJKdFc(%Tj;k-X?ADSys$%M(FzR^m|X z1#fv4Nc_hrde=4@ov}~&Q~Ym{WPaoja4B3pQV-J_tGiyh#{Lfq%sxM#7h^iG|6dSj z%eMqdZUIOE83Dy$H4x#P{Y)JDOhQ**l6&gy)gBu>--wFa@yw4aF9`wi^2-ZQq4NkOc<9@FC`kytqZ|n#e4g!ilSN z#F4Brn?R!>n+eis#l3m@4ds`B7%939ZcMC}0^k3a6o?z?8Ywu!6d9~a^R3UuS*`{e zHs#oMKtNc5Rsr7d7l4(0^+(Ykc%NqSl74HGluX`!p&V?s+#QusExBBIInm^EAo4=? zdjVJ20{aY5vc11(5&sKAPkS8;$&G_QS1bawc@V&+s7gZl!>1(;rMz-S59*3<_6vo2++JR&@A~Z})=tAU)EdUx{n8E+^ZZE)U z7EWK16mEbIu-e-UTM&|JHldw?MeQB-uuPNSitHqDr@k?kTZO=qQm2vVy5z@ZdT4BI zslmCK_tXJj0*tqFpLXlj{hfv|3BuKCB#Bt#k`CZpmaPb#U`UT)8-?1JE_5{ z0gyI>ljf?aIa05MPJ`3`EKO~u%nUdTm5Tu0$Rzwj)kCiUfvJ;#pQnnHbB2l!D6zvY zo#$zbJ&%Awd%dwxw9$omj7hS!X4;^nay_)wni^D_3D2llW`S&*O>B_F*Y8v%ozzG( zv)7c&0KMRFf@{B*T+(4I{{)i)KJDwLCrazQ-v4D6j;HY6H(nE8;Rxr02x1q)T&b6+ z>(-*vc(9^51yiy+YNlQHn`D%c1dsu4q3og>C53n(Gw;}D#!_F?wqMbA_;0n4l&yGL z&||PMWZ(sSp4jpe|IYz_{67+5MqqaiQ%Y_*`%=RgjHPA+58q+pAZv^3v7z^X_kdtOIw0uh+hs32rfGyyW0GzYX z#~*(j4hjt86c+TFRsAr9c9NcZDi6ebEzX4LCSu4Yk{kfMfxQKOtHCVvFy4{ZBE%VG zCTx5h8Ndefts#VN>sE?7>=qOjdVT7XIjFsMIQ6y3jl9WA%n90B^Li&oF~7Gug|^R@ zkg=8)o%hci)nuVaMw!wqWrDVbw-~cD^`$I#r0`Wv%-qscYR$dJ&(2jX52h9}HA?f5 zAM7y~b~sEw)1c@**#hM#cI@`>6!?%s4`zEIo(T9Uyvz`DKq6QW#DPDV3Ze>R-2)zx@nMt&yiYEsk$n&+2fhU&d7j>LvGNS#U}ZAmp9 zO*8P8TnZq=0;-_fyaa`3G6KiQXa7(_3v9vhKX=0xm<53Ajc?kIDm<4*E}DWt%)1aY z1tmLdtbaq*J8{Lg-G+?Oj00iA+~lP7U*A$D@er>Bs)<^yV=~kPu zY}bu}E5u46(6H2|g}T9{fU_Ag)3T8Xq>V95N~EzyAW%gLoEX9Xzy1xogOY`gAfyj8o^B-vrA{-FNk60O%vcnPy) zJOv+JdLeTP{ z99b}Y{12xMiADF^YY(pEu?R*A?10QDs6|F|ZouwUL$VRK=7++x?Brg82X&_&qf1Ebg{@R|tRmj0wwjfvzGYeB)RYDIfEO4fr zNx4aJ3eg|j4*$cIiejJ~Pz3V<@EZYfH+XAFs;LrEoBUb^z7;cVZatP}bfKsoH42Hh zLHXH#LkfDsG5+&781i^aP_JNj8rZXBl_`i(-Q}7if>?)7)N=OEwn5<^7Ex)oIjq#iDF-CX|CP3Xc?cq|Q?qq`>|I3o%MS zl`6GzlBXoAx4KY?lgN_CHjQye)#b*6H`S#Oaa8opWKmkPyjnwWt$#};ZFwcwt%`CE z@+1-4WIw~$dH#rRXa8T{Lf6~AI*}5}8N#weG^Gk^TM+woN{bF(wCD)FLLX_-kw=j7 z&qeHS*X2+K!N@BFTUd)7cGw}@0!*|zskfB;aQzJ7)||M|aygWJJfwr95VTuRBcc3#hBH?IY~ih8 zF9`Y-o02)=8P)5t)cFY;ud8(OO$*Y)jGo48#a;wxvIQFgQV3gk#Vh{sEpIvL%?Ip{ z6hg^CmAY2!MH;(=;*@0(K&j2e0eC16of_M07fs#btn#gB+>&bWPdGQDn1T5;GU{N` z`B^tU>wKq8K~xY3UsUT@0kCo%M;^i9XV60yd^(B$WcS*zh5h!$Q%RE-AL%x`{siRH zjF|$P9BP&s*>gn96rOrCHMdjD%bjb7X4KD4tvUW2)yoXZ)12Fi{}m9HIW!{1D*k_r z_&@&V5lda3zq9hCgsB;nm}FUP#Wn>$Y245mWJf^}a{_Pdh-zMTUt*()|EblIi({LD zRT^DzUWq0XxWO#DT$rNS$$!oO!-|Jl)EZ(b<1cUwV@6VtI{w#Fc+NZne#AThqlmqd zV)(|^Uvf5>{Pebig4kfV0k9qa7lJnIjYR0HlT;RBto%U|0waxyst&?DA|B$shTm%CYR(2i4|7`U~{Le;2S_>E4VxOU< z`FclQn{wK~jD4`Q(JIGPCGD-5CauvDakF)nt3MS>3eWva&9*VD)k%sMN}GqD_B;U@ z%)S2~d!E|==kNost|Jt1gCKU;fVb>_(ui)wGcRKvDh;Kdk@1wvd!d55D0bliPAq~X z2-858D2oen#g=qJaP{@6D5F{s?R?F-UX-v!Cx60EPW$jZF_7o0s<<8Pd2V9QGp612 zFa=oY&?M$u66je#`YAQ|xdtW)T`b|@@ z4c7qiOj<6ZYuXv8ZbuXk6VYee!bY~>umV=WT($r2_wv9QknLf#3mt%m+H(ao2oQ~* zBkO6tY{Y`Fq_Su%BLiE^q34|f|FDsCZH6!{L{Bw>gt=$81PdU9e@IrJW?}B?(Va3Y2MfAH6{Fxdh)&`>fNO3ukzHv*4 z`5kI}K5{`8?D=qIMxX@iFileF08I_%8B?CbuRSt}BNv*bqd1xZF^wm4XkiOu%}FF^ zYKtgrKXo!1?@$`gp-~c0z$;2QVR;^|4BvOt3;0U*<0hxGv%B@;js&lQa22Zj23A86yS0ikYOE4 z5gBF%Kt3yTk_C<^^Pm|#l)lBc{*rM#c1$u?KMh{;^_ zd7vb9?H8qmrvTK40@!(o2#3`!Umha_{p7dm2J3vgg+B5{;^me~ zta1RArLd9*zW98=B3K~`58%-8K~p}fSDP!daN3XlUKPl+{C+GYQr&i|wN z2u0`rPdE?~+9S6#MqQ^1ABT3zub`g)k&JlepQ2Oi-oO!yxLj?QBOVCyhO-h&L~ zLf9+i$S_R@&h}%PP}I1c8yO^mT#TnILMfEL4Qb|6hM7}?`SgJ2aY?unrVP% z4Ij~$Th1f0wghF}qe|qO0xmE?lX(QdIdcL_zY8z_55m0vRoQ|z;;5x)lX-i!KCcap z(Dv!Xgsoy-1-ejD!Yyb>2DYHuvczdXs5#$S`B7-s<%Ks2$){V{Wl80#)sTIWX(USE zQVcWV{aSvUZ<)MPt6RtO;ia_mm$2Mw{gx(Xh&Quc=^!-xiPE~?_pfd01$iUhY@ zW{~YDZ1dLWnC^xY8cVO%m|kz9narwaIF$Gt;(tAD$?v>_FpYz-=2lCVpDOhJgFZXp zZ&xx4lc2%XuyEi?KB6P)LY$RfO$AGZ=fq^5j*(-eL}~s_T!b>a%$Ac>8CuTN3^!~r zK#+`K0++u$;CJ4a`eVV?m6 zumWNszQq)_5DQ`If^CF4Ts3R+UNF90F^>{5ikDT93l$+ovb681p;&>UBk>H*;Q$Bh zbiRPqFt>VYNy_^_8Lfb%aH z5TEA=g*pK{#uAHV8li5L>>gu=vN9yO^bjljM`m8|?6f(}&oqrz4A&)QzQdIoEHteV zCCv}|@c+0JRF}vzeR)#vyf2XI+B%>CRiDCL&+33YDlMG-_#QzJ9Z6(Uo7lBR{}Cdy z0M-E+Qjn5c)nTb|J}p5=C~>PtkV5^==;r0;G?-_Vm^ImWCN<>R|F7i#zvu}U+RphO z!hWjr=1ce!^56;PM5H`nj|Y_CPYDm}kre-elfV_65fgP?3-PRs?Fcj>reh0K{gSaE z2ww9UmNyoJ7|oUeRoe2y9OBX0nVW%|>IBuY67%E5=(Hh@K$Iq->xfk5&9 za^ioz6bud8j<=^4vcOqcOjG=~QvxJCI7q84ZM%6c_LBlxT&?`|Tu}0r+pHCAE947H z1DR~U<(ApbeqLfSq_%NrT3$5Y66R<%aN(|TCU~ADOZd;@&HsNvU1O1pdNcWQRtlZ#GgZkj_05Qt-7H zY#s{|1Sk(v=;1v)Y3lp`)mOOn%`4CYTa1G1{rBIC6!;gMngt$tm}5fz&(~9U9wfyr zu8u`zBNmXRY1&6*T+?Sq*=Fs!u49QT?`cNi2mbS_;|! zqtXC{&%qXOIN*ur0elJ-$@p`i0W43-2uj(w3)2Ah+i$;($QHnLUA>Z1D>fYc2W>E? zI|`W;yyKL{5*~??B$P5QC&V((Eda;hkjGy1t$&_;{9K%WfmV4}}nVppcfu7rw zXh*^kX-Gl;3l(*x1gHsPaOqI8KnQr_m>ihzxZ`tqgXxo}|08I#=R<=~Mxt4nTTa=x z%nE@T&++*=0o&oN4`y950BoKDhHi;5J%}Npb?9WE4bJ?x1(vjw69wTh02~;>^Ru5l zgEx|%|NI@_^{#h(`ixIuBj=s>m0fmu&1-gfHLnbO{1bo2|E0w>z3uI9<$Vp@*RO87 zm0Gk6=JG#YV&_>PhSrF~CXjw+(MTCm${=_A(%{b+)TGza#r$M?5~115e4Po-8L4#M z=79NY7WOkQ{J-BkCFWTh=dy4?czf_XHq7KfBeU9 zf9`XibK3h({p$JWVG$?u%F>(P#M?|SfB8%KcjPzRa2@TW)tucvvatW@F9C+*{F?J| z`B`8%1Wwb%xCU#eBcHXUu2tG^7Z%M?Um=Bv|NRKS{$Kyt6#tL@8$Er6l|2NxovEoy zF1hHWlTZBp-+vMRn)Zq-enAO;{Np747y0|%f7%(c4rxAD9_ak_U;oukJH3$qQOmJj zoKox>Y2!iF0S}0O`qQ6K!XMg>k3vj*&476pgk#Ds2m^ag zL$*0Mt|88gM4BOtB`fsPWCj^%3Jddf#+u*Es$~i90aCZQl?+@wjMzy4Yr)Xg{F`26Sp9{W6>``l;#;-ep7U5vfpyJ+*&)86}M zfA#_L@BhI2PC4a&A_e5ga!j+fzhG>vD)uxk4Fu^(`YCxhX1icjK%P;iRG(9(0Y5Vq zCBksJ)KiIy|MZAFFhQlI^WIEg&8uBFeg9ehpRkh|+5cPci9dMB?_YiORaCk5+G}>* zbr-B-#~q)ybm?vQmMgFP#rE58iwU6iI3E^bfBScT^Zxh0F9K9n7J+a$fBSy`PbLsN zgr!s*Hq*gu0Ib33;V{<*>&8dr>rM+zvW1os3K^AqZ(nPuO`v4|zZhjEk_T060X@e? zi1`mXc;OFz@I6F!*=657?X>r@{tFohopC!XqFI_Kn)GHu&2{h%BvoDK5hYc>3gFo%ktrgv&J zeyM87RO3_rCk? zJGJ5O^Kv#)u#M$a00p0o@>f<|U>ubhF~DgNm@&aH0|3%bjoYm74C%+%b!~3RsG~e+ z0G9Y%F;D()%Xzh>Jx{u0&*84S@8lHjH!t`GYJAQ)XYaoIZm9Ri{`#-ZJ@<>?tn;E) zUCOVz`buc7zUC^vV1~)i0^4+GJB=d(`n8i$%v0Eae+{Zpev76?tgU`gw|{L^tI*et z%pgAue#UvGciTEs+k#k%!%+%pVkRY{m?!Hza$$C6u*CoO+Q}#tSsQ0{ z+kgANxJ6xUO*5JMt62yotWua;6M72f@=)dm*aieyO=UY8MUzJR`KGCPkzL*CsC;K2 zHJSW?W?q5N*n<5(CsM-yFSh?@|3?uiSNs*b4g2h~S9qJHOK;<=HMie>8*j~Qzr%JH zT<}e_9kdCV{UFJ1GmF0Q^FvBise}g= z>PKDhM8jE=#?CX(Q&P#ARm3omfjMj57rkgFRsp~L+fT#oqt9#vaX@;nz4kca#N)5I z<|=}j?Y7&NeWW|?SjzQz&wDOXvmB?a-fVFT|8O;8FRQ)At3j3LV!%5S&1{yqaDU}Zdefi7h zGS#vcd)m`}lcm;e-tENgeBguc$1acye`lWgdE|)Hw%v9cy3iwge^Zgw9Lox(b;U(~ zA`@I$?FD{toB0o|PC_XIvB`whSfF7(_6E74234ihLUniEN*b;*^QENfE)7Jtvc2nW z4h1QXJRn#8U;mL7i@=j(3+yBy9XttZE~M~s91?gBsIFjJFWXKGSC-fmXy_GPT0AoIK0~16c}19|m+D zRV3yy+U)u3X$#W(2k$R}sX;mt4dbrC4^c`F8G?zIgZDcU^w@zZ`JD z{;a>A`OM$NM*KK<{SG^9$2wP0lfT+$1}feLoIk?}uBKJXGKzj?+PW=**-Iw_+qBS; zSD&Evt4ZfQUdKNGUiLi0|ML^QMWyjGJa9*ok-{E(yq@?U2XfwdU*_Xu)Y@&g*An?N z)qeZim$D*8i0si39sQ3#c{fu7OJEui_&Lgg;UtTeb4a!sdeRJPXcY5uH#!<~9qy$A z@|d5QbGk@P1+j)NJ#69-2=bFhDDlbi|7%w3EF7z_<&OYqJcVHvMg-{dm%e=Nhd=V6 z0}p)jd*6G?j=%RjO3-58Aq-D&uUMXwA`=Jm0*?|Whz>}93yzu`6TTF(v7!rYo zHYwpJ2qQSGAR`v=4y*-QdeR?Dr=%YET=Jb}NB*p(+|SpZv$5r*OFbz>c^Cq)c>iDf zKe!Rlw&&phXMBI-NkSlyeEQS>ZyXW+1nu2!yKN}B`syohzx`Kan5<*x`DHJA2_-~9 z*Ij?@uDk96>=+U4#B{7@G2x5C1_QJ--ZG+kAdT|RRAZ%1b4eqm5Xv)?YnzQckQ{lI z=fCp*apoaO0iobk@cj4PcP~gj^w5Jqz0EeyVvc5E%MRzZ+di9;-}yC>rzVI2(cshG@2QEC|KmX_dq^AQ9KEUdX*cAg|vOpYK z5&KY3){5f%V`v^!jH2-08nwtUTN0U^QmxWvKhqLcjnXXg4P!$h;I7*&JQ!^Oo_KEg zf1Qs)5O{3cf>vAd|C~r!xq@;68~h1w1l*9q>8F44rkigB?QebSQofXn+G7Ym{L$ry z9C9#E+c6nd!Dv1mQI~$H8RZGEW zYt0+ye|+XmCr|v$>(LF&wj8gG=Z1J25dQx6FT)*OapfLL5im)J1m95`E;3{lGOf(f#k>&1RDR(&hybe^QJu?={&N4@}7I{=3o!2 zEw&NBf4}|q{nf8-*=z4TdGd>CT>is<*?<2x-hcmnEW|$Wf%pH@|Nb1JA$-NpfWJGE zah1?l?k=ywK(7I9_E_}>JO52hD~>h~F|Nc$)GdaP!X%IZd=Nei9Lg@sQegeMELi>#*@Q)LT;0byJpC~_02`}{F5C8e+KmSk2g}u1H`J0d3a?8zo z@3R;Gvgg|CuEF!*c9>+t;jn{2MK84sPC=s&R6=0Xa7LB4wQlw>j+1^z+~sGbFv*IC zbSVj;iH9=}-R^qhLwP zXVUlB;|)9-zWVB`kPe2xE0c&oi-`Es=s63Yq}KZYKtARRjdVV&|jtKfDWRDcJw7#q(nt=sYjevHl{w zWWy6FoOt5#o*kw*dHWl`@#_a4yl~Gw_h4TR0?@`Hcv!%;KMy5YIIxtE|JMh#@>4L2 z#0tuv8&-4nbhMbH2UQ&drDZc_h9maYEf5AAph*6oT{wFl91dGRcpj7=88LfO zf-FEAgqPlNJ9x6Bg5a*Z?xf__U)@5SLxw+A!CYz6ny(E9dI5Rbp)X4$qTje?h3Vr~ zIG>kHzx_!{2D&ab zwQj%8&&E1(n~gFL$fc4(M!OaSJm_aVMv0)Y{3=Dew2&)sRN$bzc?K zb<9)DKLsJ^oBhA`eQAjB^BJ zMtzMUZt>8!h%A&|^-v`HdDCQqqF~7#Q7$Ps_)O>j?f>)szdkPjhF}hufi|)LVR z!^Q>EE}n|kz*(Nhl7-A_&vgtqg{EX0;{JHYHdQK?rs_<&J5i?Jb)z8_@)bYyrIuB( z;H(lkf1nT=y6S--1o8YAw3FwN|L2j`CgMpN@f7elAXAGc0?>Q+Qu6{OEN|&0&Y46B z`_B@PD`GFuWQPwk>EQ*LoCY|Aw!Xs63+oHf9=i%24&NN@MnjB^^iZ)|a%SDTl{zM}tsBVRgpksp9-~V9)fkHe_dh?W%g!v4c zCR2w)5VO3+jicKXA}0J5PM&r`h%qlftVP_DkK@50L%6toq=|G8lR1Z<*68?IhL$n| zGeSett3HH4HQt$6Q84Lag z4aeup`8XVvP~vN=6!`obAlLnE$aP5O3F zqO@O9m2MJ~c@)j`6vAo*fkgVi7BS5_({FQgmzdQU9V24w%4fZm>&WlCpz~nJDl9vH z#Gtql0<_%|Ytm}roF`mbgN!Q36jW^^l5fn&(Jxo2w?^7Rp29+qXcAe6xYQyuB$r7F zgNISqaV^xnmwaX;H#lw})i8}iiae=;Vt&Lr{-&x2fIv<%vg_rAVdqkvR3!-tngs_0RB_xD9hU>tG<6Z()Fai1WGs4Rx#J1}|K6d`7I>fY=QejCYF&9kZ{~M+4K-a-`Ez1d@*U&Zjnulr#5V*lIgpyHt zsa}oL_3&r<;wDTOnOejB|O5UCjG$!2qYz2oCb1uZ(itSLE!g0|) z;tQL?WyvC%`L{o!rleSLREeKicH%sxHkf7XgBcBcrm1u^J~Py8&4cyY?(jac-?(8yyZBA5JL^DI7o6n?U>ZtAi(<)3c zk5*xlf;_9|b7LszXf1*q^3kBYQ2y_b;G9-r+|91ft*JpL4Wzp)BaAe2uMA_AUm zF$EwF%Y{XO!X)9JqfkkWX~lskHW}GZ?za5%h`$Zh7ebFuK{&yjESoZY*+5VgV``AqsN|FNPdN>V|bUy9>c+GoE{_=p=yy5vH=Al9P8Pps3CWUPFb0`6PJp%UE25>v5 zcW5}bJmaSs*?IZ@r2h?v$$pErG}0UcU{Bcww6oy^pyz>y4$1_9$g>VDd@Gy2n$)F+%m83j1hxiaQj0LvtS`9`Ef!F}- z5_CyHJa{cMn>#buJB7L5O3-CEF}j+jb<3Iz4FCcuE;{!Ac^2mPf3+4yE|e=`u}uLQ zhLDe%Bux(g4~?-^p_AuaPu565Gl-{&xKbtJs3OYDWNVnwsSG)idB|0QG7p~-R0zyC zUtpY@tBvBSkr>K6_?iQHOQp!8g&d|g#IU*nsUsLb5mU~$3ihP;_s+A)*Km0^!A$>QrFUm-B;+4=-EK9lBaaTLybncP=f7MbU43(`pi zkd8?GL^8$y`Y$|3@{kv~u;7v{_@56#Y5JHulB=Yr@}Dq?F(zB?oTuWz2GuqXL{?f>B93GxA% z-nE8}7{SFd_*yULi2x)&}o3AXn{7Q&=-Ox>oM4 zEsLlt#dCQ$%>Uzv z@F8|1NI~vM`<(EAbpFHt7op6~|F1~ZdV9$GZE zFQ$O1ii`fk|8Exm?-x#s H|Ns91fPHQN literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/LICENSE.OpenJPH b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/LICENSE.OpenJPH new file mode 100644 index 00000000..d403b9c7 --- /dev/null +++ b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/LICENSE.OpenJPH @@ -0,0 +1,27 @@ +BSD 2-Clause License + +Copyright (c) 2019, Aous Naman +Copyright (c) 2019, Kakadu Software Pty Ltd, Australia +Copyright (c) 2019, The University of New South Wales, Sydney, Australia +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/README.md b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/README.md new file mode 100644 index 00000000..5d05bc0c --- /dev/null +++ b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/README.md @@ -0,0 +1,129 @@ +# Independent OpenJPH batch fixtures + +This directory contains small HTJ2K fixtures encoded independently of this +workspace with OpenJPH 0.27.0. The compressed fixtures use a 19 x 13 image, +11 x 7 tiles, 8 x 8 code blocks, and two wavelet decompositions. This gives +odd image, tile, edge-code-block, and multi-tile geometry without making the +checked-in corpus large. + +The reversible 5/3 matrix covers signed and unsigned Gray and RGB at 8, 12, +and 16 bits. The irreversible 9/7 matrix adds unsigned RGB8 plus unsigned and +signed Gray12, Gray16, and RGB12 coverage. The >8-bit 9/7 fixtures use +precision-scaled quantization steps (`0.003 / 2^(precision - 8)`) so independent +integer reconstruction can be compared within one native LSB. Their source +patterns deliberately stay in an 8-bit amplitude range while retaining the +declared 12/16-bit and signed sample domains; this isolates 9/7 reconstruction, +native packing, and sign restoration. `gray_u12_53.jph` contains the +independently encoded OpenJPH codestream in a minimal JPH file wrapper. + +The artifacts were generated on 2026-07-18 with the Homebrew arm64 OpenJPH +0.27.0 executables. The exact executable SHA-256 values were: + +```text +b9846d39ca27506e0a93c66e42b287f4730ad071a26fc54bd4024aedaecf280f ojph_compress +4b420506bd2a44439cf472d956bc1552c8be72ff6dffe2c10042e0e36b8de843 ojph_expand +``` + +OpenJPH is distributed under the BSD 2-Clause license reproduced in +`LICENSE.OpenJPH`. + +## Reproduction + +From this directory, use a temporary working directory and build the +deterministic source/oracle utility: + +```sh +rustc --edition 2021 generate.rs -o "$work/generate" +"$work/generate" sources "$work/sources" +``` + +For each unsigned PGM/PPM source, run `ojph_compress` with the following exact +codec options. RGB commands also include `-colour_trans false`. + +```sh +ojph_compress -i "$source" -o "$output" \ + -reversible true -num_decomps 2 -block_size '{8,8}' -tile_size '{11,7}' +``` + +For signed grayscale, run one command per precision (`8`, `12`, and `16`): + +```sh +ojph_compress -i "$work/sources/gray_s${precision}_53.source.raw" \ + -o "$work/gray_s${precision}_53.j2c" \ + -dims '{19,13}' -num_comps 1 -signed true -bit_depth "$precision" \ + -downsamp '{1,1}' -reversible true -num_decomps 2 \ + -block_size '{8,8}' -tile_size '{11,7}' +``` + +OpenJPH 0.27.0's CLI YUV reader loads samples into unsigned containers even +when signed component metadata is requested. Signed RGB is therefore encoded +through the public OpenJPH library, supplying actual negative `i32` component +lines instead of accepting invalid CLI-YUV output: + +```sh +c++ -std=c++17 -Wall -Wextra -Werror \ + -I/opt/homebrew/opt/openjph/include/openjph encode_signed_rgb.cpp \ + -L/opt/homebrew/opt/openjph/lib \ + -Wl,-rpath,/opt/homebrew/opt/openjph/lib -lopenjph \ + -o "$work/encode_signed_rgb" +"$work/encode_signed_rgb" "$work/rgb_s8_53.j2c" 8 +"$work/encode_signed_rgb" "$work/rgb_s12_53.j2c" 12 +"$work/encode_signed_rgb" "$work/rgb_s16_53.j2c" 16 +"$work/encode_signed_rgb" "$work/rgb_s8_53_single.j2c" 8 single-tile +"$work/encode_signed_rgb" "$work/rgb_s12_53_single.j2c" 12 single-tile +"$work/encode_signed_rgb" "$work/rgb_s16_53_single.j2c" 16 single-tile +``` + +The single-tile variants retain the same deterministic samples and therefore +share the corresponding independently decoded `.oracle.raw` files. They +exercise adapter offset-plan reuse, while the original variants retain the +odd multi-tile coverage. + +The irreversible fixture uses the checked-in RGB8 PPM source: + +```sh +ojph_compress -i "$work/sources/rgb_u8_97.source.ppm" \ + -o "$work/rgb_u8_97.j2c" -reversible false -qstep 0.003 \ + -num_decomps 2 -block_size '{8,8}' -tile_size '{11,7}' \ + -colour_trans false +``` + +The checked-in `*_97.source.*` files reproduce the >8-bit single-tile 9/7 +fixtures. Gray12/RGB12 use `-qstep 0.0001875`; Gray16 uses +`-qstep 0.00001171875`. Signed grayscale uses the same raw-input flags as the +reversible commands. Signed RGB12 uses the library helper with both +`irreversible` and `single-tile` arguments: + +```sh +"$work/encode_signed_rgb" "$work/rgb_s12_97.j2c" 12 irreversible single-tile +``` + +Each `.oracle.raw` file is derived from a separate OpenJPH decode. PFM is used +as the intermediate because it preserves signed reconstructed sample codes; +the utility reverses PFM's bottom-up rows and removes its left bit alignment: + +```sh +ojph_expand -i "$work/$name.j2c" -o "$work/$name.oracle.pfm" +"$work/generate" oracle "$work/$name.oracle.pfm" \ + "$work/$name.oracle.raw" "$precision" "$signed" +``` + +Oracle samples are top-down, interleaved NHWC. Precision up to 8 bits uses one +byte per sample; higher precision uses little-endian 16-bit containers. Signed +values use two's-complement `i8` or `i16` representation. The reversible +oracles were also checked against the deterministic source formula: + +```sh +"$work/generate" verify "$work/$name.oracle.raw" \ + "$components" "$precision" "$signed" +``` + +OpenJPH 0.27.0 writes raw Part 15 codestreams but does not serialize the JPH +box container. The fixture utility independently constructs the standard JPH +signature, `ftyp`, `jp2h` (`ihdr` plus `colr`), and `jp2c` boxes around the +OpenJPH-generated Gray12 codestream: + +```sh +"$work/generate" jph "$work/gray_u12_53.j2c" \ + "$work/gray_u12_53.jph" 1 12 false +``` diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/encode_signed_rgb.cpp b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/encode_signed_rgb.cpp new file mode 100644 index 00000000..ce942f65 --- /dev/null +++ b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/encode_signed_rgb.cpp @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#include +#include +#include +#include + +#include "ojph_codestream.h" +#include "ojph_defs.h" +#include "ojph_file.h" +#include "ojph_mem.h" +#include "ojph_params.h" + +namespace { + +constexpr std::uint32_t kWidth = 19; +constexpr std::uint32_t kHeight = 13; + +std::int32_t sample(std::uint32_t x, std::uint32_t y, + std::uint32_t component, std::uint32_t precision) { + const std::uint32_t modulus = 1U << precision; + const std::uint32_t code = + (x * 37U + y * 73U + component * 109U + x * y * 3U + + component * y * 11U) % + modulus; + return static_cast(code) - + static_cast(modulus / 2U); +} + +} // namespace + +int main(int argc, char** argv) { + assert(argc >= 3 && argc <= 5); + const char* output_path = argv[1]; + const std::uint32_t precision = + static_cast(std::strtoul(argv[2], nullptr, 10)); + assert(precision == 8 || precision == 12 || precision == 16); + bool single_tile = false; + bool reversible = true; + for (int index = 3; index < argc; ++index) { + if (std::strcmp(argv[index], "single-tile") == 0) { + single_tile = true; + } else if (std::strcmp(argv[index], "irreversible") == 0) { + reversible = false; + } else { + assert(false && "unsupported fixture mode"); + } + } + + ojph::codestream codestream; + ojph::param_siz siz = codestream.access_siz(); + siz.set_image_extent(ojph::point(kWidth, kHeight)); + siz.set_num_components(3); + for (std::uint32_t component = 0; component < 3; ++component) { + siz.set_component(component, ojph::point(1, 1), precision, true); + } + siz.set_tile_size(single_tile ? ojph::size(kWidth, kHeight) + : ojph::size(11, 7)); + + ojph::param_cod cod = codestream.access_cod(); + cod.set_num_decomposition(2); + cod.set_block_dims(8, 8); + cod.set_color_transform(false); + cod.set_reversible(reversible); + if (!reversible) { + const float qstep = 0.003F / static_cast(1U << (precision - 8U)); + codestream.access_qcd().set_irrev_quant(qstep); + } + codestream.set_planar(true); + + ojph::j2c_outfile output; + output.open(output_path); + codestream.write_headers(&output); + + ojph::ui32 next_component = 0; + ojph::line_buf* line = codestream.exchange(nullptr, next_component); + for (std::uint32_t component = 0; component < 3; ++component) { + for (std::uint32_t y = 0; y < kHeight; ++y) { + assert(next_component == component); + assert(line != nullptr && line->i32 != nullptr && line->size >= kWidth); + for (std::uint32_t x = 0; x < kWidth; ++x) { + line->i32[x] = sample(x, y, component, reversible ? precision : 8U); + } + line = codestream.exchange(line, next_component); + } + } + + codestream.flush(); + codestream.close(); + output.close(); + return 0; +} diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/generate.rs b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/generate.rs new file mode 100644 index 00000000..bff4b1de --- /dev/null +++ b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/generate.rs @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; + +const WIDTH: u32 = 19; +const HEIGHT: u32 = 13; + +#[derive(Clone, Copy)] +struct FixtureSpec { + components: u32, + precision: u8, + signed: bool, +} + +fn sample(spec: FixtureSpec, x: u32, y: u32, component: u32) -> i32 { + let modulus = 1_i32 << spec.precision; + let value = i32::try_from( + (x * 37 + y * 73 + component * 109 + x * y * 3 + component * y * 11) + % u32::try_from(modulus).expect("fixture modulus is positive"), + ) + .expect("fixture sample fits i32"); + if spec.signed { + value - modulus / 2 + } else { + value + } +} + +fn irreversible_sample(spec: FixtureSpec, x: u32, y: u32, component: u32) -> i32 { + let value = i32::try_from( + (x * 37 + y * 73 + component * 109 + x * y * 3 + component * y * 11) % 256, + ) + .expect("fixture sample fits i32"); + if spec.signed { + value - 128 + } else { + value + } +} + +fn append_sample(bytes: &mut Vec, sample: i32, precision: u8, signed: bool) { + if precision <= 8 { + if signed { + bytes.push(i8::try_from(sample).expect("signed 8-bit fixture sample") as u8); + } else { + bytes.push(u8::try_from(sample).expect("unsigned 8-bit fixture sample")); + } + } else if signed { + bytes.extend_from_slice( + &i16::try_from(sample) + .expect("signed 16-bit fixture sample") + .to_le_bytes(), + ); + } else { + bytes.extend_from_slice( + &u16::try_from(sample) + .expect("unsigned 16-bit fixture sample") + .to_le_bytes(), + ); + } +} + +fn planar_source_bytes(spec: FixtureSpec) -> Vec { + let bytes_per_sample = if spec.precision <= 8 { 1 } else { 2 }; + let mut bytes = Vec::with_capacity( + WIDTH as usize * HEIGHT as usize * spec.components as usize * bytes_per_sample, + ); + for component in 0..spec.components { + for y in 0..HEIGHT { + for x in 0..WIDTH { + let value = sample(spec, x, y, component); + if spec.components == 3 && spec.signed && (9..16).contains(&spec.precision) { + // OpenJPH's YUV reader loads 9-16-bit samples as unsigned + // containers even when the codestream component is signed. + // Store the precision-bit two's-complement code instead of + // a sign-extended i16 container so the independent encoder + // receives the intended signed sample modulo 2^precision. + let mask = (1_u16 << spec.precision) - 1; + bytes.extend_from_slice(&((value as u16) & mask).to_le_bytes()); + } else { + append_sample(&mut bytes, value, spec.precision, spec.signed); + } + } + } + } + bytes +} + +fn pnm_source_bytes(spec: FixtureSpec) -> Vec { + assert!(!spec.signed); + let magic = if spec.components == 1 { "P5" } else { "P6" }; + let max_value = (1_u32 << spec.precision) - 1; + let mut bytes = format!("{magic}\n{WIDTH} {HEIGHT}\n{max_value}\n").into_bytes(); + for y in 0..HEIGHT { + for x in 0..WIDTH { + for component in 0..spec.components { + let sample = u16::try_from(sample(spec, x, y, component)) + .expect("unsigned PNM fixture sample"); + if spec.precision <= 8 { + bytes.push(u8::try_from(sample).expect("unsigned 8-bit PNM sample")); + } else { + bytes.extend_from_slice(&sample.to_be_bytes()); + } + } + } + } + bytes +} + +fn irreversible_source_bytes(spec: FixtureSpec) -> Vec { + if spec.signed { + let mut bytes = Vec::new(); + for component in 0..spec.components { + for y in 0..HEIGHT { + for x in 0..WIDTH { + append_sample( + &mut bytes, + irreversible_sample(spec, x, y, component), + spec.precision, + true, + ); + } + } + } + return bytes; + } + + let magic = if spec.components == 1 { "P5" } else { "P6" }; + let max_value = (1_u32 << spec.precision) - 1; + let mut bytes = format!("{magic}\n{WIDTH} {HEIGHT}\n{max_value}\n").into_bytes(); + for y in 0..HEIGHT { + for x in 0..WIDTH { + for component in 0..spec.components { + let sample = u16::try_from(irreversible_sample(spec, x, y, component)) + .expect("unsigned irreversible fixture sample"); + if spec.precision <= 8 { + bytes.push(u8::try_from(sample).expect("8-bit irreversible sample")); + } else { + bytes.extend_from_slice(&sample.to_be_bytes()); + } + } + } + } + bytes +} + +fn interleaved_native_bytes(spec: FixtureSpec) -> Vec { + let mut bytes = Vec::new(); + for y in 0..HEIGHT { + for x in 0..WIDTH { + for component in 0..spec.components { + append_sample( + &mut bytes, + sample(spec, x, y, component), + spec.precision, + spec.signed, + ); + } + } + } + bytes +} + +fn write_sources(output: &Path) { + fs::create_dir_all(output).expect("create fixture output directory"); + + for components in [1, 3] { + for signed in [false, true] { + if components == 3 && signed { + continue; + } + for precision in [8, 12, 16] { + let shape = if components == 1 { "gray" } else { "rgb" }; + let sign = if signed { "s" } else { "u" }; + let name = format!("{shape}_{sign}{precision}_53"); + let spec = FixtureSpec { + components, + precision, + signed, + }; + let extension = match (components, signed) { + (1, false) => "pgm", + (3, false) => "ppm", + (1, true) => "raw", + (3, true) => "yuv", + _ => unreachable!("fixture catalog uses one or three components"), + }; + let source = if signed { + planar_source_bytes(spec) + } else { + pnm_source_bytes(spec) + }; + fs::write(output.join(format!("{name}.source.{extension}")), source) + .expect("write reversible fixture source"); + } + } + } + + let irreversible = FixtureSpec { + components: 3, + precision: 8, + signed: false, + }; + fs::write( + output.join("rgb_u8_97.source.ppm"), + pnm_source_bytes(irreversible), + ) + .expect("write irreversible fixture source"); + + for (components, precision, signed) in [ + (1, 12, false), + (1, 16, false), + (1, 12, true), + (1, 16, true), + (3, 12, false), + ] { + let shape = if components == 1 { "gray" } else { "rgb" }; + let sign = if signed { "s" } else { "u" }; + let extension = if signed { + "raw" + } else if components == 1 { + "pgm" + } else { + "ppm" + }; + let spec = FixtureSpec { + components, + precision, + signed, + }; + fs::write( + output.join(format!("{shape}_{sign}{precision}_97.source.{extension}")), + irreversible_source_bytes(spec), + ) + .expect("write >8-bit irreversible fixture source"); + } +} + +fn pfm_payload(bytes: &[u8]) -> (&[u8], u32, u32) { + let mut lines = bytes.splitn(4, |byte| *byte == b'\n'); + let magic = lines.next().expect("PFM magic"); + assert!(magic == b"Pf" || magic == b"PF", "PFM magic"); + let dimensions = std::str::from_utf8(lines.next().expect("PFM dimensions")) + .expect("PFM dimensions are ASCII"); + let mut dimensions = dimensions.split_ascii_whitespace(); + let width = dimensions + .next() + .expect("PFM width") + .parse::() + .expect("PFM width is an integer"); + let height = dimensions + .next() + .expect("PFM height") + .parse::() + .expect("PFM height is an integer"); + let scale = std::str::from_utf8(lines.next().expect("PFM scale")) + .expect("PFM scale is ASCII") + .parse::() + .expect("PFM scale is numeric"); + assert!( + scale.is_sign_negative(), + "fixture PFM must be little-endian" + ); + (lines.next().expect("PFM payload"), width, height) +} + +fn write_oracle(input: &Path, output: &Path, precision: u8, signed: bool) { + let bytes = fs::read(input).expect("read OpenJPH PFM oracle"); + let (payload, width, height) = pfm_payload(&bytes); + let pixels = usize::try_from(width * height).expect("fixture pixels fit usize"); + assert!(payload.len() == pixels * 4 || payload.len() == pixels * 3 * 4); + let components = payload.len() / (pixels * 4); + let shift = u32::from(32 - precision); + let mut oracle = Vec::with_capacity(pixels * components * if precision <= 8 { 1 } else { 2 }); + for y in (0..height).rev() { + for x in 0..width { + for component in 0..components { + let index = + ((y as usize * width as usize + x as usize) * components + component) * 4; + let bytes = payload[index..index + 4] + .try_into() + .expect("PFM sample is four bytes"); + let value = if signed { + i32::from_le_bytes(bytes) >> shift + } else { + i32::try_from(u32::from_le_bytes(bytes) >> shift) + .expect("unsigned fixture oracle fits i32") + }; + append_sample(&mut oracle, value, precision, signed); + } + } + } + fs::write(output, oracle).expect("write raw OpenJPH oracle"); +} + +fn append_box(output: &mut Vec, box_type: &[u8; 4], payload: &[u8]) { + let length = u32::try_from(payload.len() + 8).expect("fixture box length fits u32"); + output.extend_from_slice(&length.to_be_bytes()); + output.extend_from_slice(box_type); + output.extend_from_slice(payload); +} + +fn write_jph(input: &Path, output: &Path, components: u16, precision: u8, signed: bool) { + let codestream = fs::read(input).expect("read OpenJPH codestream"); + let mut jph = Vec::with_capacity(codestream.len() + 100); + append_box(&mut jph, b"jP ", &[0x0d, 0x0a, 0x87, 0x0a]); + let mut file_type = Vec::with_capacity(12); + file_type.extend_from_slice(b"jph "); + file_type.extend_from_slice(&0_u32.to_be_bytes()); + file_type.extend_from_slice(b"jph "); + append_box(&mut jph, b"ftyp", &file_type); + + let mut image_header = Vec::with_capacity(14); + image_header.extend_from_slice(&HEIGHT.to_be_bytes()); + image_header.extend_from_slice(&WIDTH.to_be_bytes()); + image_header.extend_from_slice(&components.to_be_bytes()); + image_header.push((precision - 1) | if signed { 0x80 } else { 0 }); + image_header.extend_from_slice(&[7, 0, 0]); + let mut jp2_header = Vec::with_capacity(45); + append_box(&mut jp2_header, b"ihdr", &image_header); + let color_space = if components == 1 { 17_u32 } else { 16_u32 }; + let mut color = vec![1, 0, 0]; + color.extend_from_slice(&color_space.to_be_bytes()); + append_box(&mut jp2_header, b"colr", &color); + append_box(&mut jph, b"jp2h", &jp2_header); + append_box(&mut jph, b"jp2c", &codestream); + fs::write(output, jph).expect("write independently boxed JPH fixture"); +} + +fn parse_bool(value: &str) -> bool { + match value { + "true" => true, + "false" => false, + _ => panic!("signedness must be true or false"), + } +} + +fn next_u32(arguments: &mut impl Iterator, what: &str) -> u32 { + arguments + .next() + .unwrap_or_else(|| panic!("missing {what}")) + .to_str() + .unwrap_or_else(|| panic!("{what} is UTF-8")) + .parse::() + .unwrap_or_else(|_| panic!("{what} is an integer")) +} + +fn main() { + let mut arguments = env::args_os().skip(1); + let mode = arguments + .next() + .expect("usage: generate sources DIR | oracle IN.pfm OUT.raw PRECISION SIGNED | verify ORACLE COMPONENTS PRECISION SIGNED | jph IN.j2c OUT.jph COMPONENTS PRECISION SIGNED"); + match mode.to_str().expect("mode is UTF-8") { + "sources" => { + let output = PathBuf::from(arguments.next().expect("sources output directory")); + write_sources(&output); + } + "oracle" => { + let input = PathBuf::from(arguments.next().expect("PFM input")); + let output = PathBuf::from(arguments.next().expect("raw oracle output")); + let precision = arguments + .next() + .expect("oracle precision") + .to_str() + .expect("precision is UTF-8") + .parse::() + .expect("precision is an integer"); + let signed = parse_bool( + arguments + .next() + .expect("oracle signedness") + .to_str() + .expect("signedness is UTF-8"), + ); + write_oracle(&input, &output, precision, signed); + } + "verify" => { + let input = PathBuf::from(arguments.next().expect("raw oracle input")); + let components = next_u32(&mut arguments, "component count"); + let precision = + u8::try_from(next_u32(&mut arguments, "precision")).expect("precision fits u8"); + let signed = parse_bool( + arguments + .next() + .expect("oracle signedness") + .to_str() + .expect("signedness is UTF-8"), + ); + let actual = fs::read(input).expect("read raw OpenJPH oracle"); + let expected = interleaved_native_bytes(FixtureSpec { + components, + precision, + signed, + }); + assert_eq!(actual, expected, "reversible OpenJPH oracle"); + } + "jph" => { + let input = PathBuf::from(arguments.next().expect("codestream input")); + let output = PathBuf::from(arguments.next().expect("JPH output")); + let components = arguments + .next() + .expect("JPH component count") + .to_str() + .expect("component count is UTF-8") + .parse::() + .expect("component count is an integer"); + let precision = arguments + .next() + .expect("JPH precision") + .to_str() + .expect("precision is UTF-8") + .parse::() + .expect("precision is an integer"); + let signed = parse_bool( + arguments + .next() + .expect("JPH signedness") + .to_str() + .expect("signedness is UTF-8"), + ); + write_jph(&input, &output, components, precision, signed); + } + mode => panic!("unknown generator mode {mode}"), + } + assert!(arguments.next().is_none(), "unexpected generator arguments"); +} diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_53.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_53.j2c new file mode 100644 index 0000000000000000000000000000000000000000..f7e6001ae9a76e344f5131060fddc3c9c4b3255d GIT binary patch literal 404 zcmezG|38pH(}4j9gn^hB#DD^BAk7ZrFfevAGX4)>;9y_^3bFkUV&DO?7#Wxt866n^ z$1rdyWE2z>WMpLgPhk*eVDv9Y&GQQIPzXycQZUdnGS@TE`#*t!3v8M#1LOb62blK! z&%btk-rggAFBn=Fer{tpz-YoCYQW$%=|m%U34_v70R|3-@-T*P4d+W37{Hntf!duQ znpaG=>6tzEH)FdAP+uSe%LN86jy9H`3{6`YHe6q-U(&p4Rj1H%f2lUh0Xhqy~2 zCV<=?1hu~N_QXp*zdtDaV@P56Zw55RhC$eYQQ_glKWw%s3==+ZeKTSB!>-J5fGL6@ zoZ(Las|v%!xgokY{#Y;qLn+GwVj?rp#Bhj-OTOmrv=(J_hB|2ig9o3`J0*tXDGaxa wj2ZqgJO;XFRRZH|a|TX_a@L6s9Q6XP%Nedeb#O9a`1lt@{z+&rVfcR&02cLnTmS$7 literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_53.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_53.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..58f3f77ef5c041e22aa35fac67d68e63e21ea5db GIT binary patch literal 494 zcmWO0%}Z2a6bIlN1Hl9qmqE;h3u}zTO+g`2Ru=X#A#@2>66InRHcn+BG(?N&16owx zX|ill3?We@B9#^vF>SB#n}6Wp@l1nBnbW3VUYoBbMw_XF z&eC0aL-VwnoA?O#^L;*K`pl3inI+>>3w6_V8ly6;;zkZQ$3>pz6|P0zyfup^LA$7j z3iOVCP&M!23=eXNe{v1>B8z*NHF0XDG~K2WEmIvIt_Y8LhGW=@66InRHcn+BG(?N&16owx zX|ill3?We@B9#^vF>SB#n}6Wp@l1nBnbW3VUYoBbMw_XF z&eC0aL-VwnoA?O#^L;*K`pl3inI+>>3w6_V8ly6;;zkZQ$3>pz6|P0zyfup^LA$7j z3iOVCP&M!23=eXNe{v1>B8z*NHF0XDG~K2WEmIvIt_Y8LhGW=@?XYT6yqE3#HpV^D_1okj8uC`)0z_9HBL$m>d?t&XC;tb)2S=}v1 zp1;br6q)paF})>5u|kP)7lYiF+~}qG^D`F6%x}A~mO&xz{vrMb2WD#q6EBhdau;O` z8uopu_FwRk;b8gwhth`x6lXI$V7S-2_5kBv=FA5ScfV~|p`j%`f6t^8VMY}uUZJP) zrE4#}_c-=}Q}f{+jb!m+t?HnQ%Mx8~T{l$M;@TBuaNZ#=_8;5#+t(Qn_Z8n&5@?WS zV91`txWsGGs^dGwfB*i-UCzK36Vd&lrTE{h?H6Wky)X6DpXb@k{X2H;{d;dQ+nQ=+ z18(k^${_g#y;9dVCb}uf{9||+T5P}m-dh$0h9&{!35-3x%WqwIv3s6J(JqC5jKND1 zr!n*`6JB{nZ0o|nn?Y%l;&esXHPsuEpRr ihmOQgTNR?f!10BFaotA7J!(=0t1pTI9aqNi|0V!9>A_I| literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_97.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_97.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..45300abd3eb59f61bfea9af7decfeb00dfb1d73a GIT binary patch literal 494 zcmZXRNhpP37=@1vWnrEzNRp5wb4YAtCAV&wWn(B3l8rH?C?p9B3uA6~a~GtW@zOP> zNLCUS7Un5M(s7o`>hJ%)bI$vm_v_*u5BN-9q@f%=T;LJk7=R2^qL*eK^Mk?I=PQGd zjam%T%4@nJ9!2P;i6{JG81m4F33?+FH5lPKZZf68!O~Kcn*2#1|o= z0oueHCG9Zn%8gZvQ6{4vvpg4H9=bI9N!n6z9?$`)VwmTJcspU`yByV+lGYFT%H8G< zQ`F6@X`#v=;U1llBkiivlhF)|Iz`JftF45n+qk&B;e`SbobdnG@aMc(?XMV)Dwz1d zI5c62pG-#wY;-~{`t=r`n9(-l(1I1w=BnYTUZMhX+G{*ou|`L=8j?RrzFp~yFs-{p Sv}1=JC{&v}CZG*=y5kp(|M)Wi literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_97.source.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s12_97.source.raw new file mode 100644 index 0000000000000000000000000000000000000000..0fb5d08167fe208996debb562e451b6dfe325462 GIT binary patch literal 494 zcmaKp$t#6n7{!kaWnrEzNRo&ob4YA%cO^=hWn+qjWMfPz3Q4!?E-Y}b$c7@xOy(5H zO2WdzJf%oF&R<~l_PyVE&iS3^?WB!|e4#f|P>OC&@tE)QLmJA_!x^6NlYuzk8v~Gu zY7B9SH*`iU3ed%Ap7NU^Fron#dLkWF80H%99p0ek7MHi$h#Y`*D>4YTtS;Y6lD3oA;>wHyOuC()V@Wqq9^5nNFzAzE> zb5pz#(hhM)zR_x7Vj}8b<%Rf+=v4Nzw8i2)q#csQFv&~tc3_DgnyAE>v_8m|?>2v# zq;4}x3zq*d_vwf%X;13B+DeeRjfmS59>@{FDgQqU)&82HsK7KI8G}YF z@QbNv$0i+MK%d^i9TVDS44SblT7w#%=_Sfw(_UlIf>qk9)u84RG`A;x0mk(&9&OmA P8}ilWo^fc!4qfpFlM(nf literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_53.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_53.j2c new file mode 100644 index 0000000000000000000000000000000000000000..a490c62f5e318edd9a1a4c478808d74885f2b6b9 GIT binary patch literal 429 zcmYL_JxD@P6vzM9G#}3jNu*V*h6+VW&_#IEP_$?zMi5b0L(vyyIQopJ*HX`Plc1&M z25FXPsHqQvGH8%!3ZjM#@%nC8O+0Y!J@>---QT&)&&JRrfro`>Yo+)P6W@*X1IQsE zEC2yIekHPTwD5{RM~Fn&G(?-Cs;Z(WY#tpT{`G~`(ZGl;w6I}wIr}|Mmy@XwtF$2y zmdwIoW3iR;la?!8X*(JbP`p4E7711t^e0ExaZ?2LQ!@l98bU$7jQ~{y;gI1Pq7O+r zN7Loak}jU%%~mt?cf48H)fcgy!TCMHL1eXo34B46Z+e&4E=^lCOl6Fzy2*a&fIiU0 zQr@RDOsX>N$R}vCZyxHGSa5?HJNY}^GA2RS4P8ha2OZDUOZdMnd28S%4xXv|{rotJ zuf@mThMx+(+}8eSM>vI-{sB0KIhFizvEcXYW CZ-|co literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_53.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_53.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..dc5e5ed6dcc875c126be43e20f0b77f327026527 GIT binary patch literal 494 zcmWO0%}Z2a6bE2qAeg}7GKjfwVU3ZvDJVqB%ECS-gf8JqqFl_v#<47fh6vtse&@XJ zw5Yt(WZ9w^LZV1SDlIHx+K8*TfzV_YZc?KVi)d5Y`~we9g{e0QbI#<=OY_-8Xgjsj z1-ebIXr9({BcI@YzRSl^!l zEm939_%c7?Io^ol!ulMIPY=u0tno;T@uKNUq3)tVp9K^}d!g zW{ zJ@=7oaRv|Y6%EoOqw-UB>m{AkRc*Cmr|bsT?grdebYm31v8TM9l1e?PL;6MQZOV?> rKeoleO}kCm9z4e?j>v$NWQ(Tsnf}wm_J;jn>)cuQ)ctjB+3VT=TJZM3 literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_53.source.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_53.source.raw new file mode 100644 index 0000000000000000000000000000000000000000..dc5e5ed6dcc875c126be43e20f0b77f327026527 GIT binary patch literal 494 zcmWO0%}Z2a6bE2qAeg}7GKjfwVU3ZvDJVqB%ECS-gf8JqqFl_v#<47fh6vtse&@XJ zw5Yt(WZ9w^LZV1SDlIHx+K8*TfzV_YZc?KVi)d5Y`~we9g{e0QbI#<=OY_-8Xgjsj z1-ebIXr9({BcI@YzRSl^!l zEm939_%c7?Io^ol!ulMIPY=u0tno;T@uKNUq3)tVp9K^}d!g zW{ zJ@=7oaRv|Y6%EoOqw-UB>m{AkRc*Cmr|bsT?grdebYm31v8TM9l1e?PL;6MQZOV?> rKeoleO}kCm9z4e?j>v$NWQ(Tsnf}wm_J;jn>)cuQ)ctjB+3VT=TJZM3 literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_97.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_97.j2c new file mode 100644 index 0000000000000000000000000000000000000000..e5abbfaf8c7e0fbaf5791b5854891d84ec660011 GIT binary patch literal 507 zcmezG|38pH(}4j9gn^hB#DD@+K4U*4^F1#fpE(d^(pF;FZ@qo5NBZYFG$Vv3h+<}OD$3`&@(dEGtm1#fq@HT0b>UP5u|kP)7lYiF+~}qG^D`F6%x}A~mO&xz{vrMb2WD#q6EBhdau;O` z8uopu_FwRk;b8gwhth`x6lXI$VBmdm>kb3sUgk_7^L{qN3Jopk`Fkd%2s5fM@d`bS zFI{`-y~nW+oSF~sXe5glYgGqbT$bo^>$;)37T2yQgYyn~vH#e<-@eXxxUcxGl0btr z14H&C#wA{hRvq6d{`>bw?s5jUn27EVEye$4ZND&U>wT%G{yfiS?%%O%@85fi+16Am z8*p>SR0hc}=#{#@G0{y)<{!hu&|>@b_ujH7Ff<7$PhjlnU4HAzi{0})igqddV+>xB zIE|rinefUpVp|sm-V92c6sIf7uDNF6@~QJv9K$(RB!67d>iT0+#OtyNl07Z^*1ci~ iK6E60+Nux*297TbjO#Ws?opF6Sbb3x=(sY5|2F|}c)ytd literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_97.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_97.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..45300abd3eb59f61bfea9af7decfeb00dfb1d73a GIT binary patch literal 494 zcmZXRNhpP37=@1vWnrEzNRp5wb4YAtCAV&wWn(B3l8rH?C?p9B3uA6~a~GtW@zOP> zNLCUS7Un5M(s7o`>hJ%)bI$vm_v_*u5BN-9q@f%=T;LJk7=R2^qL*eK^Mk?I=PQGd zjam%T%4@nJ9!2P;i6{JG81m4F33?+FH5lPKZZf68!O~Kcn*2#1|o= z0oueHCG9Zn%8gZvQ6{4vvpg4H9=bI9N!n6z9?$`)VwmTJcspU`yByV+lGYFT%H8G< zQ`F6@X`#v=;U1llBkiivlhF)|Iz`JftF45n+qk&B;e`SbobdnG@aMc(?XMV)Dwz1d zI5c62pG-#wY;-~{`t=r`n9(-l(1I1w=BnYTUZMhX+G{*ou|`L=8j?RrzFp~yFs-{p Sv}1=JC{&v}CZG*=y5kp(|M)Wi literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_97.source.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s16_97.source.raw new file mode 100644 index 0000000000000000000000000000000000000000..0fb5d08167fe208996debb562e451b6dfe325462 GIT binary patch literal 494 zcmaKp$t#6n7{!kaWnrEzNRo&ob4YA%cO^=hWn+qjWMfPz3Q4!?E-Y}b$c7@xOy(5H zO2WdzJf%oF&R<~l_PyVE&iS3^?WB!|e4#f|P>OC&@tE)QLmJA_!x^6NlYuzk8v~Gu zY7B9SH*`iU3ed%Ap7NU^Fron#dLkWF80H%99p0ek7MHi$h#Y`*D>4YTtS;Y6lD3oA;>wHyOuC()V@Wqq9^5nNFzAzE> zb5pz#(hhM)zR_x7Vj}8b<%Rf+=v4Nzw8i2)q#csQFv&~tc3_DgnyAE>v_8m|?>2v# zq;4}x3zq*d_vwf%X;13B+DeeRjfmS59>@{FDgQqU)&82HsK7KI8G}YF z@QbNv$0i+MK%d^i9TVDS44SblT7w#%=_Sfw(_UlIf>qk9)u84RG`A;x0mk(&9&OmA P8}ilWo^fc!4qfpFlM(nf literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s8_53.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s8_53.j2c new file mode 100644 index 0000000000000000000000000000000000000000..907b2bc37324ec7c0fdc0ccd5acf817621a8785f GIT binary patch literal 665 zcmX|;Ur19?9LK+Bf81~iHzIu)+0B$nGG*6@=G_SYc_FJ`|LsGilqsn?*_@ z2F{d%_1Z(wLz+ppJ2bLC52i?wFH@u>NUnd1vEA*S&Sv%FcR0WEd-(A=pC7HG^(b?J z#ZGn#v&G-AvD@-C$Qk@}#S`}iuL~{f2h*KXwWMGXBK|FuU_Cw`j zuZOT{47a{t0}uTLVh3NE$1xTZ9>-~WVk4kLXX(}WtfrZTno@fV{5eP^&dC5v$)A>M zWWPlHMXBZfZjL_-Sx}TXy-5Qay5Cf9S#^+(N#v*q)$)CZ zj@n8TH6=ktAqBssaJv#q_S*X3mER+I&&A_mbwz7iRA`;>3nV%WDW0&c_j$2LmXfKY zt>T^+7c#o;*L7q1;t+J&r6J@oSTbGECG>3AI=30JTKl#cUNx`mPmN}^Fa~#FX06e} zhkR97{*3lHFUHpjvDY})$$a#hComR^2=7*t!DwVaU8U{d8^2hu1B;LFGrC(@a8@ak z&Xa;bYt;dT__TDhfh&Co>d(+XLZd!)bw>3DPB0@1W^zit{s9qR3poG) literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s8_53.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s8_53.oracle.raw new file mode 100644 index 00000000..0ad3ad29 --- /dev/null +++ b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_s8_53.oracle.raw @@ -0,0 +1 @@ +9^;9y_^3bFkUV&DO?7#Wxt866n^ z$1rdyWE2z>WMpLgPhk*eVDv9Y&GQQIPzXycQZUdnGS@TE`#*t!3v8M#1LOb62blK! z&%btk-rggAFBn=Fer{tpz-YoCYQW$%=|m%U34_v70R|3-@-T*P4d+W37{Hntf!duQ znpaG=>6tzEH)FdAP+uSe%LN86jy9H`3{6`YHe6q-U(&p4Rj1H%f2lUh0Xhqy~2 zCV<=?1hu~N_QXp*zdtDaV@P56Zw55RhC$eYQQ_glKWw%s3==+ZeKTSB!>-J5fGL6@ zoZ(Las|v%!xgokY{#Y;qLn+GwVj?rp#Bhj-OTOmrv=(J_hB|2ig9o3`J0*tXDGaxa wj2ZqgJO;XFRRZH|a|TX_a@L6s9Q6XP%Nedeb#O9a`1lt@{z+&rVfcR&0Ac2N+yDRo literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.jph b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.jph new file mode 100644 index 0000000000000000000000000000000000000000..70e17f93b8543e6645c81c224ed187f5c335a0a8 GIT binary patch literal 489 zcmXv~IYv`e&6FoYP4pu(b!1ko5pEi8mYS%aD-M1qwEVh}76wezSH zf|Z2`f8@X`eTN8m&kY7-KO zm+PPrKMR91`s|hs$QSu^6=2KnsYi zBn^iwczv^Zbkg~Pd3^38Nluj8;a)zS6~eG?7sDh)`jDPE4+DQJD0VZ|3>Mhxk>;S4#YWAFP4f zG{Yj1_!w7nTl|Qn%xNc7V-=Ey0rW#va;g#E=&id|8;So?piy_s;*R~gM!2j(Vtg;(2z4Xw8K@DUuVHOw+}$5!^gWtxmjuYqvTS4TIhN0&G@? AlK=n! literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..d181ee66a35f3aa238cd670b0a4d7cfbb6b6624c GIT binary patch literal 494 zcmWO0O-qzv6b9hu-1j-pI}=#E3}P-^SYsq^3JQ_3vapW{p&?vJlnWO&T3HB5w1_^S zMdjFJ*`gRiqDVw4Ei7W%h^v@DXfg{osZoeUv?*==z{TZUl}osjF5_Oh&yKQDn&h9yK_GZrsHI*X0IW&Mml5>ZDb!%7_$1P>m#dki{gHP|iO0+ReL|?2vZJ$Q$`4 zrPzfO`jNwTlyNsZd56<3Dz(xkHzg;FQh|d-;UT6#Zst*5<2V;srg3SOD>5eYQjP@9 z;R$B2nmc)lS-vNAi?(UMc1l*JB#L@;VGsqBuz}rt#2JR#s69HQvs!AS@rdTkXKmb;t-)XN6Mj|D6!Zq`+0J49=Cdg pVlWx3iFEKem$^@SwV>-PWzX!N?e#bO2VWT+51s~pgT~19$bTz$dq4mH literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.source.pgm b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_53.source.pgm new file mode 100644 index 0000000000000000000000000000000000000000..8b95b8a06f22c76c1ee269a3648003d17e727929 GIT binary patch literal 508 zcmWO0&r8#B7zgmr^Sqzu^Kk=aPRtSs!uLU4p9iSn?6%~cjcBRWJs zphM->vg}ZdkSG$7N(+lv7uhKm2raYlk{X3rM3>U`54?DFhW(+)#?Ur@b5kVjH^$VP zs5xn}=B4>epsmzFr|CAmqItI5$eldEcg;c5Z-!0YERjbM>Y=MNP9-L8;25Vk#|2(N zjmenTW|0E4lX@vjZ|ECW@orA=5a;kVafHwCC@-K6-MEE!;*-5{ zUM6Kl8#S)?wPXW)fG_h~Uc(NgFo9pvBuRNJ3mViOy{EHwgYB{zPID0+Vo0M{p2Q?A zMfEhMX)RjM#%$UaoyQ+hi(`0zuhJlWGA2JYtY>vfS8bcUWT&0qb+|!nMlZ(jTeg?C z(^{#Aby&aHdYiE0_K$0I7hR!ZJ^Jt*tFm7PrKFoQq0jW6-Dhvu53bG~cTe5liuQ`@ F{{dvKeWL&X literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_97.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_97.j2c new file mode 100644 index 0000000000000000000000000000000000000000..3eeff0c164ee00f6bfd8125b491411ec0ffdc22f GIT binary patch literal 523 zcmezG|38pH(}4j9gn^hB#DD@+J|j0H)}e!>sO>BhO!DTZ&Bjz?j|=qgbKDxQjvVOK$Yi{P`IRWahWs zSj(UgcmEK7g9EcQgNc{Oez}V>1`YeZRQoUZ$Z)Xy{zK_Q0*bR49x&YNU3-9WFLUMt zhP&T3tkBSsp1)^OiZG)J6R*(I_|mnP-g_MTz^VE0jz+S0u~v1^#bt>ux2_wiYjN$0 zGC1#$7yFOx`|azDhx>}}DhV`5GcaUNVqD_2Xw~tZ;=g}?d*6R=KdYK_Wr%Mm~Bn9vH>@DOl6S#f?lcX8x!4>Wd1Qc3@x@_fA1}e0z;F4 z@&v}7-sQKhyx2X@qiC1HKgQrCiPISRmI<#sBer#6;LV`4NpZTO?3!y9E}uF-#W9?7 zMe@fLt*$>NMZ7MXAlcKhZ`~_~;6q2^r>zQ6VBq+|z_@NB;~q6BgVh&BfsQL<_H48&8WygjMCNu+`iIiK($nJUJ(z?CRU8PwMo{K!(m zET@_sP7yt9%jZKNd33R%Jfo(!u)q}_Bv7kqC(=eJVMKX0IbDep-;9XdNfPgzv|gB% zyQmi4tn-YCeDNKEW>-slDb7sV*c3xBh2s5Hvj#ct@uz@cj+C21s~BFjhf=1LZy=pc z4y27$`Z^zCu3}yuleRLcZocBSGQhD2EdM;u)xMZ%?zG2BmUt(UCiVYOiw^9#=@$Lk zM4A455+ho>8s_Q}H@aGdZt$h%?P}#I|C)Sh(mM#yzg50TrH?bUDQ87*W_tMp1k-pe literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_97.source.pgm b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u12_97.source.pgm new file mode 100644 index 0000000000000000000000000000000000000000..7d66d8bd2b8a8c67e3017f487ceb33e8901b60fc GIT binary patch literal 508 zcmaix$t#6%7=<4xb#Ea_SSWL31pG*)f_~Py!j=Hisb-K(Zm=VW z3~Cu6g(`;G;Q=Qs6wuBBr`QlkCiRT7&nxcqvc)~lL{X%?Z60wUh7zoDKH*9nWejnS zfiMc`)9VW!q$y#FW6d^^M+Y04;)*ZXw6Ur@ckwka%Ows(Q>kf3(uOL*s=RSI4Md7> zQbhK|ig#LCClZu9rxq@(@T|NX@$IW`80FGlh%=QY*2LgXu6TddtVWJIyrdoAK)LZW zis4mzC}d3e&Lq>yuCx(KU*Vl#G0e#0tF4Txo13_M>ETcWng4vx)IOgHZnei!=6S(8GI&^dwpNS3F!9~kIDoVh z!U7PW<5waZM+>hAbcD!+O+%DaP196WWnpxH_}3yUqk$1A6j_&CPPf9yK_GZrsHI*X0IW&Mml5>ZDb!%7_$1P>m#dki{gHP|iO0+ReL|?2vZJ$Q$`4 zrPzfO`jNwTlyNsZd56<3Dz(xkHzg;FQh|d-;UT6#Zst*5<2V;srg3SOD>5eYQjP@9 z;R$B2nmc)lS-vNAi?(UMc1l*JB#L@;VGsqBuz}rt#2JR#s69HQvs!AS@rdTkXKmb;t-)XN6Mj|D6!Zq`+0J49=Cdg pVlWx3iFEKem$^@SwV>-PWzX!N?e#bO2VWT+51s~pgT~19$bTz$dq4mH literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_53.source.pgm b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_53.source.pgm new file mode 100644 index 0000000000000000000000000000000000000000..978f2197c48fc42881661c18872d2943bd0286f7 GIT binary patch literal 509 zcmWO0&r8#B7zgmr^Sqzu^VtR#Uxt_mA!|nBrD02?tSs!uLU4p9iSn?6%~cjcBRWJs zphM->vg}ZdkSG$7N(+lv7uhKm2raYlk{X3rM3>U`54?DFL;~UF_2I_Awn(Hg5-`Tp znW#BwvgW1vOrXuwPN(TMy`nj`+`t{&&v(s1(`SZE-Yk+w&D2d-X^cutT+cC1agGbT zjB1lHugwAlX(#nimfp}euH@aE;6cvwcT`~ylDK1L$WKiar<;_gC9dH^W#J)D0|?_N zu3-|3Qbi%^q$@N*3tY`nKF3dZ7HhBzr;x*Yfozqy3{sMEG|hf);XWSb5-QM&emugg zc+w^*8Id`yq;dMeA@1S~7huqY6Bx!P2})Eh$O~E0O?pJLw93tVhDUfFwdlevyc56d zmGd$o%i5rEy{{!3J7)j57I)DVd~4B*=U9>bG9V@0pb34Z|Li_{!+vnJ?znsE{`%T{ G*Z%|0wSCzD literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_97.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_97.j2c new file mode 100644 index 0000000000000000000000000000000000000000..cc1101758365137283f13a4cbcc68d14f00dca01 GIT binary patch literal 536 zcmezG|38pH(}4j9gn^hB#DD@+J|jOP^F1#fpE(d^(pF;FZ@qo5NBZYFG$Vv3h+<}OD$3`&@(dEGtm1#fq@HT0pn5z#{ZKK zFmUf=jY>^16&-E!plt87b=Ngo)~TVfO|lo)q0$bHF; zUYb8YV}Z>4wi|016yok5;%{(Zwq`K#64@_zQO2NQ-mfwFUeMmrYHp2r3 z-WRv-Ffi_A&IB^=XEUtO(2}0NXHtqVqY4wR(9`(RwU^#|9Q(kj`S6ZLvUsspb(w@?TRuu?~oV!kL~;I>x_r{itj23G)OZrWKUvT;E`B{BPFw3$wP~mwM{Y^K9n+9lQ4ay|dpzPn z6eV=B#2F5RkV_+z9AL+Qg&iL7B#tuW?Mm%M0w#v!e8QU~%#3h>Gm(_iuh$oR$x^}$ zCz>5b5nXJ_=S=|lbg-^GgZP@5=L$FCsn#@`wBbq^Qr@JT&cupuT12iSig#984^ou7 zpcY=N@r;oI@g0I@S4w*+&P-a^5JM1!;{8#xIyvs~lXj3JsxMN8A?rI2J+fKVRFQ_j9!`W{NxQv4TZjiJ(FKzty4*D=xZ4 zuQpMpfA2(#)~beiy2OpHX3`Bl)Vx*Ocb9)vzBEld!(aba_#%}aPSvKIWj?i!Zhip> CRd_Q1 literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_97.source.pgm b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u16_97.source.pgm new file mode 100644 index 0000000000000000000000000000000000000000..ff9298a4dea1a5e4eb60a4d13c640e30f90f7fc7 GIT binary patch literal 509 zcmaix$t#9&6onrtjgCz?kwKNTP*ZX~UJi!aD(Cn32azTNzb1XL0w?&7lY~{`sD%eI66sYL6w%^M*mK z`v0g!GkchHi!MC1&3Qfu6YZWFX6h2xx>~Vr@TulaYGo(?vV4ispW>l^OMH_+CnsuC K#3EnXN5>zndwR_P literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u8_53.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u8_53.j2c new file mode 100644 index 0000000000000000000000000000000000000000..b24b5b630e45dc01266b148bf44461478cc9ede0 GIT binary patch literal 665 zcmX|;Ur19?9LK+Bf81~iHzIwI?B+`uBpR9ovO9=y1>rR!R#@6Xk7mkW4qfuoz3dU?{I$S_weI$K0n$-n^EZi zi|yu9QxgJoWv-EO&R@2NvLq%B({255)_Nf3%$)AxM zWWPlHCF$jXUXDKvSx}TXy-oufdf(JXRxLJ#(vc~4%rQf3k&%~o)bw%q~RA`g%3nV%MDW0^g^><;HETz&Z zTg}}roXhIEU)PP9^TW_-kA{%TV9EACmvCp@*1g4$)7rnq@Y21qH$9fq!Wf*z*+i>_ z5Bch_{2AW4E=(kfvD-M;&3yEkCovw22=7)?!DwVqU8P>|tzWEHfyIaT8QrNY*z1%j z`*A^_jq0F6d|JkB;3^(~`m;2U)TmEgol$XHzfd;K)v$JMNk)d!w}oOl*#<`rg)@CJ zEj^C-k~kbjI73J$%Av#kUxuq-ej)hw$ZYA-Kt={Pk^zwqL5jeV`(pdxhWa#V91V@5 hNDs&$uGa^VqW(>d2B=iP+nx}lCYcchGkGOn{{Y#+3b_CP literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u8_53.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u8_53.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..d83897c008740a749b645f14b69dd3e3f26cc007 GIT binary patch literal 247 zcmZQj^~#^J^B%K?Z_)HU581Q>N@sZ%&OG>1&@!=e<4s1*z{-UuKT1y8`$E_*bISf# zq7K>94!#k0%AIkW(>!(Z!S}M>mCLU(8^-tUekraJ+P3|Lq*wKtyL|Qq3okL7q)y9S zc$L$(aOq86$I_K|1zjuGJ`~;dRwa7UNe1)0rFVqA8h5-`i=Fb{vUu%t#ptQ$Ih|{_ zf7DK$e^c0__0WHt%59$wa#uZ9Nt}0EB6QNki-Lh@_?B$K;%AyU>pz)R?EdH2a-7R| tf^z2iuhxynd4p%(SIyq|-M0B8f6mr_?h|e*WpDZG)_+4WYx5u1J^+Rtcr*Y2 literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u8_53.source.pgm b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/gray_u8_53.source.pgm new file mode 100644 index 0000000000000000000000000000000000000000..50fb30db97c0f9ca5300e9662eda3e12141ce786 GIT binary patch literal 260 zcmWGA(+lmF>CW5 G*FFHSE_z`A literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s12_53.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s12_53.j2c new file mode 100644 index 0000000000000000000000000000000000000000..26175b3f7cea1963fce8877d45462c4acaa7f97a GIT binary patch literal 868 zcmZ`%eMl2w9Dd$Yx9LoVOZ=m+;gX(_Xv|t+x zy0r9MC4u--)4MWJAn6Jru|uqc3Jet#;(nx4M8FWj60rtpI=jQ+u-omT4cQuOjznPLv%Ab3a0MtHo{TR-rHg#J4Xg}6;>9z269<(Z(Ye&JC~@nnN=?5gICEM?oc5NlApJ;Rh-x;+a`v6dcxrx7Djpf*eeSQP=l%2g=l+=MBx4~F1UXohETBm51ScmRgt`2psl1RAxu2eX7|`QLXB( zou?kC2vwn?V3ul8^B`B#*UXcrezG`DaKdAXm1w2k8>k{fxYoKiVv_xq~KxWvL`_!P5Xm`N!! zjXyNKZ^B7rVJ*H*Nq8H}X`SijPpZb(DU7ehO(W(&&F&X-x1`FXiyn{8SClB2LKW32!J55R!M$$zp$4m<6 zi^gp(hf^yR^IT%8=VD6Y5Z9a3DH5XNoFh~C79Wxvj*$*I#4991ckmv`&=GP$w(Es3 z0x#h=&PQOHcyZD;^!v zrIM`^takBhpXJs;9k8xJ9qy)RwDB^%gEMFpe}f;J_HSbgRdKwzxo%!d3wXTL(ii5% z9?=CpDvA80aoNPn2Vr!VbrGK$9^kgf{wOi*| z^Ia3nRrgY^=_rwFO=p|AA3LdqXBeZQQeti(L(Wl+>0t!-nW@V-Sz`2l-eO*?o7?2L zj+5cWB~QFM$qGoNNg0qX-D=I$6RdF8XFAu)be-3=R=I1W)o*pW*5XQ9!gcta`7eUK zbdwj*E9R1SQweX74*G+eWIRX6V4cTrnf6a|wNz;42hy$QOOo0BEBVB{*i|{NPg@@S jlyP}aXIYC~Kk08x3hDdS4Ogo5vTM?yFRd-E^@IKgmWCZv literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s12_53_single.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s12_53_single.j2c new file mode 100644 index 0000000000000000000000000000000000000000..68f7110d809ea1cd71cb3f5452553a6d956cf42e GIT binary patch literal 298 zcmezG|38pH-+=)Lgn^hB#DD@+K65uCBLw{qVBlb20xDwrAH=`|WHB-@F)}(Z{*Phc zQphMMD9Fgj_@BZc&cNtjkecTe;GqzfTBKm0XJoEtp!a_Q0~gr*9Sn^BCm&#X>G0qF z_Vs^thm~OXo=Tz%(tD$ k?D5j875KnB2d08+M3(0It*k$ z%#&fqef)ONz7LC{(H+12&XMk@$^?R@44wl>^{X82fw>dQOBgtU4;b2R?PTWZ@xy<* zQI1$GLdEVNhr9CF`dwruw@B*Ab@7b7pX^nS!u^}95;41+#br1>;>>0<|fJ0bi@ zL=|4_E(ID}(O|@5;5LA(kEv{B2bIoyFrB~@O+kH}V!*rsv&hwyRp-UPfl+XyNSQ>C zuBe_34lZD#jDI0Yk3r@^WDXQ-%@U+J-gjG-PDKRdkf*}>fmgW!Xst`b+9qypcHPr_MXs~V!t7EbOrvEdA+e%3VCrRt2w6)`;)q z&GC$Qur?QJcelFG!P!try&CWyv$}`wFp*ow=}9aF~Qv zX&IOd9*QaLoY?XE%wdYhRD1%J>p=+}H)peixZ|IaHSkZ|@ z)@lE5L>ZJ9l8|kd)CCyW$K6ui{}xY!^O#j8O3x_l?HZ}c+YwbwTR?jYLbBn~O;XXL zrB&DW^!OhUupP{FZA-2fUKJkp*uO*^Iwii~JtGJlkIh`3&VNug274Qt4rSj8pn={r zpfU-nT|=Q4cbKUU7psw zZxr;b>byf4#)md6Nm7Bz&-J7~p+xc(BO}z0T5OgY!lM?aldo(^m3es)Bhx+n=gP*c z_30yXqqdKw;}umK*EPTGZKFq>vA^nD2N-i_-Xo+Ezxg5a_S zfo?>w2Ei+?ud({_iGq##_SZ~XmggJhCe|FRDt5{!+-G|yW}DtWU4efREV5|NBght( z0i+Wi<-~~^#r&tUQM2x^G0?uvr*W1=hl&ra9!?G-8Id*#|Z&o*2cI{kU{AxMoQlbVRL2YjY zTPvBnw)^xkEVMVRufGdSuQKt`oi6n(`vB6=8|j6hq-FVkw8|egj~Ni?gG6!`_?{6S k$cQukID<=Ii{~redCS4d2JmjDP;Il%%Wdcfox90GBdJ9i2~*f&73Xg!m&g}q-NR*1)8f52@)8iA~Spm zl{pL%VilESWSWLAi4Zcg&5;#zR+OkI&6E6x4|d>P+^Bs03~6;WAnKJWWgWyO*g9uI1hf zlLPMZ^E!{-jqsbIuRI+l-Odi3+3euVAqQuALT84<_JC(7IFlPXQ{mvu*Z7#Ni}cy- z5t>i;7wH_0v%AK~n@s~!aYSWVIqKMBvjyxJQ*TA%n`eE!ZS;5k4ya3(<$bqtS}(sh%|{Qwy@ z`)U2{t6t}Eiq%0cRJ#838o%i=c=IK0_mFXkM=3_1dyZ1@s>p>N4x^o(rJqrer)Wj# zLmcS#TF3L{LAs>s3Q9`AZTB90?+?-jQWy=-EK=)prAx{1g!g-~2I>-wCTlFJa^Mh# zO99qN2UvqbSjh^oRxnZhNqRk4r#wMt;GU)7XsLqVb?WeI?1YneHo0H+4zI%BAMpGc zMd}=!9*4uXXjq_|&{XEBFl+Wpx|^=`%x?m|Z}4)BfJ?Pg)sH@JFvCODM85e-B-2K+ z8^`*-m$3edmL=%!V4Y8w4%p3RZA1TTv^B9#!{;WiB!fP<V zL~|Ky1l`MUcmQ)$>J&6Llgm`<7U$~`P`_}2AH|uyXqmtq?1NJR==FG%L>3=2Z}%{h zJK=UO-Kuc%K5kUCpF`Ve*TD65y1v8YjUtClXb)#%wlHCx{>V?DsD~6X6^HMYaQ*-P zbNTQp@aHLBgH=HabHLh-|99vgpCq$nvaMz2GIZ95y^SnygmbQDkynG? zLd#_z_Xp$?1=~C(YdqO+L3_Ut(hRiqF%ygRFpk#X)nI1$5Vxa*jJw?C1~4P>Zv}TW zS-VkM>@a4am5Cll4yE{-#(ixhpJ6f+pP}jGww=4016DhE#&HMw=$Wo*+7rUQK7>6} z0rr{@_F12zTfFY#+dO4!kq-Ma*TN@Ey(In^cX%6_7xA`yPxtZ6X_QuKnKF29cESA? l35|1^>m*){x6$&OJKW+2bthe3Ru=E_L4QiZH{;#~{|mKW@I?Rs literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s16_53.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s16_53.j2c new file mode 100644 index 0000000000000000000000000000000000000000..0e1bf84978483dfe409e53713c02fe4e87a8149c GIT binary patch literal 942 zcmaJG)}Pu^ zA~3^06+}^xL=nO)iTSG!3^NL|=to2{&FSv$x%YPN-R?4?4(HtSzR&ZVm-pk!)p9Mi zu%OFA&uYbpEfc+yH3tco878{r8W^CXgi+py6lyWhF$~Md2Vk^%yk4)z!t?f3kbUBmZH~>>pdf z(k#_v8EQO@CYz6ba)=CyrvzQk2o0W(C&F{VF20Ru{!AoS7$e(aeuf$PS2Q(4pPt9r z5j=Q}hI;s=qB{CmA%hCp8zn!~hH>t1Hult@g`P`IpUXzW0NxiCf#8c)MXEWd!t27&DOD(Pzd3zD1#hSzZTT%QgC)R# zJJH7)y3OC+nEJ|A6k}dgzU1Hqmok{ER8_91dIJ?3$4%|{At-e6CW$>9Q+-0#-D^nd dBGA>i6S=^ya?h%ENNfWVa#Yp&y5C}X`WKnRP1XPa literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s16_53.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s16_53.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..6b8ad7033a212610d2ca5494472353579a2ce1ab GIT binary patch literal 1482 zcmWlYeN2^Q7>6}9hh}ynG*rfpf=aHEk_b)14^&XZGqdK%?97oPL_CruGxEK^p65L_ zOHe=^(~d=IO6-&fNX(GPY&nO{B_iUf`9Z08WQ^KUd)_~HS_};k0UMwTra(D7hFNd|CgVovh9y`DL(qpm zKq6LP2$o_KK8{6r4X5I697@Zu8K0v~_y=awUK~arP=?iFrGn4uvAnRya)BQl4uZoS z;P7B{I1wH8pu@%JZ~z@XgARw$;TUpw4mq4h4wsU{jpT5M9FF8ND;jn~vy}#|aLdYv zyATBx5QDR!1?FQO+=fEj1!Hh8o`kvBhJQf;-ot3K|2Rs+R$M^&co%(CVdhs+3(lw0 zbQ=rlK8>Nh6w7m|jTUhM-J>GjP2+eUcUpNc0Fh7$377(HupD!t4*++98*8xzvat($ zp#(=zEFQsBT8tgI)>sUqayp2~)Jhj|4TUMf>>tNTbds`oHQk{tyxq({%uSTRzjF_L z#6x8aSM$pf56$oAo*_|uihYvF|MDRLo+jPWz-wi+>g9csr6$QmVYN!0 zP)%x=%v1N(BWT1TO2s~`p(1>qZc>fu$;~%0KsoHDPLq;D(R_}}DVrnZoN=2ck=)89 zGLIi{y_9f@gr(l3PE}z(BDrd+T$lYSSK?HM+Ak|rlN4`~F znWX+!`(%xpqC4dam8+jsUFv{dp~mWNU8dgB2o&yqqNM z6fnEHIK)Bmu-mwlvR9H+I|odPOF|M<9&xL%DiyCv((MvZKJ8K=71SQ()?r<$ygJEl zR{`y_T{@(L_GPT29UM#l&;@>%X7XtHl0Gr*-=G$*l0mThbJ;jdj@6h@70{=vF z)xDf&I!cyW)7cj3r%rB>nZ{_C0&@dd>NMAw9!5%^nYvt3Rh-@ff;Kf6oZ%-T(jq literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s16_53_single.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s16_53_single.j2c new file mode 100644 index 0000000000000000000000000000000000000000..c8f23123f2dee34dcc795f1f60093e1332400a47 GIT binary patch literal 332 zcmezG|38pH-+=)Lgn^hB#DD@+K65`KBLw{qVBlb20xIJAAH=`|WHB-@F)}(Z{*Phc zQs|g4VM0eo$Nv-taRx^Jg48^(01t(*)FK4~JtK2H1HJzf7`VXZ-(q0=KluQI-jVVKVHA@9UsxSIi_ zN%grM0-B)_Q?tjqyjnkiZzx)0E_e;@lQI*_#dFvGJ=b>rx{j+UPAk6T`@zrGQ8bA3s{fUMoOeeaw0ThP4S-En;YR%mCR)ve#W zn#&*KzHlr<;+Yo1TPkrgZDC}`4dPaGv7@Ful5gHHw=Y2fI7cd4r)L+AS@opH_jAbm z2v#Dd!n>f;)PZ(4-|V9Yf2^kk%Y!{r0m@Ya6>YN^a zc;ukPx)CBTNj_RiT`~?{NgIyVK#DFh%@)%+oP|)+JvPc=6_}EI7 z2hG8tOhv9e&GG_DAM4kH0kpE(PAXai^9ZgLj_)%f}$Kte+O+pvkuVuPGXGQZRi>2&4^y zyDy9?RvpLkCfh0H1!SclLK`oF@uMTkthnGa42T+XT{j8YMN8iIOV-M zXtYd5Wxm0zmO&FxJrq*{|4eli`RcZRrsBQ+m?n1o(HrFZ{8jf$b+I^IEQ2rbRR_6- z%65p<8B)h0T2!DSEe#q>orJqz1}c)6B7JQe+pedoS%r$)94ozR2O2xP_0~aohR7chqX*hEgIW*qz3tM|% z4!7w}m^1a&lh_q;g?mEMIo9c241Hz`*7@8?>Og_nTFG#Kcn1e=7NcWe5|X%^B@sWc z_C_akB{jn4s)I2zY^`Dyg;_zOpS{x-8GkkUcP@RDhuutZ&coQz6mS!%L(S^XhMm+J%l6o?hNf$HL@I1{4)JVi3D_aeg*~dYhDs?dg z9fAI90UXk{#b358KF%b|Oc$7?5;BqlK?aJ!@OBw!5i&@*ouyftUuA%rO`% z99Th9oTtU*&U@MfRq3F-G##beU~zwUp2>-nxrruAOqXF_;j}UAd!Av>u_^2rpMo)O z3v6r(^G4@BLX;k?y6x0q*1>1ZYiV)>kyVRYg^(g7K*Ls+i?dC|$tge{RP@3-mEDNL zot;}+u}MK?5yC1A{{x;Mb*smH^^NWs{v!%oOT7XjE{D(rvySbKnx}y?<50$GE)gWZ zD-|K?9F}#*MpL{;b+w&w0prjvcqUlrreLSgR0-Py&Ka-sCX3D7=Xm60&U=yRQuc>jv)&d=m6(0N z`Pl2ck76@r9FMr?yf2zAHTRI~iMIuxC1%S-zbx+*U%K7(^3!5Q;f4B+=VSB#cF)z` zc{=j>&vsGutkyWVtegs9y{B73-b$^8H}@L z8l66s{O9)+U7e$GvETap)DP^B`1qkyNpV+j(Ce0FnXOxVpFOP;7hCV~cA}}`(ag}F zb31h}?oDLBbU&UMtiSenSXa;oKB zv!_#XWlk>%X1d*FDRi*F^T+ZA{d?QfMBmNzQN7X}!@X;=)5lYl8jKs_EuK%ympi*G zl;v)ZwaDS3ddtr{i*-M*P1k+CFkSKX!f4UkQ~gEHclvW4?sWcpxYqpd_CoWIs|&TC zu1=MIIzLkW=Hy7>vy*+db_Dy>;0B= zosUa0^&T$EmV3M+TH)!8Sf#78;)Sly5q!8YQ1Ro$M8k^<3rS)}nF`FU2!618n)1J! zX^t{0TjJjwU!>3Vw8&Ru>xAqdm)BW~eXNT#IXI`B<^C>rrN3RN4rf<13%owIxHJFd z`K|h*ze}SXcQ0&be16tlSGc!0^7794lK-E^yJ|1(E_{7)yP^2s@>u7+i#wTLp7+!j z?JtdfbN8I7ns{A)!tos|1sFd?dRZ@;-17R)Srb*U+PwH<+gI>2e2nn4S~#is)$KFJ QDxx*HaYwf;=llNw0IcbCv;Y7A literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s8_53_single.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_s8_53_single.j2c new file mode 100644 index 0000000000000000000000000000000000000000..2663a03a17ec71d4d85bfc0e5b699c7e1ea607d5 GIT binary patch literal 1154 zcmY+Ce@t6d6vxk{3<$%lZUa;tZG+j5gu%v~j16eFC9W|hRALCu3K+y+)0Ke^`s=+8 zWAIlZ3A(bGQT&lGoo*sJhd^KHHiOBMN8&;^hddlhIxO=QUf}ideq3*H|9Eq5a&FE& z-*Z0qTwH~pfnp1gwSnvhlFt40|FfC< z4fQ}MX9~Sx$r+zW4|LqEk+$yo@yoHohW&r2Fc|tmKk}gNG}yiwZTj|B=52fF^)3nY zcG5CiD?%tW%)RFN| z-iN!9Q10Txr@*l0tnu8I>YYtl2lvG@xp1k^SWpJxz1x19oSPc*tFV5HqVwnrej!5F zl0u>MqfG~D$0kR!=VPXagu?54bv_5c?`5Foyffp{pKZ3oUkbnrXF5Xwhaq_NI2I#%;Zh{bSaHLei99T*I*8CxK&*6;z~n+jbNwhas-^^4HWH0SB2oOj2&Ti2DwX1^CS6a5-e^O0s-}4qOi7qh z5dYPSCxJFI42RpW8+%on#T_*tY3)qH~V0d3(#9XAdO;3 zYPP(v_IMI__l-3EiC#49t))I@1wk5R7#_Qw?OvwdaUmL$6rAtro#sVB#zY<02RN3L zrIE}>4pg{T-o4}RNNu%1N%tx)rlEiyT;(+for=e;xG*iqo}htY2zyZSxUqW~J7=SU zFn3SDPFbTFsBlRKe>G>tQ*~Aolno#ndn8p6q9S&%Jlbq0yc%Q1G{>*@&??F@$;6bZ zVVYsgWB?3tI8uD6^GbJE>`g%WxDWOZAo3o}n#75PwQWh(x9_u_sSOqfc#aHL(OBFV zQ(a~v!17|EB6y)c%0}3zD65sp>LS6`3~&A{+v}W(Prtqd=mwyLL8Rd$vJ3@^Qt*AJ z6U-Fll0E9m(xOZz=?RYF3X-=VNZ1j<219`1DoLxVis==~ia3qv`2sfd<9Gi?um*akbzndZ5 R*+ROa&2WN@whi#je*q`z?P4CKNffO|%@k6YE3Jet#;(laOM8FWj60sf`TD!yHu-omT4S68dj>hAr zno6yu@r1TeZz$Fm>ct*p$nU%fBK8KDXJYgF3Ge7w)hs%}Jq2Y7uFVK@<69ohCmH`< z?}S@0MGWX9r>KP(ky{AU_25}`3-a%3pkkdCEV^E|EYqr%>?>-g4?O=*qmkrfgo20h zWWL_AD+8Vw={C|(zLC<7eoKtFhNWWKw9qgal?!4<>Km9;7g9N#8^+_eu+||U6xGrw zR0>n6D=tl$qd31@OFX7xxW8H$K7=+OZGNF{#Y$MtDs917Vv3hOx!>)Jg|z}~5VzBe ze)5V_e&G)H!$wX@WA(TDe%xDgW6yd@v{MHuLk>%{Gj3k7(}tdK#-b6tQWuXMaOH7$ z$+u$sx5Ue41ar8|_SLX(GQ}LiLGX^Wjc|MFwtnDC2>o}ON^pzR-+wZ_utP`^0qV54 zqK-KmMng@0Pm-5=NTEH<#(Nly92+UkCA;Y7n8D^Lv+imFf4?2MtiYYVc(fi=W@XRi z!`g>QLlJm7M-rybWGLiPbnqyEGtCv;U4AjW4FYRL OicU7ItNrH{KmGxKxF`ky literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u12_53.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u12_53.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..a074d69e97f8b5119db7a841f99fc654b5fd516e GIT binary patch literal 1482 zcmWlYeN2^Q7>D2c>v`T6GdmF)Dq}}MC09vFgr?yKDk$QaS#xB@963V7BUv&dqcm5| z5)@F!v}2K)5<4XV5;G(+Th5_#iHLY=eo!hN8Kbt;p7+n|pZj-T&+mRLtH8Qw#ardp z11rO7u;QT5x@4up4hvv4G+E-85I`c6`Y7WQD_mi;1jeC|G+HTjhfz~bgR`$0l(F2<-jh>10RFSL2$VfTn2PG z0bTZ?%f;w&09`(XE{DN|Fv{#dic+x+=TJW0Mn6@W`PI~lv*{$=#6r4D zBWO3p@Jwo_d0aqusEBvcDBi#!BFup9fJ1cy)z z9>NrwkDa*ESZJ!C{g^~;bRJhwn8@rO%ZYTHGI=@Oq7A&&%s=~uI40}%%}Mc$>sZ8C6)ZN{2~S5!Fmipnn^hi#TY{l z)}oK5VF)*qAEQhQHJEO;?!*%E@=yxW5}rX<>0@3;(cHoZja8Vtsg&cz!}YvW(zutm zN-@VstJJbj1|-CrB|$|=lPpt5A=m7F3#yGv9G=4uC>tY8O1WwLj_G|pjc1GM=rvBJ zYgECjO*g-B4ZXx9m*`WTB9VN8{gT1|@&PI3Nzx;Yyi$g%KHejlYMh*xO=_7uq?*+Z znWgTk2hfB?l!Eplb5e!fU?<3T_z=)qWBC~P!>CK#<%+wbM_-8Jr>Bi-#xr0>cr$&=9i3j+Y6!0@LDBp0QOjg(Vu=rJ+^vbs?N5-kY z)gD=)Cg?8tLgnhmRkzxwm#C4tN0+Nt^k};bb10EJjFF!`=FbG#XAHw!Mmd})9TYIT zdpN{F@v+yqlyQzEstyjA6pw@?sC?p8VO1tMDp7YxK>4*tg;Y@clvjs!naa_Lc83aR zzwOZ>9kefEJ#FI{`iIW(>ok>z%a`P%-FWPrN3Ri+vvtxC-eWU7-~YkC+e{buT7Nl~$SmuxgI)+6oekd9YFj7z@q z>0~>os!YnD>eg-cG(FaKoDX%Lo#mX;b#{d_%pS12oK>`p7fL;SZT`!B>fks4L0@5yb|qvxn(v-{`jee+@$)G2+;_UcEC%bPme dUf_JMzcMNR>09QuCqYh6_-)=s6=TB}}7P^H#2wMd;%sMe_t6|2eslvkaE ziE6#-gvF`?2m)m zuy!k#+7@CJ46$kpF{ltjT!_IJV#p6MRD~GMh8X%njFBP6*&)WP5MyzOu|CAu6=HI% z5*49#S&b@5HCwk-w(3>ksvIKKEY$?_RF=98xoRhjggxpM%z+m57v#WQh%o<;h9qdl z`H&607=Uu4R|!ox56{4D%*A^+68BIP&cPO1h&gzd@^BZ8roC3XQ5{r{$tFf6s1{hJ z{Hh;{R4KTj)?7`44s{0#aR@}gVN8TY(1w|iZ}KUF0~n9Za0ypqAENnxEPC)1d2to? zPyuc?`iHQAQs_7ArgxZWBvtbi6|EZKWwl)O!dCSWjD-`b&Sc-MZb1=_f=Kul7eOlg zf$u^A4#yfeg2{LdI`C~8hL7R_O2QZMG6irQIkX?EXgXcQ3zW@7NR=EM~Y;wGwL9}iF$Z;?2T;Cfl9j)NbEs~)H{UZUY5 zegNqhX0DVN)w`zmjWiK0tfkj50so{jTuV{(D^}A>jChqk;pr4kr#V2W^e-PYS0{5P zo#0FvL3ensczFU}lFhtQ9+3v#DRbqXdI=O@jh9h!?{gL zwO7LBoZboxO->hJ3#QUAIE`CrF+9W{O=2&QqMtCIW*INXseqHP-~3)klWBlYQVu`I zgY-4$%2c|^M%|+Dm$QEy2}p26+A{eFbe-M8+)B*@Cf{Z-s2UfvnH;jI40A} z8eSmdsFpt#Ki%VVa**TYG2>;UEYr_$gOuw4Kae(kRA%cjdQdj&RrYi_qxai|I!xcN zf6!h#)pzg4n^-O!b_0-0{n2n8-a7~;ouWBzJm11cZbmkYH+v7(nEZBb1C44-7(77$p(0IvOKZtp$n)0YT9kn0ORH zAr)qXkpxU3${7JI0Zc$Cg4Ykl3RFl?Ia3RC;uVfGss3Z%?#{fO+4p_#eRkKtHf#oe zCK!!qTs9`>vwgY8;1T50G;qKY%#4SY7^1-1=pkT+AWSfD!QM6PuM=$Js7vxl^5CO} zeLx0;eR%)D2+juDxBC-mbic2^@uT}0asX#agDnNb@QVr*lf|2T>#z#zys%qnc(;Rj zNFizwja!`&%cgR^Z;>#>I zHL#NozLK8BWY`uEcV2jRWIR5IB$`AH2)Qn)u80DhbqPCvJtY1?LKUCssRSBN-D)Rb zVpakpf~0irqS6Job!lwL6ttwM2c238E``SGO}Yf|WlFDJ)1>1X7Ao{HF;X^?{SPEs zGRXpnD}Zt{J+7gEb+bd`e+`HD78wWwAgEmlw2t}vJ10tmgM|2d;z=7#NPdAyQ1YLk zz>Swumy>dd;)&NB*4({>CYzNYNsCVB2fjcd*6Ts3!Xxh{f zNQVGh4K*+N9heU(&m96l8$kmD*_A4!85664{&zJpPj4r2lV~3iS!`zGFtj_l^2x-8 z-zTlS248sYCG)-0qPj}=g!MfS@bJbdcF-R)5AK??c>m{feZAXw)OC3{tz?w?{<5`PzGLnugPJRn#O$bt9@<8zM;K~^L zioh^#p)4@8Y`ufT88Z@#84Fb^RrnQcS!&5)Msu8`Qd)2>lC+#TW4G&ReTHupK?t3M zba@R@eX2L>5hYHj9v!8AG~;oN5FWJqAAadYtSu^v9o6*;pK0#oF3%dB9`kzGpslH2 zwXE$;Unf2Bq)%RC87z|@_<1nxvGGT@0(8ld@=chYk!k$wX9O2b2Rz`f~s5vaMcq$?Git?%w_f&Lx-jESBuz8cm!Csm4z6$sb7ZEbX&#u<5T#~WYJ`IzRKG7(FRtJLAl?o@eRG4fags z0o{(8aU7d;ax_11RNb+1D3#Gj4)c^M+SOFd9n`3|J74(aIJ+E^d%Trp=5fEizb;ZQ zQ`ln9b85?1cNa69ZZ=IOxWHYZUWS7MR(8Nz9k8kbR$IVIGK&fxW4HZ#idU+Z z1o^$n46gH~P}F8~U+PSvzmMzrFFLtUPKV?$O${e`Q%ybCE3*tPvDL1_o!@eO?`0P2 z<&vn5<@UK=e~UBLq*{o-KHnG8TfWB-XTTZpkMa1*4*-g}^qxzR&E_=Oa%3+_V_c`_7>C70l z>^8$xIhFI08rs~&T<+o`pE$=O<+j~!38wO?8b$(d_l z0TUQxmaF94sxw=i!ES0<>Mq@GKE<+a*STG0-(T&^15%CJ@|}q$zSYs8dR5>Ik9Rxb z%y_ODjS_RM{6&EWMXNiW*!)3NLaHVQA5i}5*0ykYR#b)bH>U;pjkGfvaD83Lqw>? zG%d?ibe0)dtdxpyI+x8QA}Tf14jx{dv$mqHp8vt0o;~mLy+8NoaNlp<$k>#$K`Fyy zQ*N{gYx3Pqu$>m*QcuMTOq*?JR1-wH~UJv)3;ocPPQNHgK zuYe||`X0rz;l3f<8^b-~`v&^?-pKvFM|eEkSBHCTxVQSgnyzZ=BbPeTspl;7I8QIR z_G7G_AF%EZHJ8&@eqS+;Z~0A6E{2)`p(Zob6o;CsP}2}<+CxoTsF@gQ7Tae4>u47B zik&NUc9e+In9C)W>*dYt5FuA5FHk}^;@GO%$to*UMU3u##0YsFVU8L)%RinJdFAj3 z=UB!{@juq%N&KW^&**SpHZk1X>UfYDYI}`Wt*J4~Jij<=*Zyk##2#7v!gFHurkZVx zH`f-PrU%{ana+J2HI*V+Z`8@r{K#Q-N6Vo~MyVX+F;#S^se(KCUcEi}%rD2=<%rzl ztgJSVd!2$;MC#-8Y?9|WwUwy*Iy0PMR&6G_L`HS$$Wdd942J})nE`8Mz}g$IE(ffk z0c$q9?cZCxDzzlY?SEn``yAoDACCiCSkl&Frc@hjGTUP%UTFo$uLfmweeB^2}BL zA+=?A)~h&!f6SmnE)maS_aS1|b3NNw?-{Qz>zOI1Cg&o@^Pt&H)ThJxmt`*dyl2bd z8&U6Z>c7*Ov1-|EhUs#u5j~m9?&1ROa^AK$$EkALX14@W+3p4<^1IsRxVO!`XV;tb zd!sq@S9=)`G0+JsaKe62QL!nsn$lF2MXuuS$DEr|v-#5uA7zaGzt8hJaoWwOa<8N1 zHlGO$Hp@5U+^#d5ox$tXveaF=!+a`a+o5y2%zmKSSGXUeX;52@x=bbq%xq22Ukp5Bbo zp-Q`4(aCzJ{eSG219oG;?h&x(J4aL0`Jvx-*!z@v?p9l=-4@wlzZwRcd9;|b?7qu6 vt#D$_+b`1|Nxa32YCP&(?~&yVYI(w0%jQ!)P{H5s!3prZSc z5t!kh3Zf`bq6lG@#QfC*Xn3KZ7r}Dme>pIvIj;r=6x{ofDbmS+;dW#kpjY`bWvgq;2%Lj^W^nJ6o(@X z()LwomDDMEr3c-i-9O`~g+)dI^dndOy{0x~-!j9%x*S;MUbfLlf3o|@f1euf$PS2Q(4zn;hW zF+6;Y#(MaqqB{CmA%hCp7bQQ`hH>FCK6?Fwh{pO-cN+hXNP zLwF$+^_8}JpyyK4=d;l`i1&p>Ao!wHk!lXA@P;sSN)?LSXHH*K!J8^bTYekNPzf;T zO7yaZ9`kn>X1=l&#aIxPFFAavO&QEJsw!7heg2Bglco;*5EMFji^Lv{sXig=-gP8( d6XD2c>v`T6GdmF)Dq}}MC09vFgr?yKDk$QaS#xB@963V7BUv&dqcm5| z5)@F!v}2K)5<4XV5;G(+Th5_#iHLY=eo!hN8Kbt;p7+n|pZj-T&+mRLtH8Qw#ardp z11rO7u;QT5x@4up4hvv4G+E-85I`c6`Y7WQD_mi;1jeC|G+HTjhfz~bgR`$0l(F2<-jh>10RFSL2$VfTn2PG z0bTZ?%f;w&09`(XE{DN|Fv{#dic+x+=TJW0Mn6@W`PI~lv*{$=#6r4D zBWO3p@Jwo_d0aqusEBvcDBi#!BFup9fJ1cy)z z9>NrwkDa*ESZJ!C{g^~;bRJhwn8@rO%ZYTHGI=@Oq7A&&%s=~uI40}%%}Mc$>sZ8C6)ZN{2~S5!Fmipnn^hi#TY{l z)}oK5VF)*qAEQhQHJEO;?!*%E@=yxW5}rX<>0@3;(cHoZja8Vtsg&cz!}YvW(zutm zN-@VstJJbj1|-CrB|$|=lPpt5A=m7F3#yGv9G=4uC>tY8O1WwLj_G|pjc1GM=rvBJ zYgECjO*g-B4ZXx9m*`WTB9VN8{gT1|@&PI3Nzx;Yyi$g%KHejlYMh*xO=_7uq?*+Z znWgTk2hfB?l!Eplb5e!fU?<3T_z=)qWBC~P!>CK#<%+wbM_-8Jr>Bi-#xr0>cr$&=9i3j+Y6!0@LDBp0QOjg(Vu=rJ+^vbs?N5-kY z)gD=)Cg?8tLgnhmRkzxwm#C4tN0+Nt^k};bb10EJjFF!`=FbG#XAHw!Mmd})9TYIT zdpN{F@v+yqlyQzEstyjA6pw@?sC?p8VO1tMDp7YxK>4*tg;Y@clvjs!naa_Lc83aR zzwOZ>9kefEJ#FI{`iIW(>ok>z%a`P%-FWPrN3Ri+vvtxC-eWU7-~YkC+e{buT7Nl~$SmuxgI)+6oekd9YFj7z@q z>0~>os!YnD>eg-cG(FaKoDX%Lo#mX;b#{d_%pS12oK>`p7fL;SZT`!B>fks4L0@5yb|qvxn(v-{`jee+@$)G2+;_UcEC%bPme dUf_JMzcMNR>09{6)hyKv^HjFF4S8xOjD$Vv6wHBE^%vyAU5GIMkA`Gu z!TFE_z37JuqgMsZI1kUjZOp@aI1=|z6wbj`T8O!Lm-2BJji$X;hfy6=j>#rQC8}0f zrhKX&id7l7q0U@QhfZ||if{--!C_2-MbM5}P+;;YhXa^^EpQ1}V;`dVe=Nr1De~Yd z?4d&3ZuAdfBc;-B*iG*+(@3h}DJohu!OLp7>V>W9BNz)ORK3Z*Tit?U90if^EiQsI z_ygaCLL82@a0FBE8g$~@Gz=fb1C)#};$`yVI&x?~R?~F4h!-e_iIA!|fu_ZUH|=m=R@DO>Z4?C%tW5`9vcm<_UH*co`j^bvjWiJmgn23K;Ij*HB`W0*FB}TkTpYU`Fr_=1GH2RkhnyZt! zi%xKsjG#NbS3Ep{FUe+JDUV1a@07W6Pd$VN%!efCH;Lur(|FT3%py13Gzn#(8#~OE z42q!hSWfBe;Cb4{b4`LRTtsu}KG#zbCvqRvvtK4rA0HMUPvRT0k9{&$+IgR>(Ba%J zW!fX*a!zlBg(jy9um#g-7@Wqfv=|=Zk0!AfNYPJNK(maO<5bAW*l&I>qRBMCCn=Yo z<3akG^JFUBQ_KI8jxI{8_Jhw@; z_KQn`Iv`%{)_qcs9u2IivU6MLJC1 zuz%1VJJvzH!_IN$<9O4_9n3ZzC76Abb0H2;huPQ+c8fMk$Py2o;aVx` zTFFa{&#$-=ADDezrv?1H#L*5emi5%mznG1UlW;HH%$Yf7zPWSG z?;Dy&*CNFRw#Bhyk{&$UV%XXI-vu}`A>%u>BN_&lFwk{~Wf6ga5F4QzG0~c9x7%}b zbLsn-4wAQL>#jHLxz@M0erR2|U~$HRg$w8rOw={WDG++p`-*KI$xz8rISnixORuC7p^TPkrgZDFMNCUGme*b!4cDKM{})0e0KoFg5r@hf+aS@opH_jAbm z2v#Dd!#n@H=^WbKe2b6n3l6y?PZ#~b*Cop)e%Y)9FRbrCNX+bTv0ZQ~;xf@GP-lGj z;ojsFtB>8ALb;;O&HDfqh11afXnx{+`a27&s*YI%YvM<;>B5RU>ST>4Yds65SmLQkSlp!YkQh zFjqPenG<1i+8deET>kbDxKw!O#GPDn6yCZ1FCT-{v3^>NgC^V7zoux&Nk#nT5J(#Y zcV8GatU8Y6O}3ND3&>7I=Dn+ZIM9Z9pLUKQEAp&qC9$6_-4-b}=F}+2YQu`-*M{|n z{Rug1-8)Abqgw5&>jJ&>H>pVt$-%r;808%{4>;58H)GDW14jCkKQ2P=dXECu8YO#VwrrQ?_iK? ztZs)$ogsBBqD2KN((<6ebe?bzDnLaNQ=+eJy?bYSW0bTm^-e7{n{W39V;hcxJH}o} zMZEOw!7_*~zCzy%Lla6N^k_;bEOU+Dw?oqxr&2L)%)o`qBh``hMHR6OhPhut1RLN z*52sE&g3T8T=g(!g{?KLVt00s=x6V=g~nfv{%uPh8A3 zpd-+KJ%9t+mV_&oMaP(Ah3OKrRAOdIAjp7I6+t5g?K~q6AqTqoRw0u=)YCv`sDd-o zC8*etE%wG3t=#L-L@FwRDACD0j2Xig52{m|nBsP3(zCYZd0ow2lLx7w2*fm?WR1aC z<-l^9>O3tjb>7z|sLDCYOYtb*0*m{*i%d?W%uO^!V!90bDyNNMf9DzY9GgP%_!Nx! zn_**9m^(V>5u)^9)m^6!vkpFM-l=9s5ZQI8QwS+RA~Y1ST%2txPD}yvp`sVwsq97^ zy0>j^#YP3yB?zl9{113R)a^U&Yj1Q-_a9c+S{f7(aVdmmn00K6Yo7+rj6)f#xkQkH z&UA#Rb6D0L8)c38njmz6Q7+;7bIiLpCuG~pKW1N;qhO^pXmd3(a*;E5s0`mo22Ik) z&7yjkY5s`Z&(s;0Mv47hs2FXV{vya&<#XR8WRKhMw_l=`*;=|rna;`%e@*B-yb^Ns ucXk$*(Ij8QI5VYujg8;b*qH{SLLFV~&%!>+D?Twmehng4>@}Bx{{AnFJ=dWC literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_53.oracle.raw b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_53.oracle.raw new file mode 100644 index 0000000000000000000000000000000000000000..938f445b8e3b9d1c11f3fc63d66a7c8ad5410bff GIT binary patch literal 741 zcmZR`y`?(ozt?t^{5zgg81r_jO}^*7gSp_Y##AQXUFt>meWtS%?$Mm~&~GpVlVMLXj%9&ai(DBLCcLV6K@K3GFoobOuQM`$ym8jbK%XvlZ=%gH5W<- zo^-7In7L4L;z`GyA2T0HPUM^uW~<#&X9IE z?3VqmaGKEWx%ADJi08}9dwe_P!vy>hqb^_S(W;!BNH_J)PNYj2a^u|44Bi$*EQ zP2OIQYpO-ou5rD0w~U`}xr6stk9QkY1-C}JzU-}(J-9aT>%&$}(Vel| zQbpd*@m9Iq6wS4BlH-Sy73vJ@UDJwMf-^L(c>=iyrCzuRlgKW;D7{O#w6+YYU!F#FIi~mr&&;KKxHjKwA?U=V!JAB?zYw&efuF>~Z`6kcT z6so*jm#Fr7Ub4pR1*sBu7llgQpA;@LG12hi!a~7^8v_+TlEl8Xb%N}V%L&$Ei|Znv z9h{@dazDpiX?<7fr?V?e1zuMK>+POa`0wTxN15*}@m9wd)p9*O;H$wrA=^!v^L?hj z>H4YFUvD3@R%CBX_dULLGVkY`!Ittf>r?L^U9HOfF+0$FA%-@mHa!NQyBAhs!hJnuEta*-(BiO_kE_b z6z|!XA2k1dQBsycS!{Ha%?}VpG+8uPse3L&# zeAa%aqpxy5h|Z99IP8}Fu5g;os$xYr~k87$$)~<2Acejk6Z@Gj0jlu%XMGLJiU&>}-o?~iqHZAqv z^lADhas{_8bbWcXQkHXVpzXufLeZT|J>TA}Rpi|m>iD#?RC4c1-;Z}2RRy<3y1wkK zls&jM@aw}?P0^jP+fqf|&hb{c+!W2VbCTnSlNIU=>*LIyP0f=%vowVHPPditp+c{p zD;f>%??@MWKhIa~YD+B7?kUcn{_iZd{JgeU_xaj%#pes76>m?C7Ck@JpYwdDGw0!2 z=fB%)%|C80)c&|SRr~4uRQa3pBZY5H_7y(c?!kMh)rceD(#rJRXcp% zQETvZSFX|bRrw~**A%L}T$iZ!dS0@|?FFe4cNc|9-JcXLGcnQd;=)3~hZ_SGKa#}0 zwRM8*kIMZh|SOa)$71ncddR`~Dc7Dt)yE%8>z7u9k- zJ>aXsJR#dnne%<7zv=p^)n9KPv{qzqO!qy$b~5kho57ayGwW0DA6>1={V_YxeB<=m z?{^Q|Dswhx`k!1smH+GQoLIlD%clQ(|G?csb3$X$%~QLTgjupscPR=nXGQyLUNZI1+k38N>V5SE0Qxg-VgLXD literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_97.j2c b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_97.j2c new file mode 100644 index 0000000000000000000000000000000000000000..b8782c6e0ecd72966dca7d6dfa0f16e144b62f0e GIT binary patch literal 1774 zcmXw(4OCNQ7{~wbCNKr+PHZ5QXAV`2i_~;W7r9r!e9`(2j|W17;y1 z;lTNoU#a6zegrHOW|u*UG!e-p7RN700!<_^L&jb2z0+6h#1R|PHI9I`o-VulhiG))FsbcM2O$_~1#Z8Nw zwz}2%E_^{^($=TG78@;pbA7secIe!&(AlBRVvOU_YoRvmU1){k|weiGf;FTt@RR->Z;+cdnILR0!N#m=~M2{&o$hI z92~#0iNqtA9g~-0?HHwRJ6DF?Vn?3pHp6C@!t2|!$@W`Jy`_{^n#fxey1t~y)JJ?M z)RbPdSW4HBEoPW+Ug%wP4|0-2dz0NIAF?vm6cikApsu84Q^(%zVY-)ff%`_M^v!7u z(<5aM+%;>q-_OtBjHfrQjA<&(uU$LyLBqsFOyF)59rG!-$*L!IS9ByE6M0^`&D4RM zs1>-WtgI>Iu>Y^h+6k3|Ti3JK@3?Kte~6wX3dR9(P%Z9nP;F9umniLg!Ry2L_jDTb z$bgD-`ZY7Tr_aK6epcG8={{=@FZt_RG7R}U?UIjx%RG9Bq(;h`Ze^^Lk|`q3lHRRh zjXp_R;_sLS;Ve{#<;!W$CqIZSOiF3VI?`Ejp(N*ckUCn`<17pj&@oXbOdFRroX&r{ zZ^D0IX1(Hw(qYVtn<0#m&|TW)8l(R2mj?!srvy8eg?beN4RmYuO*67NEa4yCa8tuH zYmz?{25jE3>OP_j=o9C+A-9Saeoj_nTKHdH13V##k&fr^a}^$J^TVsI^y>(Q*|xKj zFLw{}xyoPt_T~=p^*6kam$UvC--F%1_{AR<1Wpf(zqmnV74nzURaaEmcEo`?Jg2%6^BH;w?thP5lGg20XJUxSWD56dX}} zD2)mJKHix2MNe7}$!UW=+lXdm_9KC;TlLXDUZY}TZZ%HkbQGHH4q~I={Uu^|)!tXl z*NLc8SWvNI2p%pn-?EGBH^(cYAX%$@|BVaX;woCV3+rs@50eMPnb>jE<7vxPVoNGT zb9ae4pn|@nz8aerO6`d&y9S4_jDwzU`KK0F%feWwL&;&+SPoW3^;QpwcW$btyL?cg zQEGdUM@3GJHuIO2aqgTvcMBgi5xT>Xf2gx6HYma9EoJ!SrUOxzGoiI~fru|>(IOL_ zQZOR#Ntc=Zh1x;?|FM-J?cL>!O%Z|%34UCx%RWSM%Lc(etLjRv$}O>s9S&kz~_?4_6R&I zE{f{ESv1)9oi5RGg%kOjlLM^GoIO!KN8WM&c>(n<*0QTM@%P_`Pex73WDxo8}AWTc|d}N2D*)U_Zd;U z(Lf(kYbas#78QFV*#P5FEK&kP{4e#?7(JOe46y;jlQV?4Rg{>jYT`Xs)knz3XdFfE z$5h9MOjK>_U{5GMmuY`)N_3q=vdjiJ!|`meWtM#?$Mm~&~GnVlVMLXj%9&ai(DBLCcLV6K@K3GFoobOuQM`$ym8jbK%XvlZ=%gH5W<- zo^-7In7L4L;z`GyA2T0HPUM^uW~<#&X9IE z?3VqmaGKEWx%ADJi08}9c_cw5~oy>hqb^_S(W;!BNH_J)PNYj2a^u|44Bi$*EQ z&E8&*YpO-ou5rD0w~U`}xr6stk9QkY1-C}JzU-}(J-9aT>%&$}(Ot3I zQbpd*@m9Iq6wS4BlGBHimFf%|;>@2-&67Q|G=%w1x0UdrLa(1I8V&C6NEdrQ&sXhg zODxasDb64N?UDJwMf-^L(c>=iyrCzuRlgKW;D7{O#w6*}AQ!F#FIi~mr&&;KKxHjKwA?U=V!JAB?zYw&efuF>~Z`6kcT z6so*jm#Fr7Ub4pR1*sBu7llgQpA;c8G12hi!a~7^8v_+T5@ZJ;skyavg6xmW3D#nZ z>mr{WoTJHdKgV5ZeOKzIvnxymURMO`?VeWn@8%XqneVOfR>v3Bay>oZtHC@W+fAAC zeWt(Z`l;1lZy&T)WN%FOJ-&7_@8_Grmhv;}Q|}*Lt;+o|JJ5XN^xE%t58EnpHfQ>u zTtAin>+RfFzpcxr|9k(y-9mFhQ_;;+yOo4lvSWO=ES>iE-F-JRjsE(=8z*-u3NdFz S`)pn^_0QXTu4Za|^#uT#Gi);e literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_97.source.ppm b/crates/j2k-test-support/fixtures/htj2k/openjph_batch/rgb_u8_97.source.ppm new file mode 100644 index 0000000000000000000000000000000000000000..ee4ec521991223b1ced9df3ea0a6ba8328be9443 GIT binary patch literal 754 zcmWGA<1(~VFf`^eGBxF5$i1aH>A%-@mHa!NQyBAhs!hJnuEta*-(BiO_kE_b z6z|!XA2k1dQBsycS!{Ha%?}VpG+8uPse3L&# zeAa%aqpxy5h|Z99IP8}Fu5g;os$xYr~k87$$)~<2Acejk6Z@Gj0jlu%XMGLJiU&>}-o?~iqHZAqv z^lADhas{_8bbWcXQkHXVpzXufLeZT|J>TA}Rpi|m>iD#?RC4c1-;Z}2RRy<3y1wkK zls&jM@aw}?P0^jP+fqf|&hb{c+!W2VbCTnSlNIU=>*LIyP0f=%vowVHPPditp+c{p zD;f>%??@MWKhIa~YD+B7?kUcn{_iZd{JgeU_xaj%#pes76>m?C7Ck@JpYwdDGw0!2 z=fB%)%|C80)c&|SRr~4uRQa3pBZY5H_7y(c?!kMh)rceD(#rJRXcp% zQETvZSFX|bRrw~**A%L}T$iZ!dS0@|?FFe4cNc|9-JcXLGcnQd;=)3~hZ_SGKa#}0 zwRM8*kIMZh|SOa)$71ncddR`~Dc7Dt)yE%8>z7u9k- zJ>aXsJR#dnne%<7zv=p^)n9KPv{qzqO!qy$b~5kho57ayGwW0DA6>1={V_YxeB<=m z?{^Q|Dswhx`k!1smH+GQoLIlD%clQ(|G?csb3$X$%~QLTgjupscPR=nXGQyLUNZI1+k38N>V5SE0Qxg-VgLXD literal 0 HcmV?d00001 diff --git a/crates/j2k-test-support/src/fixtures.rs b/crates/j2k-test-support/src/fixtures.rs index 7fe6b434..bc59dd20 100644 --- a/crates/j2k-test-support/src/fixtures.rs +++ b/crates/j2k-test-support/src/fixtures.rs @@ -22,429 +22,30 @@ pub const JPEG_BASELINE_420_RESTART_32X16: &[u8] = pub const JPEG_BASELINE_420_RESTART_32X16_RGB: &[u8] = include_bytes!("../fixtures/conformance/baseline_420_restart_32x16.rgb"); -/// Minimal grayscale JPEG with caller-provided dimensions and one entropy byte. -/// -/// This is intentionally not a complete image for large dimensions; tests use -/// it to exercise header validation and row-streaming paths without allocating -/// full-image entropy payloads. -pub fn minimal_grayscale_jpeg_with_dimensions(width: u16, height: u16) -> Vec { - let mut bytes = grayscale_jpeg_header(width, height); - bytes.extend_from_slice(&[0x00, 0xff, 0xd9]); - bytes -} - -/// Baseline grayscale JPEG with one zero-DC entropy byte per MCU. -pub fn baseline_grayscale_jpeg(width: u16, height: u16) -> Vec { - let mut bytes = grayscale_jpeg_header(width, height); - let mcu_cols = u32::from(width).div_ceil(8); - let mcu_rows = u32::from(height).div_ceil(8); - let mcu_count = (mcu_cols * mcu_rows) as usize; - bytes.extend(core::iter::repeat_n(0x00, mcu_count)); - bytes.extend_from_slice(&[0xff, 0xd9]); - bytes -} - -/// Minimal 16x16 baseline JPEG with 4:2:0 sampling. -pub fn minimal_baseline_jpeg() -> Vec { - let mut out = Vec::new(); - out.extend_from_slice(&[0xff, 0xd8]); - out.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - out.extend(core::iter::repeat_n(1u8, 64)); - out.extend_from_slice(&[ - 0xff, 0xc0, 0x00, 17, 8, 0, 16, 0, 16, 3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0, - ]); - out.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xaa, - ]); - out.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xbb, - ]); - out.extend_from_slice(&[0xff, 0xda, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); - out.extend_from_slice(&[0x00, 0xff, 0xd9]); - out -} - -/// Minimal baseline JPEG with a DRI marker inserted before SOS. -/// -/// # Panics -/// -/// Panics if the generated minimal fixture no longer contains an SOS marker. -pub fn minimal_baseline_jpeg_with_restart_interval(interval: u16) -> Vec { - let mut bytes = minimal_baseline_jpeg(); - let sos_pos = bytes - .windows(2) - .position(|window| window == [0xff, 0xda]) - .expect("minimal fixture includes SOS marker"); - let [interval_high, interval_low] = interval.to_be_bytes(); - bytes.splice( - sos_pos..sos_pos, - [0xff, 0xdd, 0x00, 0x04, interval_high, interval_low], - ); - bytes -} - -/// Restart-coded grayscale JPEG with one zero-DC block per MCU. -pub fn restart_coded_grayscale_jpeg(width: u16, height: u16) -> Vec { - let mut bytes = grayscale_jpeg_prefix(width, height); - bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); - append_grayscale_huffman_and_scan_header(&mut bytes); - - let mcu_cols = u32::from(width).div_ceil(8); - let mcu_rows = u32::from(height).div_ceil(8); - let mcu_count = (mcu_cols * mcu_rows) as usize; - for mcu in 0..mcu_count { - bytes.push(0x00); - if mcu + 1 != mcu_count { - bytes.extend_from_slice(&[0xff, 0xd0 | restart_index(mcu)]); - } - } - - bytes.extend_from_slice(&[0xff, 0xd9]); - bytes -} - -fn grayscale_jpeg_header(width: u16, height: u16) -> Vec { - let mut bytes = grayscale_jpeg_prefix(width, height); - append_grayscale_huffman_and_scan_header(&mut bytes); - bytes -} - -fn restart_index(mcu: usize) -> u8 { - u8::try_from(mcu & 0x07).expect("restart index is three bits") -} - -fn grayscale_jpeg_prefix(width: u16, height: u16) -> Vec { - let mut bytes = Vec::new(); - let [height_high, height_low] = height.to_be_bytes(); - let [width_high, width_low] = width.to_be_bytes(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(core::iter::repeat_n(16u8, 64)); - bytes.extend_from_slice(&[ - 0xff, - 0xc0, - 0x00, - 11, - 8, - height_high, - height_low, - width_high, - width_low, - 1, - 1, - 0x11, - 0, - ]); - bytes -} - -fn append_grayscale_huffman_and_scan_header(bytes: &mut Vec) { - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); -} - -/// Minimal raw J2K codestream containing SIZ, COD, and SOT markers. -pub fn minimal_j2k_codestream() -> Vec { - let mut bytes = vec![0xff, 0x4f]; - let mut siz = Vec::new(); - push_u16(&mut siz, 0); - push_u32(&mut siz, 128); - push_u32(&mut siz, 64); - push_u32(&mut siz, 0); - push_u32(&mut siz, 0); - push_u32(&mut siz, 64); - push_u32(&mut siz, 64); - push_u32(&mut siz, 0); - push_u32(&mut siz, 0); - push_u16(&mut siz, 3); - for _ in 0..3 { - siz.extend_from_slice(&[0x07, 0x01, 0x01]); - } - bytes.extend_from_slice(&[0xff, 0x51]); - push_u16(&mut bytes, segment_length_u16(siz.len())); - bytes.extend_from_slice(&siz); - - let cod = [0x00, 0x00, 0x00, 0x01, 0x01, 0x05, 0x04, 0x04, 0x00, 0x01]; - bytes.extend_from_slice(&[0xff, 0x52]); - push_u16(&mut bytes, segment_length_u16(cod.len())); - bytes.extend_from_slice(&cod); - bytes.extend_from_slice(&[0xff, 0x90, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); - bytes -} - -/// Rewrites one component's SIZ sampling factors in a raw codestream fixture. -/// -/// # Panics -/// -/// Panics if `codestream` does not contain a SIZ marker or the requested -/// component descriptor is not present. -pub fn rewrite_j2k_component_sampling( - codestream: &mut [u8], - component: usize, - x_rsiz: u8, - y_rsiz: u8, -) { - let siz = codestream - .windows(2) - .position(|marker| marker == [0xFF, 0x51]) - .expect("SIZ marker"); - let component_offset = siz + 40 + component * 3; - codestream[component_offset + 1] = x_rsiz; - codestream[component_offset + 2] = y_rsiz; -} - -/// Minimal JP2 wrapper around [`minimal_j2k_codestream`]. -pub fn minimal_jp2() -> Vec { - wrap_jp2_codestream(&minimal_j2k_codestream(), 128, 64, 3, 8, 16) -} - -/// Wraps a codestream in a JP2 container with an enumerated colorspace. -pub fn wrap_jp2_codestream( - codestream: &[u8], - width: u32, - height: u32, - components: u16, - bit_depth: u8, - colorspace_enum: u32, -) -> Vec { - let mut bytes = jp2_prefix(); - let bpc = bit_depth.saturating_sub(1); - bytes.extend_from_slice(&[ - 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', - ]); - bytes.extend_from_slice(&height.to_be_bytes()); - bytes.extend_from_slice(&width.to_be_bytes()); - bytes.extend_from_slice(&components.to_be_bytes()); - bytes.extend_from_slice(&[bpc, 7, 0, 0]); - bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); - bytes.extend_from_slice(&colorspace_enum.to_be_bytes()); - append_jp2c(&mut bytes, codestream); - bytes -} - -/// Wraps a four-component codestream in a JP2 container with an alpha channel. -pub fn wrap_jp2_rgba_codestream( - codestream: &[u8], - width: u32, - height: u32, - bit_depth: u8, -) -> Vec { - let mut bytes = jp2_prefix(); - let bpc = bit_depth.saturating_sub(1); - let jp2h_len = 8_u32 + 22 + 15 + 34; - bytes.extend_from_slice(&jp2h_len.to_be_bytes()); - bytes.extend_from_slice(b"jp2h"); - bytes.extend_from_slice(&[0, 0, 0, 22, b'i', b'h', b'd', b'r']); - bytes.extend_from_slice(&height.to_be_bytes()); - bytes.extend_from_slice(&width.to_be_bytes()); - bytes.extend_from_slice(&4_u16.to_be_bytes()); - bytes.extend_from_slice(&[bpc, 7, 0, 0]); - bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); - bytes.extend_from_slice(&16_u32.to_be_bytes()); - bytes.extend_from_slice(&[0, 0, 0, 34, b'c', b'd', b'e', b'f']); - bytes.extend_from_slice(&4_u16.to_be_bytes()); - for (channel, channel_type, association) in [ - (0_u16, 0_u16, 1_u16), - (1_u16, 0_u16, 2_u16), - (2_u16, 0_u16, 3_u16), - (3_u16, 1_u16, 0_u16), - ] { - bytes.extend_from_slice(&channel.to_be_bytes()); - bytes.extend_from_slice(&channel_type.to_be_bytes()); - bytes.extend_from_slice(&association.to_be_bytes()); - } - append_jp2c(&mut bytes, codestream); - bytes -} - -fn jp2_prefix() -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0d, 0x0a, 0x87, 0x0a]); - bytes.extend_from_slice(&[ - 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', - b' ', - ]); - bytes -} - -fn append_jp2c(bytes: &mut Vec, codestream: &[u8]) { - let len = u32::try_from( - codestream - .len() - .checked_add(8) - .expect("JP2 codestream box length must not overflow usize"), - ) - .expect("JP2 codestream box length must fit u32"); - bytes.extend_from_slice(&len.to_be_bytes()); - bytes.extend_from_slice(b"jp2c"); - bytes.extend_from_slice(codestream); -} - -fn segment_length_u16(payload_len: usize) -> u16 { - u16::try_from( - payload_len - .checked_add(2) - .expect("marker segment length must not overflow usize"), - ) - .expect("marker segment length must fit u16") -} - -fn push_u16(out: &mut Vec, value: u16) { - out.extend_from_slice(&value.to_be_bytes()); -} - -fn push_u32(out: &mut Vec, value: u32) { - out.extend_from_slice(&value.to_be_bytes()); -} - #[cfg(feature = "j2k-native-fixtures")] -fn htj2k_options(reversible: bool) -> j2k_native::EncodeOptions { - j2k_native::EncodeOptions { - reversible, - num_decomposition_levels: 1, - ..j2k_native::EncodeOptions::default() - } -} - -#[cfg(feature = "j2k-native-fixtures")] -fn encode_htj2k_fixture( - pixels: &[u8], - width: u32, - height: u32, - components: u8, - reversible: bool, -) -> Vec { - j2k_native::encode_htj2k( - pixels, - width, - height, - u16::from(components), - 8, - false, - &htj2k_options(reversible), - ) - .expect("encode HTJ2K fixture") -} - -#[cfg(feature = "j2k-native-fixtures")] -/// Deterministic reversible HTJ2K grayscale fixture. -pub fn htj2k_gray8_fixture(width: u32, height: u32) -> Vec { - let pixels = (0..width * height) - .map(|idx| (idx & 0xff) as u8) - .collect::>(); - encode_htj2k_fixture(&pixels, width, height, 1, true) -} - -#[cfg(feature = "j2k-native-fixtures")] -/// Deterministic irreversible 9/7 HTJ2K grayscale fixture. -pub fn htj2k_gray8_97_fixture(width: u32, height: u32) -> Vec { - let pixels = (0..width * height) - .map(|idx| ((idx * 11) & 0xff) as u8) - .collect::>(); - encode_htj2k_fixture(&pixels, width, height, 1, false) -} - -#[cfg(feature = "j2k-native-fixtures")] -/// Larger reversible grayscale HTJ2K fixture with 64x64 code blocks. -/// -/// # Panics -/// -/// Panics if the native encoder rejects the deterministic fixture input. -pub fn htj2k_gray8_large_fixture(width: u32, height: u32) -> Vec { - let mut pixels = Vec::with_capacity(width as usize * height as usize); - for y in 0..height { - for x in 0..width { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - } - } - let options = j2k_native::EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..j2k_native::EncodeOptions::default() - }; - j2k_native::encode_htj2k(&pixels, width, height, 1, 8, false, &options) - .expect("encode large HTJ2K grayscale fixture") -} - -#[cfg(feature = "j2k-native-fixtures")] -/// Deterministic reversible HTJ2K RGB fixture. -pub fn htj2k_rgb8_fixture(width: u32, height: u32) -> Vec { - htj2k_rgb8_fixture_with_pixels(width, height).0 -} - -#[cfg(feature = "j2k-native-fixtures")] -/// Deterministic reversible HTJ2K RGB fixture and its source pixels. -pub fn htj2k_rgb8_fixture_with_pixels(width: u32, height: u32) -> (Vec, Vec) { - let pixels = (0u32..width * height * 3) - .map(|idx| ((idx * 13 + idx / 3) & 0xff) as u8) - .collect::>(); - let codestream = encode_htj2k_fixture(&pixels, width, height, 3, true); - (codestream, pixels) -} - -#[cfg(feature = "j2k-native-fixtures")] -/// Seeded reversible HTJ2K RGB fixture. -pub fn htj2k_rgb8_pattern_fixture(width: u32, height: u32, seed: u32) -> Vec { - let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); - for idx in 0..width * height { - pixels.push(((idx * seed + idx / 3) & 0xff) as u8); - pixels.push(((idx * (seed + 11) + 7) & 0xff) as u8); - pixels.push(((idx * (seed + 23) + 19) & 0xff) as u8); - } - encode_htj2k_fixture(&pixels, width, height, 3, true) -} - -#[cfg(feature = "j2k-native-fixtures")] -/// Deterministic irreversible 9/7 HTJ2K RGB fixture. -pub fn htj2k_rgb8_97_fixture(width: u32, height: u32) -> Vec { - let pixels = (0u32..width * height * 3) - .map(|idx| ((idx * 17 + idx / 5) & 0xff) as u8) - .collect::>(); - encode_htj2k_fixture(&pixels, width, height, 3, false) -} - -#[cfg(feature = "j2k-native-fixtures")] -/// Deterministic classic J2K grayscale fixture. -/// -/// # Panics -/// -/// Panics if the native encoder rejects the deterministic fixture input. -pub fn classic_j2k_gray8_fixture(width: u32, height: u32) -> Vec { - let pixels = (0..width * height) - .map(|idx| (idx & 0xff) as u8) - .collect::>(); - let options = j2k_native::EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..j2k_native::EncodeOptions::default() - }; - j2k_native::encode(&pixels, width, height, 1, 8, false, &options) - .expect("encode classic J2K grayscale fixture") -} - -/// `OpenHTJ2K` refinement fixture with a compact output plane. -pub fn openhtj2k_refinement_fixture() -> &'static [u8] { - include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_12_b11.j2k") -} - -/// Expected grayscale pixels for [`openhtj2k_refinement_fixture`]. -pub fn openhtj2k_refinement_pixels() -> &'static [u8] { - include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_12_b11.gray") -} - -/// `OpenHTJ2K` odd refinement fixture used by CUDA plan tests. -pub fn openhtj2k_refinement_odd_fixture() -> &'static [u8] { - include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k") -} - -/// Expected grayscale pixels for [`openhtj2k_refinement_odd_fixture`]. -pub fn openhtj2k_refinement_odd_pixels() -> &'static [u8] { - include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray") -} +mod generated_htj2k; +mod jp2; +mod jpeg; +mod openjph; + +#[cfg(feature = "j2k-native-fixtures")] +pub use generated_htj2k::{ + classic_j2k_gray8_fixture, generated_htj2k_rgba_fixture, htj2k_gray8_97_fixture, + htj2k_gray8_fixture, htj2k_gray8_large_fixture, htj2k_rgb8_97_fixture, htj2k_rgb8_fixture, + htj2k_rgb8_fixture_with_pixels, htj2k_rgb8_pattern_fixture, Htj2kRgbaAlpha, Htj2kRgbaFixture, + Htj2kRgbaSampleProfile, Htj2kRgbaSamples, +}; +pub use jp2::{ + minimal_j2k_codestream, minimal_jp2, rewrite_j2k_component_sampling, wrap_jp2_codestream, + wrap_jp2_rgba_codestream, +}; +pub use jpeg::{ + baseline_grayscale_jpeg, minimal_baseline_jpeg, minimal_baseline_jpeg_with_restart_interval, + minimal_grayscale_jpeg_with_dimensions, restart_coded_grayscale_jpeg, +}; +pub use openjph::{ + openhtj2k_refinement_fixture, openhtj2k_refinement_odd_fixture, + openhtj2k_refinement_odd_pixels, openhtj2k_refinement_pixels, openhtj2k_sigprop_fixture, + openhtj2k_sigprop_overlap_fixture, openhtj2k_sigprop_overlap_pixels, + openhtj2k_sigprop_pixels_le, openjph_batch_fixtures, OpenJphBatchFixture, +}; diff --git a/crates/j2k-test-support/src/fixtures/generated_htj2k.rs b/crates/j2k-test-support/src/fixtures/generated_htj2k.rs new file mode 100644 index 00000000..c44946a8 --- /dev/null +++ b/crates/j2k-test-support/src/fixtures/generated_htj2k.rs @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +fn htj2k_options(reversible: bool) -> j2k_native::EncodeOptions { + j2k_native::EncodeOptions { + reversible, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + } +} + +#[cfg(feature = "j2k-native-fixtures")] +fn encode_htj2k_fixture( + pixels: &[u8], + width: u32, + height: u32, + components: u8, + reversible: bool, +) -> Vec { + j2k_native::encode_htj2k( + pixels, + width, + height, + u16::from(components), + 8, + false, + &htj2k_options(reversible), + ) + .expect("encode HTJ2K fixture") +} + +/// Native sample profile for the shared generated RGBA HTJ2K fixture. +#[cfg(feature = "j2k-native-fixtures")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Htj2kRgbaSampleProfile { + /// Unsigned eight-bit samples with reversible RCT enabled for RGB. + U8Rct, + /// Unsigned twelve-bit samples without MCT. + U12, + /// Signed sixteen-bit samples without MCT. + I16, +} + +/// Alpha interpretation callers must encode in the surrounding JPH/JP2 `cdef` box. +#[cfg(feature = "j2k-native-fixtures")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Htj2kRgbaAlpha { + /// Unassociated (straight) opacity. + Straight, + /// Premultiplied opacity metadata. Source samples remain unchanged. + Premultiplied, +} + +/// Exact interleaved source samples for a generated RGBA HTJ2K fixture. +#[cfg(feature = "j2k-native-fixtures")] +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Htj2kRgbaSamples { + /// Unsigned eight-bit RGBA samples. + U8(Vec), + /// Unsigned twelve-bit RGBA samples stored in `u16` values. + U16(Vec), + /// Signed sixteen-bit RGBA samples. + I16(Vec), +} + +/// Shared deterministic single-tile RGBA HTJ2K fixture and exact source oracle. +#[cfg(feature = "j2k-native-fixtures")] +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Htj2kRgbaFixture { + /// Raw Part 15 codestream. Callers add identity RGBA `cdef` metadata. + pub encoded: Vec, + /// Exact interleaved source samples in R, G, B, A order. + pub samples: Htj2kRgbaSamples, + /// Image width. + pub width: u32, + /// Image height. + pub height: u32, + /// Declared component precision. + pub bit_depth: u8, + /// Whether every component is signed. + pub signed: bool, + /// Whether reversible RCT is applied to RGB. Alpha is never transformed. + pub use_mct: bool, + /// Alpha interpretation callers must preserve in container metadata. + pub alpha: Htj2kRgbaAlpha, +} + +/// Generate the shared exact RGBA HTJ2K fixture used by CPU and GPU batch tests. +/// +/// The raw codestream cannot carry alpha semantics. Callers must wrap `encoded` +/// in JPH/JP2 with identity R, G, B channel definitions followed by the requested +/// whole-image alpha interpretation. +/// +/// # Panics +/// +/// Panics if the native encoder rejects the deterministic reversible fixture. +#[cfg(feature = "j2k-native-fixtures")] +pub fn generated_htj2k_rgba_fixture( + profile: Htj2kRgbaSampleProfile, + alpha: Htj2kRgbaAlpha, +) -> Htj2kRgbaFixture { + const WIDTH: u32 = 8; + const HEIGHT: u32 = 8; + const COMPONENTS: u16 = 4; + let sample_count = WIDTH as usize * HEIGHT as usize * usize::from(COMPONENTS); + let (samples, pixels, bit_depth, signed, use_mct) = match profile { + Htj2kRgbaSampleProfile::U8Rct => { + let values = (0..sample_count) + .map(|index| { + u8::try_from((index * 37 + index / 3) & 0xff) + .expect("masked RGBA fixture sample must fit u8") + }) + .collect::>(); + (Htj2kRgbaSamples::U8(values.clone()), values, 8, false, true) + } + Htj2kRgbaSampleProfile::U12 => { + let values = (0..sample_count) + .map(|index| { + let index = u32::try_from(index).expect("RGBA fixture index must fit u32"); + u16::try_from((index * 977 + 31) & 0x0fff) + .expect("masked RGBA fixture sample must fit u16") + }) + .collect::>(); + let pixels = values + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect(); + (Htj2kRgbaSamples::U16(values), pixels, 12, false, false) + } + Htj2kRgbaSampleProfile::I16 => { + let values = (0..sample_count) + .map(|index| { + let index = i32::try_from(index).expect("RGBA fixture index must fit i32"); + i16::try_from((index * 113 + 19) % 20_001 - 10_000) + .expect("bounded RGBA fixture sample must fit i16") + }) + .collect::>(); + let pixels = values + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect(); + (Htj2kRgbaSamples::I16(values), pixels, 16, true, false) + } + }; + let options = j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + use_mct, + ..j2k_native::EncodeOptions::default() + }; + let encoded = j2k_native::encode_htj2k( + &pixels, WIDTH, HEIGHT, COMPONENTS, bit_depth, signed, &options, + ) + .expect("encode shared HTJ2K RGBA fixture"); + Htj2kRgbaFixture { + encoded, + samples, + width: WIDTH, + height: HEIGHT, + bit_depth, + signed, + use_mct, + alpha, + } +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic reversible HTJ2K grayscale fixture. +pub fn htj2k_gray8_fixture(width: u32, height: u32) -> Vec { + let pixels = (0..width * height) + .map(|idx| (idx & 0xff) as u8) + .collect::>(); + encode_htj2k_fixture(&pixels, width, height, 1, true) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic irreversible 9/7 HTJ2K grayscale fixture. +pub fn htj2k_gray8_97_fixture(width: u32, height: u32) -> Vec { + let pixels = (0..width * height) + .map(|idx| ((idx * 11) & 0xff) as u8) + .collect::>(); + encode_htj2k_fixture(&pixels, width, height, 1, false) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Larger reversible grayscale HTJ2K fixture with 64x64 code blocks. +/// +/// # Panics +/// +/// Panics if the native encoder rejects the deterministic fixture input. +pub fn htj2k_gray8_large_fixture(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..j2k_native::EncodeOptions::default() + }; + j2k_native::encode_htj2k(&pixels, width, height, 1, 8, false, &options) + .expect("encode large HTJ2K grayscale fixture") +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic reversible HTJ2K RGB fixture. +pub fn htj2k_rgb8_fixture(width: u32, height: u32) -> Vec { + htj2k_rgb8_fixture_with_pixels(width, height).0 +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic reversible HTJ2K RGB fixture and its source pixels. +pub fn htj2k_rgb8_fixture_with_pixels(width: u32, height: u32) -> (Vec, Vec) { + let pixels = (0u32..width * height * 3) + .map(|idx| ((idx * 13 + idx / 3) & 0xff) as u8) + .collect::>(); + let codestream = encode_htj2k_fixture(&pixels, width, height, 3, true); + (codestream, pixels) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Seeded reversible HTJ2K RGB fixture. +pub fn htj2k_rgb8_pattern_fixture(width: u32, height: u32, seed: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for idx in 0..width * height { + pixels.push(((idx * seed + idx / 3) & 0xff) as u8); + pixels.push(((idx * (seed + 11) + 7) & 0xff) as u8); + pixels.push(((idx * (seed + 23) + 19) & 0xff) as u8); + } + encode_htj2k_fixture(&pixels, width, height, 3, true) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic irreversible 9/7 HTJ2K RGB fixture. +pub fn htj2k_rgb8_97_fixture(width: u32, height: u32) -> Vec { + let pixels = (0u32..width * height * 3) + .map(|idx| ((idx * 17 + idx / 5) & 0xff) as u8) + .collect::>(); + encode_htj2k_fixture(&pixels, width, height, 3, false) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic classic J2K grayscale fixture. +/// +/// # Panics +/// +/// Panics if the native encoder rejects the deterministic fixture input. +pub fn classic_j2k_gray8_fixture(width: u32, height: u32) -> Vec { + let pixels = (0..width * height) + .map(|idx| (idx & 0xff) as u8) + .collect::>(); + let options = j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + }; + j2k_native::encode(&pixels, width, height, 1, 8, false, &options) + .expect("encode classic J2K grayscale fixture") +} diff --git a/crates/j2k-test-support/src/fixtures/jp2.rs b/crates/j2k-test-support/src/fixtures/jp2.rs new file mode 100644 index 00000000..691f7f8f --- /dev/null +++ b/crates/j2k-test-support/src/fixtures/jp2.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +pub fn minimal_j2k_codestream() -> Vec { + let mut bytes = vec![0xff, 0x4f]; + let mut siz = Vec::new(); + push_u16(&mut siz, 0); + push_u32(&mut siz, 128); + push_u32(&mut siz, 64); + push_u32(&mut siz, 0); + push_u32(&mut siz, 0); + push_u32(&mut siz, 64); + push_u32(&mut siz, 64); + push_u32(&mut siz, 0); + push_u32(&mut siz, 0); + push_u16(&mut siz, 3); + for _ in 0..3 { + siz.extend_from_slice(&[0x07, 0x01, 0x01]); + } + bytes.extend_from_slice(&[0xff, 0x51]); + push_u16(&mut bytes, segment_length_u16(siz.len())); + bytes.extend_from_slice(&siz); + + let cod = [0x00, 0x00, 0x00, 0x01, 0x01, 0x05, 0x04, 0x04, 0x00, 0x01]; + bytes.extend_from_slice(&[0xff, 0x52]); + push_u16(&mut bytes, segment_length_u16(cod.len())); + bytes.extend_from_slice(&cod); + bytes.extend_from_slice(&[0xff, 0x90, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); + bytes +} + +/// Rewrites one component's SIZ sampling factors in a raw codestream fixture. +/// +/// # Panics +/// +/// Panics if `codestream` does not contain a SIZ marker or the requested +/// component descriptor is not present. +pub fn rewrite_j2k_component_sampling( + codestream: &mut [u8], + component: usize, + x_rsiz: u8, + y_rsiz: u8, +) { + let siz = codestream + .windows(2) + .position(|marker| marker == [0xFF, 0x51]) + .expect("SIZ marker"); + let component_offset = siz + 40 + component * 3; + codestream[component_offset + 1] = x_rsiz; + codestream[component_offset + 2] = y_rsiz; +} + +/// Minimal JP2 wrapper around [`minimal_j2k_codestream`]. +pub fn minimal_jp2() -> Vec { + wrap_jp2_codestream(&minimal_j2k_codestream(), 128, 64, 3, 8, 16) +} + +/// Wraps a codestream in a JP2 container with an enumerated colorspace. +pub fn wrap_jp2_codestream( + codestream: &[u8], + width: u32, + height: u32, + components: u16, + bit_depth: u8, + colorspace_enum: u32, +) -> Vec { + let mut bytes = jp2_prefix(); + let bpc = bit_depth.saturating_sub(1); + bytes.extend_from_slice(&[ + 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', + ]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&components.to_be_bytes()); + bytes.extend_from_slice(&[bpc, 7, 0, 0]); + bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); + bytes.extend_from_slice(&colorspace_enum.to_be_bytes()); + append_jp2c(&mut bytes, codestream); + bytes +} + +/// Wraps a four-component codestream in a JP2 container with an alpha channel. +pub fn wrap_jp2_rgba_codestream( + codestream: &[u8], + width: u32, + height: u32, + bit_depth: u8, +) -> Vec { + let mut bytes = jp2_prefix(); + let bpc = bit_depth.saturating_sub(1); + let jp2h_len = 8_u32 + 22 + 15 + 34; + bytes.extend_from_slice(&jp2h_len.to_be_bytes()); + bytes.extend_from_slice(b"jp2h"); + bytes.extend_from_slice(&[0, 0, 0, 22, b'i', b'h', b'd', b'r']); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&4_u16.to_be_bytes()); + bytes.extend_from_slice(&[bpc, 7, 0, 0]); + bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); + bytes.extend_from_slice(&16_u32.to_be_bytes()); + bytes.extend_from_slice(&[0, 0, 0, 34, b'c', b'd', b'e', b'f']); + bytes.extend_from_slice(&4_u16.to_be_bytes()); + for (channel, channel_type, association) in [ + (0_u16, 0_u16, 1_u16), + (1_u16, 0_u16, 2_u16), + (2_u16, 0_u16, 3_u16), + (3_u16, 1_u16, 0_u16), + ] { + bytes.extend_from_slice(&channel.to_be_bytes()); + bytes.extend_from_slice(&channel_type.to_be_bytes()); + bytes.extend_from_slice(&association.to_be_bytes()); + } + append_jp2c(&mut bytes, codestream); + bytes +} + +fn jp2_prefix() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0d, 0x0a, 0x87, 0x0a]); + bytes.extend_from_slice(&[ + 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', + b' ', + ]); + bytes +} + +fn append_jp2c(bytes: &mut Vec, codestream: &[u8]) { + let len = u32::try_from( + codestream + .len() + .checked_add(8) + .expect("JP2 codestream box length must not overflow usize"), + ) + .expect("JP2 codestream box length must fit u32"); + bytes.extend_from_slice(&len.to_be_bytes()); + bytes.extend_from_slice(b"jp2c"); + bytes.extend_from_slice(codestream); +} + +fn segment_length_u16(payload_len: usize) -> u16 { + u16::try_from( + payload_len + .checked_add(2) + .expect("marker segment length must not overflow usize"), + ) + .expect("marker segment length must fit u16") +} + +fn push_u16(out: &mut Vec, value: u16) { + out.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_be_bytes()); +} diff --git a/crates/j2k-test-support/src/fixtures/jpeg.rs b/crates/j2k-test-support/src/fixtures/jpeg.rs new file mode 100644 index 00000000..93380c40 --- /dev/null +++ b/crates/j2k-test-support/src/fixtures/jpeg.rs @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +/// Minimal grayscale JPEG with caller-provided dimensions and one entropy byte. +/// +/// This is intentionally not a complete image for large dimensions; tests use +/// it to exercise header validation and row-streaming paths without allocating +/// full-image entropy payloads. +pub fn minimal_grayscale_jpeg_with_dimensions(width: u16, height: u16) -> Vec { + let mut bytes = grayscale_jpeg_header(width, height); + bytes.extend_from_slice(&[0x00, 0xff, 0xd9]); + bytes +} + +/// Baseline grayscale JPEG with one zero-DC entropy byte per MCU. +pub fn baseline_grayscale_jpeg(width: u16, height: u16) -> Vec { + let mut bytes = grayscale_jpeg_header(width, height); + let mcu_cols = u32::from(width).div_ceil(8); + let mcu_rows = u32::from(height).div_ceil(8); + let mcu_count = (mcu_cols * mcu_rows) as usize; + bytes.extend(core::iter::repeat_n(0x00, mcu_count)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// Minimal 16x16 baseline JPEG with 4:2:0 sampling. +pub fn minimal_baseline_jpeg() -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(&[0xff, 0xd8]); + out.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + out.extend(core::iter::repeat_n(1u8, 64)); + out.extend_from_slice(&[ + 0xff, 0xc0, 0x00, 17, 8, 0, 16, 0, 16, 3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + out.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xaa, + ]); + out.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xbb, + ]); + out.extend_from_slice(&[0xff, 0xda, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); + out.extend_from_slice(&[0x00, 0xff, 0xd9]); + out +} + +/// Minimal baseline JPEG with a DRI marker inserted before SOS. +/// +/// # Panics +/// +/// Panics if the generated minimal fixture no longer contains an SOS marker. +pub fn minimal_baseline_jpeg_with_restart_interval(interval: u16) -> Vec { + let mut bytes = minimal_baseline_jpeg(); + let sos_pos = bytes + .windows(2) + .position(|window| window == [0xff, 0xda]) + .expect("minimal fixture includes SOS marker"); + let [interval_high, interval_low] = interval.to_be_bytes(); + bytes.splice( + sos_pos..sos_pos, + [0xff, 0xdd, 0x00, 0x04, interval_high, interval_low], + ); + bytes +} + +/// Restart-coded grayscale JPEG with one zero-DC block per MCU. +pub fn restart_coded_grayscale_jpeg(width: u16, height: u16) -> Vec { + let mut bytes = grayscale_jpeg_prefix(width, height); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); + append_grayscale_huffman_and_scan_header(&mut bytes); + + let mcu_cols = u32::from(width).div_ceil(8); + let mcu_rows = u32::from(height).div_ceil(8); + let mcu_count = (mcu_cols * mcu_rows) as usize; + for mcu in 0..mcu_count { + bytes.push(0x00); + if mcu + 1 != mcu_count { + bytes.extend_from_slice(&[0xff, 0xd0 | restart_index(mcu)]); + } + } + + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn grayscale_jpeg_header(width: u16, height: u16) -> Vec { + let mut bytes = grayscale_jpeg_prefix(width, height); + append_grayscale_huffman_and_scan_header(&mut bytes); + bytes +} + +fn restart_index(mcu: usize) -> u8 { + u8::try_from(mcu & 0x07).expect("restart index is three bits") +} + +fn grayscale_jpeg_prefix(width: u16, height: u16) -> Vec { + let mut bytes = Vec::new(); + let [height_high, height_low] = height.to_be_bytes(); + let [width_high, width_low] = width.to_be_bytes(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(core::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, + 0xc0, + 0x00, + 11, + 8, + height_high, + height_low, + width_high, + width_low, + 1, + 1, + 0x11, + 0, + ]); + bytes +} + +fn append_grayscale_huffman_and_scan_header(bytes: &mut Vec) { + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); +} diff --git a/crates/j2k-test-support/src/fixtures/openjph.rs b/crates/j2k-test-support/src/fixtures/openjph.rs new file mode 100644 index 00000000..f3d01192 --- /dev/null +++ b/crates/j2k-test-support/src/fixtures/openjph.rs @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[derive(Clone, Copy, Debug)] +pub struct OpenJphBatchFixture { + /// Stable fixture label used in assertion diagnostics. + pub name: &'static str, + /// Raw Part 15 codestream or boxed JPH file bytes. + pub encoded: &'static [u8], + /// OpenJPH-decoded interleaved native sample bytes. + pub oracle: &'static [u8], + /// Decoded image width. + pub width: u32, + /// Decoded image height. + pub height: u32, + /// One for grayscale or three for RGB. + pub components: usize, + /// Uniform component precision. + pub precision: u8, + /// Whether every component uses a signed sample domain. + pub signed: bool, + /// Whether the codestream uses the reversible 5/3 transform. + pub reversible: bool, + /// Whether [`Self::encoded`] is a boxed JPH file. + pub jph: bool, +} + +macro_rules! openjph_fixture { + ($name:literal, $encoded:literal, $oracle:literal, $components:literal, $precision:literal, $signed:literal, $reversible:literal, $jph:literal) => { + OpenJphBatchFixture { + name: $name, + encoded: include_bytes!(concat!("../../fixtures/htj2k/openjph_batch/", $encoded)), + oracle: include_bytes!(concat!("../../fixtures/htj2k/openjph_batch/", $oracle)), + width: 19, + height: 13, + components: $components, + precision: $precision, + signed: $signed, + reversible: $reversible, + jph: $jph, + } + }; +} + +static OPENJPH_BATCH_FIXTURES: &[OpenJphBatchFixture] = &[ + openjph_fixture!( + "openjph-gray-u8-53-raw", + "gray_u8_53.j2c", + "gray_u8_53.oracle.raw", + 1, + 8, + false, + true, + false + ), + openjph_fixture!( + "openjph-gray-u12-53-raw", + "gray_u12_53.j2c", + "gray_u12_53.oracle.raw", + 1, + 12, + false, + true, + false + ), + openjph_fixture!( + "openjph-gray-u12-53-jph", + "gray_u12_53.jph", + "gray_u12_53.oracle.raw", + 1, + 12, + false, + true, + true + ), + openjph_fixture!( + "openjph-gray-u16-53-raw", + "gray_u16_53.j2c", + "gray_u16_53.oracle.raw", + 1, + 16, + false, + true, + false + ), + openjph_fixture!( + "openjph-gray-s8-53-raw", + "gray_s8_53.j2c", + "gray_s8_53.oracle.raw", + 1, + 8, + true, + true, + false + ), + openjph_fixture!( + "openjph-gray-s12-53-raw", + "gray_s12_53.j2c", + "gray_s12_53.oracle.raw", + 1, + 12, + true, + true, + false + ), + openjph_fixture!( + "openjph-gray-s16-53-raw", + "gray_s16_53.j2c", + "gray_s16_53.oracle.raw", + 1, + 16, + true, + true, + false + ), + openjph_fixture!( + "openjph-rgb-u8-53-raw", + "rgb_u8_53.j2c", + "rgb_u8_53.oracle.raw", + 3, + 8, + false, + true, + false + ), + openjph_fixture!( + "openjph-rgb-u12-53-raw", + "rgb_u12_53.j2c", + "rgb_u12_53.oracle.raw", + 3, + 12, + false, + true, + false + ), + openjph_fixture!( + "openjph-rgb-u16-53-raw", + "rgb_u16_53.j2c", + "rgb_u16_53.oracle.raw", + 3, + 16, + false, + true, + false + ), + openjph_fixture!( + "openjph-rgb-s8-53-raw", + "rgb_s8_53.j2c", + "rgb_s8_53.oracle.raw", + 3, + 8, + true, + true, + false + ), + openjph_fixture!( + "openjph-rgb-s12-53-raw", + "rgb_s12_53.j2c", + "rgb_s12_53.oracle.raw", + 3, + 12, + true, + true, + false + ), + openjph_fixture!( + "openjph-rgb-s16-53-raw", + "rgb_s16_53.j2c", + "rgb_s16_53.oracle.raw", + 3, + 16, + true, + true, + false + ), + openjph_fixture!( + "openjph-rgb-s8-53-single-raw", + "rgb_s8_53_single.j2c", + "rgb_s8_53.oracle.raw", + 3, + 8, + true, + true, + false + ), + openjph_fixture!( + "openjph-rgb-s12-53-single-raw", + "rgb_s12_53_single.j2c", + "rgb_s12_53.oracle.raw", + 3, + 12, + true, + true, + false + ), + openjph_fixture!( + "openjph-rgb-s16-53-single-raw", + "rgb_s16_53_single.j2c", + "rgb_s16_53.oracle.raw", + 3, + 16, + true, + true, + false + ), + openjph_fixture!( + "openjph-gray-u12-97-raw", + "gray_u12_97.j2c", + "gray_u12_97.oracle.raw", + 1, + 12, + false, + false, + false + ), + openjph_fixture!( + "openjph-gray-u16-97-raw", + "gray_u16_97.j2c", + "gray_u16_97.oracle.raw", + 1, + 16, + false, + false, + false + ), + openjph_fixture!( + "openjph-gray-s12-97-raw", + "gray_s12_97.j2c", + "gray_s12_97.oracle.raw", + 1, + 12, + true, + false, + false + ), + openjph_fixture!( + "openjph-gray-s16-97-raw", + "gray_s16_97.j2c", + "gray_s16_97.oracle.raw", + 1, + 16, + true, + false, + false + ), + openjph_fixture!( + "openjph-rgb-u8-97-raw", + "rgb_u8_97.j2c", + "rgb_u8_97.oracle.raw", + 3, + 8, + false, + false, + false + ), + openjph_fixture!( + "openjph-rgb-u12-97-raw", + "rgb_u12_97.j2c", + "rgb_u12_97.oracle.raw", + 3, + 12, + false, + false, + false + ), + openjph_fixture!( + "openjph-rgb-s12-97-raw", + "rgb_s12_97.j2c", + "rgb_s12_97.oracle.raw", + 3, + 12, + true, + false, + false + ), +]; + +/// Return the independently encoded `OpenJPH` HTJ2K batch fixture matrix. +pub fn openjph_batch_fixtures() -> &'static [OpenJphBatchFixture] { + OPENJPH_BATCH_FIXTURES +} + +/// `OpenHTJ2K` refinement fixture with a compact output plane. +pub fn openhtj2k_refinement_fixture() -> &'static [u8] { + include_bytes!("../../fixtures/htj2k/openhtj2k_ds0_ht_12_b11.j2k") +} + +/// Expected grayscale pixels for [`openhtj2k_refinement_fixture`]. +pub fn openhtj2k_refinement_pixels() -> &'static [u8] { + include_bytes!("../../fixtures/htj2k/openhtj2k_ds0_ht_12_b11.gray") +} + +/// `OpenHTJ2K` odd refinement fixture used by CUDA plan tests. +pub fn openhtj2k_refinement_odd_fixture() -> &'static [u8] { + include_bytes!("../../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k") +} + +/// Expected grayscale pixels for [`openhtj2k_refinement_odd_fixture`]. +pub fn openhtj2k_refinement_odd_pixels() -> &'static [u8] { + include_bytes!("../../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray") +} + +/// Independent `OpenHTJ2K` RGB12 conformance codestream containing exactly-two-pass jobs. +pub fn openhtj2k_sigprop_fixture() -> &'static [u8] { + include_bytes!("../../fixtures/htj2k/openhtj2k_hifi_ht1_02.j2k") +} + +/// OpenJPH-decoded little-endian RGB12 oracle for [`openhtj2k_sigprop_fixture`]. +pub fn openhtj2k_sigprop_pixels_le() -> &'static [u8] { + include_bytes!("../../fixtures/htj2k/openhtj2k_hifi_ht1_02.oracle.raw") +} + +/// Independent `OpenHTJ2K` stream exercising overlapping SigProp/MagRef refinement bits. +pub fn openhtj2k_sigprop_overlap_fixture() -> &'static [u8] { + include_bytes!("../../fixtures/htj2k/openhtj2k_sigprop_refinement_overlap.j2k") +} + +/// OpenHTJ2K-decoded RGB8 oracle for [`openhtj2k_sigprop_overlap_fixture`]. +pub fn openhtj2k_sigprop_overlap_pixels() -> &'static [u8] { + const PPM_HEADER_LEN: usize = b"P6 512 64 255\n".len(); + &include_bytes!("../../fixtures/htj2k/openhtj2k_sigprop_refinement_overlap.openht.oracle.ppm") + [PPM_HEADER_LEN..] +} diff --git a/crates/j2k-test-support/src/lib.rs b/crates/j2k-test-support/src/lib.rs index cba759dc..cdd460a6 100644 --- a/crates/j2k-test-support/src/lib.rs +++ b/crates/j2k-test-support/src/lib.rs @@ -28,17 +28,21 @@ pub use fixtures::{ baseline_grayscale_jpeg, minimal_baseline_jpeg, minimal_baseline_jpeg_with_restart_interval, minimal_grayscale_jpeg_with_dimensions, minimal_j2k_codestream, minimal_jp2, openhtj2k_refinement_fixture, openhtj2k_refinement_odd_fixture, - openhtj2k_refinement_odd_pixels, openhtj2k_refinement_pixels, restart_coded_grayscale_jpeg, + openhtj2k_refinement_odd_pixels, openhtj2k_refinement_pixels, openhtj2k_sigprop_fixture, + openhtj2k_sigprop_overlap_fixture, openhtj2k_sigprop_overlap_pixels, + openhtj2k_sigprop_pixels_le, openjph_batch_fixtures, restart_coded_grayscale_jpeg, rewrite_j2k_component_sampling, wrap_jp2_codestream, wrap_jp2_rgba_codestream, - JPEG_BASELINE_420_16X16, JPEG_BASELINE_420_16X16_RGB, JPEG_BASELINE_420_RESTART_32X16, - JPEG_BASELINE_420_RESTART_32X16_RGB, JPEG_BASELINE_422_16X8, JPEG_BASELINE_422_16X8_RGB, - JPEG_BASELINE_444_8X8, JPEG_BASELINE_444_8X8_RGB, JPEG_GRAYSCALE_8X8, JPEG_GRAYSCALE_8X8_GRAY, + OpenJphBatchFixture, JPEG_BASELINE_420_16X16, JPEG_BASELINE_420_16X16_RGB, + JPEG_BASELINE_420_RESTART_32X16, JPEG_BASELINE_420_RESTART_32X16_RGB, JPEG_BASELINE_422_16X8, + JPEG_BASELINE_422_16X8_RGB, JPEG_BASELINE_444_8X8, JPEG_BASELINE_444_8X8_RGB, + JPEG_GRAYSCALE_8X8, JPEG_GRAYSCALE_8X8_GRAY, }; #[cfg(feature = "j2k-native-fixtures")] pub use fixtures::{ - classic_j2k_gray8_fixture, htj2k_gray8_97_fixture, htj2k_gray8_fixture, - htj2k_gray8_large_fixture, htj2k_rgb8_97_fixture, htj2k_rgb8_fixture, - htj2k_rgb8_fixture_with_pixels, htj2k_rgb8_pattern_fixture, + classic_j2k_gray8_fixture, generated_htj2k_rgba_fixture, htj2k_gray8_97_fixture, + htj2k_gray8_fixture, htj2k_gray8_large_fixture, htj2k_rgb8_97_fixture, htj2k_rgb8_fixture, + htj2k_rgb8_fixture_with_pixels, htj2k_rgb8_pattern_fixture, Htj2kRgbaAlpha, Htj2kRgbaFixture, + Htj2kRgbaSampleProfile, Htj2kRgbaSamples, }; pub use gpu_gate::{gpu_device_unavailable_is_skip, gpu_test_gate, GPU_TEST_SKIP_MARKER}; /// Convenience prelude for the intentionally broad JPEG fixture catalog. diff --git a/crates/j2k-types/src/decode_payload.rs b/crates/j2k-types/src/decode_payload.rs new file mode 100644 index 00000000..e46418e1 --- /dev/null +++ b/crates/j2k-types/src/decode_payload.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Borrowed-input byte descriptors shared by prepared decode plans. + +/// Byte range inside a complete encoded J2K, JP2, or JPH input. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct J2kCodestreamRange { + /// Zero-based offset from the beginning of the encoded input. + pub offset: usize, + /// Number of bytes in the range. + pub length: usize, +} + +impl J2kCodestreamRange { + /// End offset, or `None` when the range overflows `usize`. + #[must_use] + pub const fn end(self) -> Option { + self.offset.checked_add(self.length) + } +} + +/// Cleanup and optional refinement byte ranges for one HTJ2K code block. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct HtCodeBlockPayloadRanges { + /// Cleanup-pass payload range. + pub cleanup: J2kCodestreamRange, + /// Concatenated SigProp/MagRef payload range, when refinement is present. + pub refinement: Option, +} + +/// Ordered encoded-input fragment span for one classic JPEG 2000 code block. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct J2kClassicCodeBlockPayload { + /// Index of the first fragment in the owning plan's range array. + pub first_range: usize, + /// Number of ordered fragments contributing to this code block. + pub range_count: usize, + /// Total bytes after concatenating every fragment. + pub combined_length: usize, +} + +impl J2kClassicCodeBlockPayload { + /// Exclusive fragment index, or `None` when the descriptor overflows. + #[must_use] + pub const fn end_range(self) -> Option { + self.first_range.checked_add(self.range_count) + } +} diff --git a/crates/j2k-types/src/lib.rs b/crates/j2k-types/src/lib.rs index e889f0be..512bd3ce 100644 --- a/crates/j2k-types/src/lib.rs +++ b/crates/j2k-types/src/lib.rs @@ -15,6 +15,10 @@ extern crate alloc; use alloc::vec::Vec; use core::ops::Range; +mod decode_payload; +pub use decode_payload::{ + HtCodeBlockPayloadRanges, J2kClassicCodeBlockPayload, J2kCodestreamRange, +}; mod limits; #[doc(hidden)] pub use limits::{MAX_JPEG2000_PART1_COMPONENTS, MAX_JPEG2000_PART1_SAMPLE_BIT_DEPTH}; diff --git a/crates/j2k/Cargo.toml b/crates/j2k/Cargo.toml index ddc49ace..da492b98 100644 --- a/crates/j2k/Cargo.toml +++ b/crates/j2k/Cargo.toml @@ -30,7 +30,7 @@ thiserror = { workspace = true } proptest = { workspace = true } criterion = { workspace = true } j2k-native = { path = "../j2k-native", version = "=0.7.4" } -j2k-test-support = { path = "../j2k-test-support" } +j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [[bench]] name = "public_api" diff --git a/crates/j2k/src/adapter/device_plan.rs b/crates/j2k/src/adapter/device_plan.rs index 1b4675ce..2402fa27 100644 --- a/crates/j2k/src/adapter/device_plan.rs +++ b/crates/j2k/src/adapter/device_plan.rs @@ -50,7 +50,7 @@ impl DeviceDecodePlan { DeviceDecodeRequest::RegionScaled { roi, scale } => (roi, scale), }; - if !source_rect.is_within(source_dims) { + if source_rect.w == 0 || source_rect.h == 0 || !source_rect.is_within(source_dims) { return Err(J2kError::InvalidRegion { x: source_rect.x, y: source_rect.y, diff --git a/crates/j2k/src/backend.rs b/crates/j2k/src/backend.rs index 60689d9e..684d27f3 100644 --- a/crates/j2k/src/backend.rs +++ b/crates/j2k/src/backend.rs @@ -1,16 +1,21 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use crate::J2kError; +use crate::{DecodeSettings, J2kError}; use j2k_core::{Colorspace, Info}; -pub(crate) use j2k_native::{ColorSpace, DecodeSettings, DecodedComponents, Image, RawBitmap}; +pub(crate) use j2k_native::{ColorSpace, DecodedComponents, Image, RawBitmap}; -pub(crate) fn image(bytes: &[u8], settings: DecodeSettings) -> Result, J2kError> { - Image::new(bytes, &settings).map_err(J2kError::from_native_decode_error) +pub(crate) fn image( + bytes: &[u8], + settings: DecodeSettings, + target_resolution: Option<(u32, u32)>, +) -> Result, J2kError> { + Image::new(bytes, &settings.to_native(target_resolution)) + .map_err(J2kError::from_native_decode_error) } pub(crate) fn inspect_info(bytes: &[u8]) -> Result { - let image = image(bytes, DecodeSettings::default())?; + let image = image(bytes, DecodeSettings::default(), None)?; Ok(inspect_info_from_image(&image)) } diff --git a/crates/j2k/src/batch.rs b/crates/j2k/src/batch.rs index a64502e7..0ac87e19 100644 --- a/crates/j2k/src/batch.rs +++ b/crates/j2k/src/batch.rs @@ -11,11 +11,11 @@ use j2k_core::{BatchResultSlot, DecodeOutcome, PixelFormat, Rect, TileBatchDecod use crate::{J2kCodec, J2kContext, J2kDecodeWarning, J2kError, J2kScratchPool}; mod admission; -mod allocation; +pub(crate) mod allocation; mod direct; mod planning; -mod scheduler; -mod worker; +pub(crate) mod scheduler; +pub(crate) mod worker; use admission::BatchAllocationBudget; use direct::build_repeated_direct_color_region_plan; diff --git a/crates/j2k/src/batch/allocation.rs b/crates/j2k/src/batch/allocation.rs index 33b3dca4..4d03b842 100644 --- a/crates/j2k/src/batch/allocation.rs +++ b/crates/j2k/src/batch/allocation.rs @@ -15,7 +15,7 @@ use crate::J2kDecodeWarning; /// /// This is a 0.7 safety/performance policy, not an estimate of a particular /// image. Requested concurrency above four is reduced before spawning. -pub(super) const MAX_GENERIC_BATCH_WORKERS: usize = 4; +pub(crate) const MAX_GENERIC_BATCH_WORKERS: usize = 4; /// Maximum number of scheduled region-scaled workers when weighted runtime /// admission is active. At most four worst-case generic claims execute at once; @@ -26,7 +26,7 @@ pub(super) const MAX_ADMITTED_BATCH_WORKERS: usize = MAX_GENERIC_BATCH_WORKERS * /// Fixed ceiling for batch-owned slots, handles, outcomes, and warnings. /// /// This is a facade policy allowance rather than a codec memory requirement. -pub(super) const J2K_BATCH_METADATA_ALLOWANCE_BYTES: usize = 64 * 1024 * 1024; +pub(crate) const J2K_BATCH_METADATA_ALLOWANCE_BYTES: usize = 64 * 1024 * 1024; /// Authoritative worst-case claim for one generic native decoder worker. pub(super) const GENERIC_WORKER_CLAIM_BYTES: usize = j2k_native::DEFAULT_MAX_DECODE_BYTES; diff --git a/crates/j2k/src/batch/direct.rs b/crates/j2k/src/batch/direct.rs index 8a2ca038..db10f091 100644 --- a/crates/j2k/src/batch/direct.rs +++ b/crates/j2k/src/batch/direct.rs @@ -8,9 +8,8 @@ use j2k_native::{ J2kDirectCpuScratch, J2kRect, }; -use crate::backend::DecodeSettings; use crate::decode::{decode_warnings_for_settings, validate_buffer}; -use crate::{J2kError, TileRegionScaledDecodeJob}; +use crate::{DecodeSettings, J2kError, TileRegionScaledDecodeJob}; use super::admission::{BatchAllocationBudget, BatchAllocationClaim}; use super::allocation::GENERIC_WORKER_CLAIM_BYTES; diff --git a/crates/j2k/src/batch/direct/planning.rs b/crates/j2k/src/batch/direct/planning.rs index 791fed26..e61a28cd 100644 --- a/crates/j2k/src/batch/direct/planning.rs +++ b/crates/j2k/src/batch/direct/planning.rs @@ -8,10 +8,10 @@ use j2k_native::{ DecodingError as NativeDecodingError, J2kDirectColorPlan, J2kRect, }; -use crate::backend::{self, DecodeSettings}; +use crate::backend; use crate::decode::validate_region; use crate::parse::parse_image_info; -use crate::J2kError; +use crate::{DecodeSettings, J2kError}; pub(super) fn build_direct_color_region_plan( input: &[u8], @@ -37,13 +37,7 @@ pub(super) fn build_direct_color_region_plan( parsed.info.dimensions.1.div_ceil(scale.denominator()), ); let output_region = roi.scaled_covering(scale); - let image = backend::image( - input, - DecodeSettings { - target_resolution: Some(target_dims), - ..DecodeSettings::default() - }, - )?; + let image = backend::image(input, DecodeSettings::default(), Some(target_dims))?; validate_region(output_region, (image.width(), image.height()))?; let mut native_context = j2k_native::DecoderContext::default(); diff --git a/crates/j2k/src/batch/scheduler.rs b/crates/j2k/src/batch/scheduler.rs index 4785f49e..db69428e 100644 --- a/crates/j2k/src/batch/scheduler.rs +++ b/crates/j2k/src/batch/scheduler.rs @@ -21,6 +21,10 @@ use super::planning::{select_batch_plan, select_direct_batch_plan, BatchPlan}; use super::worker::BatchWorker; use super::{BatchOutcome, J2kBatchResultSlot, TileBatchError}; +mod retained; + +pub(crate) use self::retained::run_retained_chunks; + pub(super) type ScopedWorkerHandle<'scope> = ( usize, std::thread::ScopedJoinHandle<'scope, Result<(), BatchInfrastructureError>>, diff --git a/crates/j2k/src/batch/scheduler/retained.rs b/crates/j2k/src/batch/scheduler/retained.rs new file mode 100644 index 00000000..35e9fa9b --- /dev/null +++ b/crates/j2k/src/batch/scheduler/retained.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Scoped scheduling over worker contexts retained by an owned batch session. + +use j2k_core::{BatchInfrastructureError, TileBatchOptions}; + +use super::{join_batch_workers, plan_batch, ScopedWorkerHandle}; +use crate::batch::allocation::try_vec_with_capacity; +use crate::batch::worker::BatchWorker; + +/// Run disjoint jobs without destroying worker-owned scratch after each call. +pub(crate) fn run_retained_chunks( + workers: &mut [BatchWorker], + jobs: &mut [J], + results: &mut [Option], + options: TileBatchOptions, + decode_chunk: F, +) -> Result<(), BatchInfrastructureError> +where + J: Send, + R: Send, + F: Fn(&mut BatchWorker, &mut [J], &mut [Option]) -> Result<(), BatchInfrastructureError> + + Sync, +{ + if jobs.is_empty() { + return Ok(()); + } + if jobs.len() != results.len() { + return Err(BatchInfrastructureError::MissingResult { + index: results.len().min(jobs.len()), + }); + } + let plan = plan_batch(jobs.len(), options)?; + if workers.len() < plan.worker_count { + return Err(BatchInfrastructureError::WorkerSlotMissing { + worker: workers.len(), + available: workers.len(), + }); + } + if plan.worker_count == 1 { + return decode_chunk(&mut workers[0], jobs, results); + } + + std::thread::scope(|scope| { + let mut handles = try_vec_with_capacity::>( + plan.worker_count, + "J2K retained scoped worker handles", + )?; + let mut spawn_error = None; + for (worker_index, ((worker, job_chunk), result_chunk)) in workers + .iter_mut() + .take(plan.worker_count) + .zip(jobs.chunks_mut(plan.chunk_size)) + .zip(results.chunks_mut(plan.chunk_size)) + .enumerate() + { + if let Ok(handle) = std::thread::Builder::new() + .spawn_scoped(scope, || decode_chunk(worker, job_chunk, result_chunk)) + { + handles.push((worker_index, handle)); + } else { + spawn_error = Some(BatchInfrastructureError::WorkerSpawnFailed { + worker: worker_index, + }); + break; + } + } + join_batch_workers(handles, spawn_error) + }) +} diff --git a/crates/j2k/src/batch/worker.rs b/crates/j2k/src/batch/worker.rs index e9b47096..c5023dca 100644 --- a/crates/j2k/src/batch/worker.rs +++ b/crates/j2k/src/batch/worker.rs @@ -16,13 +16,21 @@ use super::{ }; use crate::{CpuDecodeParallelism, J2kContext, J2kError, J2kScratchPool}; +mod owned; + /// One stack-owned worker. Dynamic native/direct ownership is covered by the /// authoritative per-worker claim in `allocation`; these exact owners are also /// counted structurally in the batch metadata plan. -pub(super) struct BatchWorker { +pub(crate) struct BatchWorker { ctx: J2kContext, pool: J2kScratchPool, direct: DirectWorkerState, + native_workspace: j2k_native::DecoderWorkspace, + prepared_plan_scratch: j2k_native::J2kDirectCpuScratch, + prepared_entropy_workspace: j2k_native::J2kDirectCpuEntropyWorkspace, + preparation_calls: u64, + preparation_worker_reuses: u64, + prepared_plan_decode_calls: u64, allocation_budget: Option>, } @@ -37,6 +45,12 @@ impl BatchWorker { ctx, pool: J2kScratchPool::new(), direct: DirectWorkerState::default(), + native_workspace: j2k_native::DecoderWorkspace::default(), + prepared_plan_scratch: j2k_native::J2kDirectCpuScratch::new(), + prepared_entropy_workspace: j2k_native::J2kDirectCpuEntropyWorkspace::default(), + preparation_calls: 0, + preparation_worker_reuses: 0, + prepared_plan_decode_calls: 0, allocation_budget, } } diff --git a/crates/j2k/src/batch/worker/owned.rs b/crates/j2k/src/batch/worker/owned.rs new file mode 100644 index 00000000..973f3c25 --- /dev/null +++ b/crates/j2k/src/batch/worker/owned.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Owned-batch lifecycle for reusable native worker state. + +use j2k_core::ScratchPool; + +use super::BatchWorker; +use crate::CpuDecodeParallelism; + +impl BatchWorker { + pub(crate) fn new_owned(batch_size: usize) -> Self { + Self::new(batch_size, None) + } + + /// Prepare retained facade context and row scratch for one independent + /// owned batch job while keeping nested native parallelism disabled. + pub(crate) fn prepare_owned_decode(&mut self) -> CpuDecodeParallelism { + self.ctx + .set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + self.pool.reset(); + self.ctx.cpu_decode_parallelism() + } + + pub(crate) fn take_native_workspace(&mut self) -> j2k_native::DecoderWorkspace { + core::mem::take(&mut self.native_workspace) + } + + pub(crate) fn restore_native_workspace(&mut self, workspace: j2k_native::DecoderWorkspace) { + self.native_workspace = workspace; + } + + pub(crate) fn build_prepared_htj2k_plan( + &mut self, + image: &j2k_native::Image<'_>, + output_region: (u32, u32, u32, u32), + ) -> j2k_native::Result { + self.record_preparation_call(); + let workspace = self.take_native_workspace(); + let mut context = j2k_native::DecoderContext::from_workspace(workspace); + context.set_cpu_decode_parallelism(j2k_native::CpuDecodeParallelism::Serial); + let result = + image.build_referenced_htj2k_plan_region_with_context(&mut context, output_region); + self.restore_native_workspace(context.into_workspace()); + result + } + + pub(crate) fn build_prepared_classic_plan( + &mut self, + image: &j2k_native::Image<'_>, + output_region: (u32, u32, u32, u32), + ) -> j2k_native::Result { + self.record_preparation_call(); + let workspace = self.take_native_workspace(); + let mut context = j2k_native::DecoderContext::from_workspace(workspace); + context.set_cpu_decode_parallelism(j2k_native::CpuDecodeParallelism::Serial); + let result = + image.build_referenced_classic_plan_region_with_context(&mut context, output_region); + self.restore_native_workspace(context.into_workspace()); + result + } + + fn record_preparation_call(&mut self) { + if self.preparation_calls != 0 { + self.preparation_worker_reuses = self.preparation_worker_reuses.saturating_add(1); + } + self.preparation_calls = self.preparation_calls.saturating_add(1); + } + + pub(crate) const fn preparation_calls(&self) -> u64 { + self.preparation_calls + } + + pub(crate) const fn preparation_worker_reuses(&self) -> u64 { + self.preparation_worker_reuses + } + + pub(crate) fn native_workspace_stats(&self) -> j2k_native::DecoderWorkspaceStats { + self.native_workspace.stats() + } + + pub(crate) fn execute_prepared_htj2k_plan<'scratch>( + &'scratch mut self, + plan: &j2k_native::J2kReferencedHtj2kPlan, + encoded_input: &[u8], + signed: bool, + ) -> j2k_native::Result> { + self.prepared_plan_decode_calls = self.prepared_plan_decode_calls.saturating_add(1); + j2k_native::execute_referenced_htj2k_plan( + plan, + encoded_input, + signed, + &mut self.prepared_plan_scratch, + ) + } + + pub(crate) fn execute_prepared_classic_plan<'scratch>( + &'scratch mut self, + plan: &j2k_native::J2kReferencedClassicPlan, + encoded_input: &[u8], + signed: bool, + ) -> j2k_native::Result> { + self.prepared_plan_decode_calls = self.prepared_plan_decode_calls.saturating_add(1); + j2k_native::execute_referenced_classic_plan( + plan, + encoded_input, + signed, + &mut self.prepared_plan_scratch, + ) + } + + pub(crate) const fn prepared_plan_decode_calls(&self) -> u64 { + self.prepared_plan_decode_calls + } + + pub(crate) fn prepare_staged_htj2k_plan( + &mut self, + plan: &j2k_native::J2kReferencedHtj2kPlan, + image_scratch: &mut j2k_native::J2kDirectCpuScratch, + ) -> j2k_native::Result<()> { + self.prepared_plan_decode_calls = self.prepared_plan_decode_calls.saturating_add(1); + j2k_native::prepare_referenced_htj2k_staged( + plan, + image_scratch, + &mut self.prepared_entropy_workspace, + ) + } + + pub(crate) fn prepare_staged_classic_plan( + &mut self, + plan: &j2k_native::J2kReferencedClassicPlan, + image_scratch: &mut j2k_native::J2kDirectCpuScratch, + ) -> j2k_native::Result<()> { + self.prepared_plan_decode_calls = self.prepared_plan_decode_calls.saturating_add(1); + j2k_native::prepare_referenced_classic_staged( + plan, + image_scratch, + &mut self.prepared_entropy_workspace, + ) + } + + pub(crate) fn execute_staged_htj2k_job( + &mut self, + plan: &j2k_native::J2kReferencedHtj2kPlan, + index: j2k_native::J2kDirectCodeBlockIndex, + payload_arena: &[u8], + payload: j2k_native::HtCodeBlockPayloadRanges, + image_scratch: &mut j2k_native::J2kDirectCpuScratch, + ) -> j2k_native::Result<()> { + j2k_native::execute_referenced_htj2k_entropy_job( + plan, + index, + payload_arena, + payload, + image_scratch, + &mut self.prepared_entropy_workspace, + ) + } + + pub(crate) fn prepare_staged_htj2k_entropy_workspace( + &mut self, + plan: &j2k_native::J2kReferencedHtj2kPlan, + ) -> j2k_native::Result<()> { + j2k_native::prepare_referenced_htj2k_entropy_workspace( + plan, + &mut self.prepared_entropy_workspace, + ) + } + + pub(crate) fn prepare_staged_classic_entropy_workspace( + &mut self, + plan: &j2k_native::J2kReferencedClassicPlan, + ) -> j2k_native::Result<()> { + j2k_native::prepare_referenced_classic_entropy_workspace( + plan, + &mut self.prepared_entropy_workspace, + ) + } + + pub(crate) fn execute_staged_classic_job( + &mut self, + plan: &j2k_native::J2kReferencedClassicPlan, + index: j2k_native::J2kDirectCodeBlockIndex, + payload_arena: &[u8], + payload: j2k_native::J2kCodestreamRange, + image_scratch: &mut j2k_native::J2kDirectCpuScratch, + ) -> j2k_native::Result<()> { + j2k_native::execute_referenced_classic_entropy_job( + plan, + index, + payload_arena, + payload, + image_scratch, + &mut self.prepared_entropy_workspace, + ) + } + + pub(crate) fn prepared_plan_ht_workspace_bytes(&self) -> usize { + self.prepared_plan_scratch + .retained_ht_workspace_bytes() + .saturating_add(self.prepared_entropy_workspace.retained_ht_bytes()) + } + + pub(crate) fn prepared_plan_classic_workspace_bytes(&self) -> usize { + self.prepared_plan_scratch + .retained_classic_workspace_bytes() + .saturating_add(self.prepared_entropy_workspace.retained_classic_bytes()) + } +} diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index e72328d2..e5209a23 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -5,7 +5,6 @@ use crate::J2kError; use alloc::vec::Vec; use core::fmt; use j2k_core::{validate_strided_output_buffer, DecodeOutcome, PixelFormat, Rect, Unsupported}; -use j2k_native::DecodeSettings; mod component_handoff; pub use component_handoff::{ @@ -13,9 +12,11 @@ pub use component_handoff::{ J2kNativeComponentPlane, }; mod output; +mod settings; use output::{ can_decode_u8_directly, write_components_u8_output, write_u16_output, write_u8_output, }; +pub use settings::DecodeSettings; /// Non-fatal JPEG 2000 decode warning surfaced through decode outcomes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -160,8 +161,8 @@ pub(crate) fn validate_region(roi: Rect, dims: (u32, u32)) -> Result<(), J2kErro #[cfg(test)] mod tests { + use super::DecodeSettings; use super::{decode_warnings_for_settings, J2kDecodeWarning}; - use j2k_native::DecodeSettings; #[test] fn decode_warnings_report_lenient_decode_mode() { diff --git a/crates/j2k/src/decode/settings.rs b/crates/j2k/src/decode/settings.rs new file mode 100644 index 00000000..8041447b --- /dev/null +++ b/crates/j2k/src/decode/settings.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Facade-owned decode validation policy. + +/// Validation policy used by prepared and one-shot J2K batch decoding. +/// +/// Decode geometry belongs to [`crate::DecodeRequest`], so this value does not +/// carry a second target-resolution setting that could conflict with the +/// request. Palette and component mappings are validated by batch preparation; +/// the native decoder always resolves them on non-batch facade paths. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DecodeSettings { + strict: bool, +} + +impl DecodeSettings { + /// Compatibility policy that permits recoverable optional-metadata errors. + #[must_use] + pub const fn lenient() -> Self { + Self { strict: false } + } + + /// Fail-closed validation for malformed codec and container metadata. + #[must_use] + pub const fn strict() -> Self { + Self { strict: true } + } + + /// Whether strict validation is enabled. + #[must_use] + pub const fn is_strict(self) -> bool { + self.strict + } + + /// Whether recoverable optional-metadata errors may be tolerated. + #[must_use] + pub const fn lenient_tolerance_enabled(self) -> bool { + !self.strict + } + + pub(crate) const fn to_native( + self, + target_resolution: Option<(u32, u32)>, + ) -> j2k_native::DecodeSettings { + j2k_native::DecodeSettings { + resolve_palette_indices: true, + strict: self.strict, + target_resolution, + } + } +} + +impl Default for DecodeSettings { + fn default() -> Self { + Self::lenient() + } +} + +#[cfg(test)] +mod tests { + use super::DecodeSettings; + + #[test] + fn facade_policy_preserves_strictness_and_internal_geometry() { + assert!(DecodeSettings::default().lenient_tolerance_enabled()); + assert!(!DecodeSettings::lenient().is_strict()); + assert!(DecodeSettings::strict().is_strict()); + + let native = DecodeSettings::strict().to_native(Some((17, 11))); + assert!(native.resolve_palette_indices); + assert!(native.strict); + assert_eq!(native.target_resolution, Some((17, 11))); + } +} diff --git a/crates/j2k/src/lib.rs b/crates/j2k/src/lib.rs index 45eceae9..c77a99d1 100644 --- a/crates/j2k/src/lib.rs +++ b/crates/j2k/src/lib.rs @@ -17,6 +17,7 @@ mod batch; mod decode; mod encode; mod metadata; +mod owned_batch; mod parallelism; mod recode; mod wrap; @@ -67,8 +68,8 @@ pub use batch::{ }; pub use decode::{ - J2kComponentPlane, J2kDecodeWarning, J2kDecodedColorSpace, J2kDecodedComponents, - J2kDecodedNativeComponents, J2kNativeComponentPlane, + DecodeSettings, J2kComponentPlane, J2kDecodeWarning, J2kDecodedColorSpace, + J2kDecodedComponents, J2kDecodedNativeComponents, J2kNativeComponentPlane, }; pub use parallelism::CpuDecodeParallelism; @@ -79,6 +80,16 @@ pub use metadata::{ J2kPaletteMetadata, J2kSupportInfo, }; +pub use owned_batch::{ + prepare_batch, prepare_batch_from_images, BatchAlpha, BatchCodecRoute, BatchColor, + BatchDecodeOptions, BatchDecoder, BatchErrorStage, BatchGroupInfo, BatchItemError, BatchLayout, + BatchWaveletTransform, ClassicCodeBlockPayload, CpuBatchDecodeResult, CpuBatchDecoder, + CpuBatchGroup, CpuBatchSamples, CpuBatchWorkspaceStats, DecodeRequest, EncodedImage, + Htj2kPayloadRanges, IndexedBatchError, J2kCodestreamRange, NativeSampleType, + NonRepresentableReason, PreparationDepth, PreparedBatch, PreparedBatchGroup, + PreparedClassicPlan, PreparedHtj2kPlan, PreparedImage, +}; + pub use encode::{ encode_j2k_lossless, encode_j2k_lossless_components, encode_j2k_lossless_resident_with_accelerator, encode_j2k_lossless_typed_components, @@ -103,10 +114,10 @@ pub use wrap::{wrap_j2k_codestream, J2kFileBoxMetadata, J2kFileColorSpec, J2kFil pub use parse::{extract_j2k_codestream_payload, J2kCodestreamPayload}; pub use j2k_core::{ - BackendKind, BackendRequest, BufferError, CodecError, CompressedPayloadKind, - CompressedTransferSyntax, DecodeOutcome, DecodeRowsError, Downscale, ImageCodec, ImageDecode, - ImageDecodeRows, PassthroughCandidate, PassthroughDecision, PassthroughRejectReason, - PassthroughRequirements, PixelFormat, Rect, RowSink, TileBatchDecode, + BackendKind, BackendRequest, BatchInfrastructureError, BufferError, CodecError, + CompressedPayloadKind, CompressedTransferSyntax, DecodeOutcome, DecodeRowsError, Downscale, + ImageCodec, ImageDecode, ImageDecodeRows, PassthroughCandidate, PassthroughDecision, + PassthroughRejectReason, PassthroughRequirements, PixelFormat, Rect, RowSink, TileBatchDecode, }; pub(crate) mod parse; diff --git a/crates/j2k/src/owned_batch.rs b/crates/j2k/src/owned_batch.rs new file mode 100644 index 00000000..5b874a15 --- /dev/null +++ b/crates/j2k/src/owned_batch.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Owned, reusable batch contracts for native-width JPEG 2000 decode. + +use alloc::{sync::Arc, vec::Vec}; +use core::{fmt, mem::size_of, num::NonZeroUsize, ops::Range, sync::atomic::Ordering}; +use std::sync::Mutex; + +use j2k_core::{BatchInfrastructureError, Colorspace, CompressedTransferSyntax, Downscale, Rect}; + +pub use j2k_core::{PixelLayout as BatchColor, SampleType as NativeSampleType}; + +use crate::{ + backend, + batch::{ + allocation::{J2K_BATCH_METADATA_ALLOWANCE_BYTES, MAX_GENERIC_BATCH_WORKERS}, + scheduler::run_retained_chunks, + worker::BatchWorker, + }, + decode::decode_warnings_for_settings, + parse::{extract_j2k_codestream_payload, parse_image_info}, + CpuDecodeParallelism, DecodeSettings, DeviceDecodePlan, J2kDecodeWarning, J2kError, + J2kSupportInfo, TileBatchOptions, +}; + +mod contracts; +pub use self::contracts::{ + BatchAlpha, BatchCodecRoute, BatchDecodeOptions, BatchGroupInfo, BatchLayout, + BatchWaveletTransform, DecodeRequest, EncodedImage, PreparationDepth, +}; +use self::contracts::{PrepareImageResult, PrepareJob}; + +mod errors; +pub use self::errors::{ + BatchErrorStage, BatchItemError, IndexedBatchError, NonRepresentableReason, +}; + +mod prepared; +pub use self::prepared::{BatchDecoder, PreparedBatch, PreparedBatchGroup, PreparedImage}; +use self::prepared::{BatchExecutionShape, PreparedCodecPlan, PreparedImageInner}; + +mod prepared_plan; +pub use self::prepared_plan::{ + ClassicCodeBlockPayload, Htj2kPayloadRanges, J2kCodestreamRange, PreparedClassicPlan, + PreparedHtj2kPlan, +}; +mod cpu_prepared; +use self::cpu_prepared::{ + decode_prepared_plan_samples, execute_staged_entropy_job, finish_staged_plan_samples, + finish_staged_tile, prepare_staged_entropy_worker, prepare_staged_image, prepare_staged_tile, + staged_tile_count, +}; +mod cpu_fast; +use self::cpu_fast::CpuGroupFastWorkspace; +mod cpu_staged; +use self::cpu_staged::{CpuEntropyOutcome, CpuStagedWorkspace, CPU_ENTROPY_IMAGES_PER_WINDOW}; +mod cpu_execute; +use self::cpu_execute::decode_cpu_group; +mod cpu; +mod cpu_materialize; +mod cpu_staged_execute; +pub use self::cpu::{CpuBatchDecodeResult, CpuBatchDecoder, CpuBatchGroup, CpuBatchSamples}; +mod cpu_diagnostics; +pub use self::cpu_diagnostics::CpuBatchWorkspaceStats; +mod prepare; +use self::prepare::prepare_batch_with_workers; +#[cfg(test)] +use self::prepare::prepare_staging_bytes; +pub use self::prepare::{prepare_batch, prepare_batch_from_images}; + +#[cfg(test)] +mod tests { + use super::{prepare_staging_bytes, J2K_BATCH_METADATA_ALLOWANCE_BYTES}; + + #[test] + fn preparation_staging_size_rejects_overflowing_input_count() { + assert!(prepare_staging_bytes(usize::MAX).is_err()); + } + + #[test] + fn preparation_staging_size_accepts_its_exact_logical_boundary() { + let bytes_per_input = prepare_staging_bytes(1).expect("one preparation item fits"); + let exact_count = J2K_BATCH_METADATA_ALLOWANCE_BYTES / bytes_per_input; + + assert!(prepare_staging_bytes(exact_count).is_ok()); + assert!(prepare_staging_bytes(exact_count + 1).is_err()); + } +} diff --git a/crates/j2k/src/owned_batch/contracts.rs b/crates/j2k/src/owned_batch/contracts.rs new file mode 100644 index 00000000..aae10c26 --- /dev/null +++ b/crates/j2k/src/owned_batch/contracts.rs @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use alloc::sync::Arc; +use core::num::NonZeroUsize; + +use j2k_core::{ + Colorspace, CompressedPayloadKind, CompressedTransferSyntax, Downscale, PixelFormat, + PixelLayout, Rect, SampleType, +}; + +use super::{BatchExecutionShape, BatchItemError, PreparedImage}; +use crate::{DecodeSettings, DeviceDecodeRequest}; + +/// Amount of native execution planning retained by a prepared image. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum PreparationDepth { + /// Only inspection and geometry are retained. Decoding reparses native + /// packet structure because the current general parser is borrowing. + MetadataOnly, + /// Per-tile Gray/RGB/RGBA HTJ2K packet and code-block geometry is retained; + /// compressed payloads remain `(offset, length)` references into the + /// original [`EncodedImage::bytes`] allocation. + Htj2kOffsetPlan, + /// Per-tile Gray/RGB/RGBA classic JPEG 2000 packet and code-block + /// geometry is retained; compressed fragments remain byte ranges into the + /// original [`EncodedImage::bytes`] allocation. + ClassicOffsetPlan, +} + +/// Requested pixels from one encoded image. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum DecodeRequest { + /// Decode the complete image at its native resolution. + #[default] + Full, + /// Decode a full-resolution source region. + Region { + /// Source-coordinate region of interest. + roi: Rect, + }, + /// Decode the complete image at a reduced resolution. + Reduced { + /// Power-of-two reduction. + scale: Downscale, + }, + /// Decode a source region at a reduced resolution. + RegionReduced { + /// Source-coordinate region of interest. + roi: Rect, + /// Power-of-two reduction. + scale: Downscale, + }, +} + +impl DecodeRequest { + pub(super) fn device_request(self) -> DeviceDecodeRequest { + match self { + Self::Full => DeviceDecodeRequest::Full, + Self::Region { roi } => DeviceDecodeRequest::Region { roi }, + Self::Reduced { scale } => DeviceDecodeRequest::Scaled { scale }, + Self::RegionReduced { roi, scale } => DeviceDecodeRequest::RegionScaled { roi, scale }, + } + } +} + +/// Tensor-compatible channel ordering requested from a batch decoder. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum BatchLayout { + /// Batch, channel, height, width. + #[default] + Nchw, + /// Batch, height, width, channel. + Nhwc, +} + +/// One owned encoded image submitted to a batch decoder. +#[derive(Debug, Clone)] +pub struct EncodedImage { + /// Complete J2K, JP2, or JPH bytes. + pub bytes: Arc<[u8]>, + /// Pixels requested from the image. + pub request: DecodeRequest, +} + +pub(super) struct PrepareJob { + pub(super) source_index: usize, + pub(super) input: Option, +} + +pub(super) type PrepareImageResult = + Result<(PreparedImage, BatchGroupInfo, BatchExecutionShape), BatchItemError>; + +impl EncodedImage { + /// Construct an owned image request without copying `bytes`. + #[must_use] + pub fn new(bytes: Arc<[u8]>, request: DecodeRequest) -> Self { + Self { bytes, request } + } + + /// Construct a full-resolution, full-image request. + #[must_use] + pub fn full(bytes: Arc<[u8]>) -> Self { + Self::new(bytes, DecodeRequest::Full) + } +} + +/// Options shared by preparation and CPU batch decode. +#[derive(Debug, Clone, Copy)] +pub struct BatchDecodeOptions { + /// Output channel ordering. + pub layout: BatchLayout, + /// Native decoder validation settings. The request controls target resolution. + pub settings: DecodeSettings, + /// CPU worker count. `None` uses available parallelism. + pub workers: Option, +} + +impl Default for BatchDecodeOptions { + fn default() -> Self { + Self { + layout: BatchLayout::Nchw, + settings: DecodeSettings::strict(), + workers: None, + } + } +} + +/// Interpretation of an output alpha channel. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum BatchAlpha { + /// The group has no alpha channel. + None, + /// Color samples are independent of straight opacity. + Straight, + /// Color samples are premultiplied by opacity. + Premultiplied, +} + +/// Compressed block-coding route selected during preparation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum BatchCodecRoute { + /// JPEG 2000 Part 1 block coding. + Classic, + /// JPEG 2000 Part 15 high-throughput block coding. + Htj2k, +} + +/// Wavelet transform declared by the compressed syntax. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum BatchWaveletTransform { + /// Reversible integer 5/3 transform. + Reversible53, + /// Irreversible floating-point 9/7 transform. + Irreversible97, +} + +/// Metadata shared by every image in one homogeneous output group. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BatchGroupInfo { + /// Decoded width and height. + pub dimensions: (u32, u32), + /// Number and interpretation of output channels. + pub color: PixelLayout, + /// Alpha interpretation declared by JP2 channel definitions. + pub alpha: BatchAlpha, + /// Exact significant sample precision. + pub precision: u8, + /// Whether the codestream samples are signed. + pub signed: bool, + /// Native Rust/tensor integer type. + pub sample_type: SampleType, + /// Requested output channel ordering. + pub layout: BatchLayout, + /// Parsed color-space declaration. + pub colorspace: Colorspace, + /// Classic or high-throughput block coding. + pub route: BatchCodecRoute, + /// Reversible or irreversible wavelet transform. + pub transform: BatchWaveletTransform, + /// Compressed transfer syntax. + pub transfer_syntax: CompressedTransferSyntax, + /// Raw codestream or still-image wrapper shape. + pub payload_kind: CompressedPayloadKind, +} + +impl BatchGroupInfo { + /// Number of samples in one image in this group. + #[must_use] + pub fn samples_per_image(&self) -> Option { + (self.dimensions.0 as usize) + .checked_mul(self.dimensions.1 as usize)? + .checked_mul(self.color.channels()) + } + + /// Exact native integer pixel format represented by this group. + /// + /// `None` reports a newer color/sample contract or metadata that does not + /// follow the fast-batch width and signedness rules. + #[doc(hidden)] + #[must_use] + pub const fn native_pixel_format(&self) -> Option { + if !matches!( + (self.sample_type, self.precision, self.signed), + (SampleType::U8, 1..=8, false) + | (SampleType::U16, 9..=16, false) + | (SampleType::I16, 1..=16, true) + ) { + return None; + } + match (self.color, self.sample_type) { + (PixelLayout::Gray, SampleType::U8) => Some(PixelFormat::Gray8), + (PixelLayout::Gray, SampleType::U16) => Some(PixelFormat::Gray16), + (PixelLayout::Gray, SampleType::I16) => Some(PixelFormat::GrayI16), + (PixelLayout::Rgb, SampleType::U8) => Some(PixelFormat::Rgb8), + (PixelLayout::Rgb, SampleType::U16) => Some(PixelFormat::Rgb16), + (PixelLayout::Rgb, SampleType::I16) => Some(PixelFormat::RgbI16), + (PixelLayout::Rgba, SampleType::U8) => Some(PixelFormat::Rgba8), + (PixelLayout::Rgba, SampleType::U16) => Some(PixelFormat::Rgba16), + (PixelLayout::Rgba, SampleType::I16) => Some(PixelFormat::RgbaI16), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use j2k_core::{ + Colorspace, CompressedPayloadKind, CompressedTransferSyntax, PixelFormat, PixelLayout, + SampleType, + }; + + use super::{BatchAlpha, BatchCodecRoute, BatchGroupInfo, BatchLayout, BatchWaveletTransform}; + + fn group_info( + color: PixelLayout, + sample_type: SampleType, + precision: u8, + signed: bool, + ) -> BatchGroupInfo { + BatchGroupInfo { + dimensions: (8, 8), + color, + alpha: if color == PixelLayout::Rgba { + BatchAlpha::Straight + } else { + BatchAlpha::None + }, + precision, + signed, + sample_type, + layout: BatchLayout::Nchw, + colorspace: if color == PixelLayout::Gray { + Colorspace::Grayscale + } else { + Colorspace::SRgb + }, + route: BatchCodecRoute::Htj2k, + transform: BatchWaveletTransform::Reversible53, + transfer_syntax: CompressedTransferSyntax::HtJpeg2000Lossless, + payload_kind: CompressedPayloadKind::Jpeg2000Codestream, + } + } + + #[test] + fn native_pixel_format_maps_every_representable_color_and_sample_type() { + for (color, sample_type, precision, signed, expected) in [ + ( + PixelLayout::Gray, + SampleType::U8, + 8, + false, + PixelFormat::Gray8, + ), + ( + PixelLayout::Gray, + SampleType::U16, + 12, + false, + PixelFormat::Gray16, + ), + ( + PixelLayout::Gray, + SampleType::I16, + 12, + true, + PixelFormat::GrayI16, + ), + ( + PixelLayout::Rgb, + SampleType::U8, + 8, + false, + PixelFormat::Rgb8, + ), + ( + PixelLayout::Rgb, + SampleType::U16, + 12, + false, + PixelFormat::Rgb16, + ), + ( + PixelLayout::Rgb, + SampleType::I16, + 12, + true, + PixelFormat::RgbI16, + ), + ( + PixelLayout::Rgba, + SampleType::U8, + 8, + false, + PixelFormat::Rgba8, + ), + ( + PixelLayout::Rgba, + SampleType::U16, + 12, + false, + PixelFormat::Rgba16, + ), + ( + PixelLayout::Rgba, + SampleType::I16, + 12, + true, + PixelFormat::RgbaI16, + ), + ] { + assert_eq!( + group_info(color, sample_type, precision, signed).native_pixel_format(), + Some(expected) + ); + } + } + + #[test] + fn native_pixel_format_rejects_inconsistent_sample_metadata() { + for info in [ + group_info(PixelLayout::Gray, SampleType::U8, 0, false), + group_info(PixelLayout::Gray, SampleType::U8, 9, false), + group_info(PixelLayout::Gray, SampleType::U8, 8, true), + group_info(PixelLayout::Rgb, SampleType::U16, 8, false), + group_info(PixelLayout::Rgb, SampleType::U16, 17, false), + group_info(PixelLayout::Rgba, SampleType::I16, 12, false), + group_info(PixelLayout::Rgba, SampleType::I16, 17, true), + ] { + assert_eq!(info.native_pixel_format(), None); + } + } +} diff --git a/crates/j2k/src/owned_batch/cpu.rs b/crates/j2k/src/owned_batch/cpu.rs new file mode 100644 index 00000000..45f44753 --- /dev/null +++ b/crates/j2k/src/owned_batch/cpu.rs @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Persistent CPU batch session and public native output owners. + +use super::{ + decode_cpu_group, fmt, prepare_batch_from_images, prepare_batch_with_workers, + BatchDecodeOptions, BatchDecoder, BatchGroupInfo, BatchInfrastructureError, BatchWorker, + CpuBatchWorkspaceStats, CpuGroupFastWorkspace, CpuStagedWorkspace, EncodedImage, + IndexedBatchError, J2kDecodeWarning, Mutex, NativeSampleType, NonZeroUsize, PreparedBatch, + PreparedImage, Rect, Vec, MAX_GENERIC_BATCH_WORKERS, +}; + +/// Native-width contiguous samples returned by the CPU batch decoder. +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum CpuBatchSamples { + /// Unsigned samples with precision at most eight bits. + U8(Vec), + /// Unsigned samples with precision from nine through sixteen bits. + U16(Vec), + /// Signed samples with precision at most sixteen bits. + I16(Vec), +} + +impl CpuBatchSamples { + /// Native sample type stored by this owner. + #[must_use] + pub const fn sample_type(&self) -> NativeSampleType { + match self { + Self::U8(_) => NativeSampleType::U8, + Self::U16(_) => NativeSampleType::U16, + Self::I16(_) => NativeSampleType::I16, + } + } + + /// Total number of channel samples across every image. + #[must_use] + pub fn len(&self) -> usize { + match self { + Self::U8(samples) => samples.len(), + Self::U16(samples) => samples.len(), + Self::I16(samples) => samples.len(), + } + } + + /// Whether no channel samples are stored. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// One successfully decoded homogeneous CPU output group. +#[derive(Debug, PartialEq, Eq)] +pub struct CpuBatchGroup { + info: BatchGroupInfo, + source_indices: Vec, + decoded_rects: Vec, + warnings: Vec>, + samples: CpuBatchSamples, +} + +impl CpuBatchGroup { + pub(super) fn new( + info: BatchGroupInfo, + source_indices: Vec, + decoded_rects: Vec, + warnings: Vec>, + samples: CpuBatchSamples, + ) -> Self { + Self { + info, + source_indices, + decoded_rects, + warnings, + samples, + } + } + + /// Shared output metadata. + #[must_use] + pub fn info(&self) -> &BatchGroupInfo { + &self.info + } + + /// Original input indices for the dense batch dimension. + #[must_use] + pub fn source_indices(&self) -> &[usize] { + &self.source_indices + } + + /// Actual decoded rectangle for each output image. + #[must_use] + pub fn decoded_rects(&self) -> &[Rect] { + &self.decoded_rects + } + + /// Non-fatal warnings for each output image. + #[must_use] + pub fn warnings(&self) -> &[Vec] { + &self.warnings + } + + /// Contiguous native-width samples. + #[must_use] + pub const fn samples(&self) -> &CpuBatchSamples { + &self.samples + } + + /// Consume the group into its metadata, indices, rectangles, warnings, and samples. + #[must_use] + pub fn into_parts( + self, + ) -> ( + BatchGroupInfo, + Vec, + Vec, + Vec>, + CpuBatchSamples, + ) { + ( + self.info, + self.source_indices, + self.decoded_rects, + self.warnings, + self.samples, + ) + } +} + +/// CPU batch decode successes and indexed item failures. +#[derive(Debug, PartialEq, Eq)] +pub struct CpuBatchDecodeResult { + groups: Vec, + errors: Vec, +} + +impl CpuBatchDecodeResult { + /// Successfully decoded homogeneous groups. + #[must_use] + pub fn groups(&self) -> &[CpuBatchGroup] { + &self.groups + } + + /// Preparation and decode failures in original input order. + #[must_use] + pub fn errors(&self) -> &[IndexedBatchError] { + &self.errors + } + + /// Consume the result into successful groups and indexed errors. + #[must_use] + pub fn into_parts(self) -> (Vec, Vec) { + (self.groups, self.errors) + } +} + +/// Persistent CPU facade for repeated owned/prepared batch decode. +/// +/// Prepared metadata is borrowed across calls. Decoder settings and worker +/// policy are retained by the session. Each worker owns a lifetime-free native +/// workspace whose decoded-component, Tier-1, and IDWT allocations move +/// through the borrowing context and return to the worker after every group, +/// including across calls. +/// Supported Gray/RGB/RGBA classic and HTJ2K inputs execute their retained +/// per-tile offset plans directly. Metadata-only inputs retain the general +/// decode fallback. +pub struct CpuBatchDecoder { + options: BatchDecodeOptions, + workers: Mutex>, + fast_workspace: CpuGroupFastWorkspace, + staged_workspace: CpuStagedWorkspace, +} + +impl fmt::Debug for CpuBatchDecoder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CpuBatchDecoder") + .field("options", &self.options) + .field("retained_workers", &self.retained_worker_count()) + .finish_non_exhaustive() + } +} + +impl CpuBatchDecoder { + /// Create a persistent CPU batch decoder. + #[must_use] + pub fn new(options: BatchDecodeOptions) -> Self { + let available = std::thread::available_parallelism().map_or(1, NonZeroUsize::get); + let worker_count = options + .workers + .map_or(available, NonZeroUsize::get) + .clamp(1, MAX_GENERIC_BATCH_WORKERS); + let workers = (0..worker_count) + .map(|_| BatchWorker::new_owned(worker_count.max(2))) + .collect(); + Self { + options, + workers: Mutex::new(workers), + fast_workspace: CpuGroupFastWorkspace::default(), + staged_workspace: CpuStagedWorkspace::default(), + } + } + + /// Retained session options. + #[must_use] + pub const fn options(&self) -> BatchDecodeOptions { + self.options + } + + /// Number of reusable worker context/scratch owners retained by this session. + #[must_use] + pub fn retained_worker_count(&self) -> usize { + self.workers + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len() + } + + /// Aggregate native workspace reuse counters for this session. + #[must_use] + pub fn workspace_stats(&self) -> CpuBatchWorkspaceStats { + let mut aggregate = self + .workers + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .fold( + CpuBatchWorkspaceStats::default(), + |mut aggregate, worker| { + aggregate.preparation_calls = aggregate + .preparation_calls + .saturating_add(worker.preparation_calls()); + aggregate.preparation_worker_reuses = aggregate + .preparation_worker_reuses + .saturating_add(worker.preparation_worker_reuses()); + aggregate.prepared_plan_decode_calls = aggregate + .prepared_plan_decode_calls + .saturating_add(worker.prepared_plan_decode_calls()); + aggregate.retained_prepared_plan_ht_workspace_bytes = aggregate + .retained_prepared_plan_ht_workspace_bytes + .saturating_add(worker.prepared_plan_ht_workspace_bytes()); + aggregate.retained_prepared_plan_classic_workspace_bytes = aggregate + .retained_prepared_plan_classic_workspace_bytes + .saturating_add(worker.prepared_plan_classic_workspace_bytes()); + let worker = worker.native_workspace_stats(); + aggregate.decode_calls = + aggregate.decode_calls.saturating_add(worker.decode_calls()); + aggregate.component_owner_reuses = aggregate + .component_owner_reuses + .saturating_add(worker.component_owner_reuses()); + aggregate.tier1_owner_reuses = aggregate + .tier1_owner_reuses + .saturating_add(worker.tier1_owner_reuses()); + aggregate.idwt_owner_reuses = aggregate + .idwt_owner_reuses + .saturating_add(worker.idwt_owner_reuses()); + aggregate.scratch_capacity_retries = aggregate + .scratch_capacity_retries + .saturating_add(worker.scratch_capacity_retries()); + aggregate.retained_component_bytes = aggregate + .retained_component_bytes + .saturating_add(worker.retained_component_bytes()); + aggregate.retained_tier1_bytes = aggregate + .retained_tier1_bytes + .saturating_add(worker.retained_tier1_bytes()); + aggregate.retained_idwt_bytes = aggregate + .retained_idwt_bytes + .saturating_add(worker.retained_idwt_bytes()); + aggregate + }, + ); + let fast = self.fast_workspace.stats(); + aggregate.flattened_group_plans = fast.flattened_group_plans; + aggregate.flattened_payload_jobs = fast.flattened_payload_jobs; + aggregate.flattened_cleanup_jobs = fast.flattened_cleanup_jobs; + aggregate.flattened_sigprop_jobs = fast.flattened_sigprop_jobs; + aggregate.flattened_magref_jobs = fast.flattened_magref_jobs; + aggregate.flattened_classic_jobs = fast.flattened_classic_jobs; + aggregate.entropy_job_dispatches = fast.entropy_job_dispatches; + aggregate.cross_image_entropy_windows = fast.cross_image_entropy_windows; + aggregate.compressed_arena_reuses = fast.compressed_arena_reuses; + aggregate.retained_compressed_arena_bytes = fast.retained_compressed_arena_bytes; + aggregate.output_group_allocations = fast.output_group_allocations; + aggregate.output_compaction_copied_samples = fast.output_compaction_copied_samples; + aggregate + } + + /// Inspect and group owned encoded inputs for repeated decode. + pub fn prepare( + &self, + inputs: Vec, + ) -> Result { + let mut workers = self + .workers + .lock() + .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)?; + prepare_batch_with_workers(inputs, self.options, &mut workers) + } + + /// Regroup caller-supplied prepared images under this session's output policy. + pub fn prepare_prepared_images( + &self, + images: Vec, + ) -> Result { + prepare_batch_from_images(images, self.options) + } + + /// Prepare and decode one batch. + pub fn decode( + &mut self, + inputs: Vec, + ) -> Result { + let prepared = self.prepare(inputs)?; + self.decode_prepared(&prepared) + } + + /// Regroup and decode caller-supplied prepared images without reparsing them. + pub fn decode_prepared_images( + &mut self, + images: Vec, + ) -> Result { + let prepared = self.prepare_prepared_images(images)?; + self.decode_prepared(&prepared) + } + + /// Decode a previously prepared batch without consuming it. + pub fn decode_prepared( + &mut self, + prepared: &PreparedBatch, + ) -> Result { + let workers = self + .workers + .get_mut() + .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)?; + let mut groups = Vec::new(); + let mut errors = prepared.errors.to_vec(); + for group in prepared.groups() { + if let Some(decoded) = decode_cpu_group( + workers, + &mut self.fast_workspace, + &mut self.staged_workspace, + group, + prepared.options, + self.options.workers, + &mut errors, + )? { + groups.push(decoded); + } + } + errors.sort_by_key(|error| error.index); + Ok(CpuBatchDecodeResult { groups, errors }) + } +} + +impl BatchDecoder for CpuBatchDecoder { + type Output = CpuBatchDecodeResult; + type Error = BatchInfrastructureError; + + fn options(&self) -> BatchDecodeOptions { + self.options + } + + fn prepare_batch(&self, inputs: Vec) -> Result { + self.prepare(inputs) + } + + fn prepare_prepared_images( + &self, + images: Vec, + ) -> Result { + CpuBatchDecoder::prepare_prepared_images(self, images) + } + + fn decode_prepared(&mut self, prepared: &PreparedBatch) -> Result { + CpuBatchDecoder::decode_prepared(self, prepared) + } +} diff --git a/crates/j2k/src/owned_batch/cpu_diagnostics.rs b/crates/j2k/src/owned_batch/cpu_diagnostics.rs new file mode 100644 index 00000000..abbaeab6 --- /dev/null +++ b/crates/j2k/src/owned_batch/cpu_diagnostics.rs @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Observable reuse and retained-memory counters for CPU batch sessions. + +/// Aggregate reuse counters for native workspaces retained by a CPU batch session. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CpuBatchWorkspaceStats { + pub(super) preparation_calls: u64, + pub(super) preparation_worker_reuses: u64, + pub(super) decode_calls: u64, + pub(super) prepared_plan_decode_calls: u64, + pub(super) component_owner_reuses: u64, + pub(super) tier1_owner_reuses: u64, + pub(super) idwt_owner_reuses: u64, + pub(super) scratch_capacity_retries: u64, + pub(super) retained_component_bytes: usize, + pub(super) retained_tier1_bytes: usize, + pub(super) retained_idwt_bytes: usize, + pub(super) retained_prepared_plan_classic_workspace_bytes: usize, + pub(super) retained_prepared_plan_ht_workspace_bytes: usize, + pub(super) flattened_group_plans: u64, + pub(super) flattened_payload_jobs: u64, + pub(super) flattened_cleanup_jobs: u64, + pub(super) flattened_sigprop_jobs: u64, + pub(super) flattened_magref_jobs: u64, + pub(super) flattened_classic_jobs: u64, + pub(super) entropy_job_dispatches: u64, + pub(super) cross_image_entropy_windows: u64, + pub(super) compressed_arena_reuses: u64, + pub(super) retained_compressed_arena_bytes: usize, + pub(super) output_group_allocations: u64, + pub(super) output_compaction_copied_samples: u64, +} + +impl CpuBatchWorkspaceStats { + /// Number of image plan preparations performed by retained session workers. + #[must_use] + pub const fn preparation_calls(self) -> u64 { + self.preparation_calls + } + + /// Number of image preparations performed by a worker that had already + /// completed at least one earlier preparation. + #[must_use] + pub const fn preparation_worker_reuses(self) -> u64 { + self.preparation_worker_reuses + } + + /// Number of native image decode calls across retained workers. + #[must_use] + pub const fn decode_calls(self) -> u64 { + self.decode_calls + } + + /// Number of image decodes executed from retained codec geometry without + /// constructing or reparsing a native image decoder. + #[must_use] + pub const fn prepared_plan_decode_calls(self) -> u64 { + self.prepared_plan_decode_calls + } + + /// Number of image decodes that reused an existing component owner. + #[must_use] + pub const fn component_owner_reuses(self) -> u64 { + self.component_owner_reuses + } + + /// Number of image decodes that began with retained Tier-1 owners. + #[must_use] + pub const fn tier1_owner_reuses(self) -> u64 { + self.tier1_owner_reuses + } + + /// Number of image decodes that began with retained IDWT owners. + #[must_use] + pub const fn idwt_owner_reuses(self) -> u64 { + self.idwt_owner_reuses + } + + /// Number of retained-scratch evictions followed by a fresh retry. + #[must_use] + pub const fn scratch_capacity_retries(self) -> u64 { + self.scratch_capacity_retries + } + + /// Total retained decoded-component capacity across workers. + #[must_use] + pub const fn retained_component_bytes(self) -> usize { + self.retained_component_bytes + } + + /// Total retained classic and HT Tier-1 capacity across workers. + #[must_use] + pub const fn retained_tier1_bytes(self) -> usize { + self.retained_tier1_bytes + } + + /// Total retained floating-point and exact-integer IDWT capacity across workers. + #[must_use] + pub const fn retained_idwt_bytes(self) -> usize { + self.retained_idwt_bytes + } + + /// Total HT code-block workspace retained by parse-free prepared-plan workers. + #[must_use] + pub const fn retained_prepared_plan_ht_workspace_bytes(self) -> usize { + self.retained_prepared_plan_ht_workspace_bytes + } + + /// Total classic code-block workspace retained by parse-free prepared-plan workers. + #[must_use] + pub const fn retained_prepared_plan_classic_workspace_bytes(self) -> usize { + self.retained_prepared_plan_classic_workspace_bytes + } + + /// Number of homogeneous prepared groups flattened into one compressed arena. + #[must_use] + pub const fn flattened_group_plans(self) -> u64 { + self.flattened_group_plans + } + + /// Number of source-indexed code-block payload jobs flattened across images. + #[must_use] + pub const fn flattened_payload_jobs(self) -> u64 { + self.flattened_payload_jobs + } + + /// Number of flattened one-pass HT cleanup jobs. + #[must_use] + pub const fn flattened_cleanup_jobs(self) -> u64 { + self.flattened_cleanup_jobs + } + + /// Number of flattened two-pass HT `SigProp` jobs. + #[must_use] + pub const fn flattened_sigprop_jobs(self) -> u64 { + self.flattened_sigprop_jobs + } + + /// Number of flattened three-or-more-pass HT `MagRef` jobs. + #[must_use] + pub const fn flattened_magref_jobs(self) -> u64 { + self.flattened_magref_jobs + } + + /// Number of flattened classic JPEG 2000 code-block jobs. + #[must_use] + pub const fn flattened_classic_jobs(self) -> u64 { + self.flattened_classic_jobs + } + + /// Number of flattened code blocks submitted to retained CPU workers. + #[must_use] + pub const fn entropy_job_dispatches(self) -> u64 { + self.entropy_job_dispatches + } + + /// Number of bounded entropy windows containing more than one image. + #[must_use] + pub const fn cross_image_entropy_windows(self) -> u64 { + self.cross_image_entropy_windows + } + + /// Number of group plans that reused the existing compressed arena capacity. + #[must_use] + pub const fn compressed_arena_reuses(self) -> u64 { + self.compressed_arena_reuses + } + + /// Compressed payload arena capacity retained by the session. + #[must_use] + pub const fn retained_compressed_arena_bytes(self) -> usize { + self.retained_compressed_arena_bytes + } + + /// Number of successful final typed group allocations. + #[must_use] + pub const fn output_group_allocations(self) -> u64 { + self.output_group_allocations + } + + /// Samples moved only to compact successful images after indexed decode failures. + #[must_use] + pub const fn output_compaction_copied_samples(self) -> u64 { + self.output_compaction_copied_samples + } + + /// Total retained lifetime-free decode scratch across workers. + #[must_use] + pub const fn retained_scratch_bytes(self) -> usize { + self.retained_tier1_bytes + .saturating_add(self.retained_idwt_bytes) + } +} diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs new file mode 100644 index 00000000..3acd017b --- /dev/null +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! CPU homogeneous-group execution and final materialization. + +use super::cpu_materialize::{ + convert_u16, convert_u8, decode_image_i16, decode_image_u16, decode_image_u8, + ensure_batch_output_within_cap, round_signed, try_zeroed_vec, +}; +use super::cpu_staged_execute::run_staged_typed_group; +use super::{ + decode_warnings_for_settings, run_retained_chunks, Arc, BatchDecodeOptions, BatchErrorStage, + BatchGroupInfo, BatchInfrastructureError, BatchItemError, BatchWorker, CpuBatchGroup, + CpuBatchSamples, CpuDecodeParallelism, CpuGroupFastWorkspace, CpuStagedWorkspace, + IndexedBatchError, J2kError, NativeSampleType, NonZeroUsize, PreparedBatchGroup, PreparedImage, + TileBatchOptions, Vec, +}; + +#[expect( + clippy::too_many_lines, + reason = "the typed output allocation and one shared decode/compaction boundary remain together so every native sample type follows the same group-level ownership rules" +)] +pub(super) fn decode_cpu_group( + workers: &mut [BatchWorker], + fast_workspace: &mut CpuGroupFastWorkspace, + staged_workspace: &mut CpuStagedWorkspace, + group: &PreparedBatchGroup, + options: BatchDecodeOptions, + workers_option: Option, + errors: &mut Vec, +) -> Result, BatchInfrastructureError> { + let has_flattened_payloads = fast_workspace.prepare_group(group)?; + let samples_per_image = + group + .info + .samples_per_image() + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what: "J2K owned batch output", + requested: usize::MAX, + cap: j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES, + })?; + let sample_count = samples_per_image.checked_mul(group.images.len()).ok_or( + BatchInfrastructureError::AllocationTooLarge { + what: "J2K owned batch output", + requested: usize::MAX, + cap: j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES, + }, + )?; + ensure_batch_output_within_cap(sample_count, group.info.sample_type)?; + + let tile_options = TileBatchOptions::new(workers_option); + + let (samples, successful_slots, copied_samples) = match group.info.sample_type { + NativeSampleType::U8 => { + let mut values = try_zeroed_vec::(sample_count, 0)?; + let (successful_slots, copied_samples) = run_typed_group( + workers, + group, + options, + tile_options, + samples_per_image, + &mut values, + has_flattened_payloads.then_some(&mut *fast_workspace), + staged_workspace, + decode_image_u8, + convert_u8, + errors, + )?; + ( + CpuBatchSamples::U8(values), + successful_slots, + copied_samples, + ) + } + NativeSampleType::U16 => { + let mut values = try_zeroed_vec::(sample_count, 0)?; + let (successful_slots, copied_samples) = run_typed_group( + workers, + group, + options, + tile_options, + samples_per_image, + &mut values, + has_flattened_payloads.then_some(&mut *fast_workspace), + staged_workspace, + decode_image_u16, + convert_u16, + errors, + )?; + ( + CpuBatchSamples::U16(values), + successful_slots, + copied_samples, + ) + } + NativeSampleType::I16 => { + let mut values = try_zeroed_vec::(sample_count, 0)?; + let (successful_slots, copied_samples) = run_typed_group( + workers, + group, + options, + tile_options, + samples_per_image, + &mut values, + has_flattened_payloads.then_some(&mut *fast_workspace), + staged_workspace, + decode_image_i16, + round_signed, + errors, + )?; + ( + CpuBatchSamples::I16(values), + successful_slots, + copied_samples, + ) + } + _ => { + return Err(BatchInfrastructureError::UnsupportedContract { + what: "owned CPU batch sample type", + }); + } + }; + fast_workspace.record_output_group(copied_samples)?; + if successful_slots.is_empty() { + return Ok(None); + } + let source_indices = successful_slots + .iter() + .map(|&slot| group.source_indices[slot]) + .collect(); + let decoded_rects = successful_slots + .iter() + .map(|&slot| group.images[slot].plan().output_rect()) + .collect(); + let warnings = successful_slots + .iter() + .map(|_| decode_warnings_for_settings(options.settings)) + .collect(); + Ok(Some(CpuBatchGroup::new( + group.info.clone(), + source_indices, + decoded_rects, + warnings, + samples, + ))) +} + +pub(super) struct CpuTypedDecodeJob<'image, 'output, T> { + pub(super) slot: usize, + pub(super) image: &'image PreparedImage, + pub(super) output: &'output mut [T], +} + +type CpuDecodeFn = for<'image> fn( + &'image PreparedImage, + BatchDecodeOptions, + &BatchGroupInfo, + CpuDecodeParallelism, + &mut j2k_native::DecoderContext<'image>, + &mut BatchWorker, + &mut [T], +) -> Result<(), J2kError>; + +#[expect( + clippy::too_many_arguments, + reason = "the retained-worker boundary keeps output ownership, decode policy, and indexed error collection explicit" +)] +fn run_typed_group( + workers: &mut [BatchWorker], + group: &PreparedBatchGroup, + options: BatchDecodeOptions, + tile_options: TileBatchOptions, + samples_per_image: usize, + output: &mut Vec, + flattened_payloads: Option<&mut CpuGroupFastWorkspace>, + staged_workspace: &mut CpuStagedWorkspace, + decode: CpuDecodeFn, + convert: fn(f32, u8) -> T, + errors: &mut Vec, +) -> Result<(Vec, usize), BatchInfrastructureError> { + let mut jobs = group + .images + .iter() + .zip(output.chunks_exact_mut(samples_per_image)) + .enumerate() + .map(|(slot, (image, output))| CpuTypedDecodeJob { + slot, + image, + output, + }) + .collect::>(); + let mut results = Vec::with_capacity(jobs.len()); + results.resize_with(jobs.len(), || None); + if let Some(flattened) = flattened_payloads { + run_staged_typed_group( + workers, + group, + tile_options, + &mut jobs, + &mut results, + flattened, + staged_workspace, + convert, + )?; + } else { + run_retained_chunks( + workers, + &mut jobs, + &mut results, + tile_options, + |worker, jobs, results| { + let parallelism = worker.prepare_owned_decode(); + let workspace = worker.take_native_workspace(); + let mut context = j2k_native::DecoderContext::from_workspace(workspace); + for (job, slot) in jobs.iter_mut().zip(results) { + *slot = Some(decode( + job.image, + options, + &group.info, + parallelism, + &mut context, + worker, + job.output, + )); + } + worker.restore_native_workspace(context.into_workspace()); + Ok(()) + }, + )?; + } + + let mut successful_slots = Vec::with_capacity(group.images.len()); + for (slot_index, (source_index, result)) in group + .source_indices + .iter() + .copied() + .zip(results) + .enumerate() + { + match result { + Some(Ok(())) => successful_slots.push(slot_index), + Some(Err(source)) => { + errors.push(IndexedBatchError { + index: source_index, + source: BatchItemError::Codec { + stage: BatchErrorStage::Decode, + source: Arc::new(source), + }, + }); + } + None => { + return Err(BatchInfrastructureError::MissingResult { + index: source_index, + }) + } + } + } + let mut copied_samples = 0usize; + if successful_slots.len() != group.images.len() { + for (destination_slot, &source_slot) in successful_slots.iter().enumerate() { + if destination_slot == source_slot { + continue; + } + let source_start = source_slot * samples_per_image; + let source_end = source_start + samples_per_image; + let destination_start = destination_slot * samples_per_image; + output.copy_within(source_start..source_end, destination_start); + copied_samples = copied_samples.saturating_add(samples_per_image); + } + output.truncate(successful_slots.len() * samples_per_image); + } + Ok((successful_slots, copied_samples)) +} diff --git a/crates/j2k/src/owned_batch/cpu_fast.rs b/crates/j2k/src/owned_batch/cpu_fast.rs new file mode 100644 index 00000000..893cb949 --- /dev/null +++ b/crates/j2k/src/owned_batch/cpu_fast.rs @@ -0,0 +1,493 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Retained cross-image compressed arenas for the CPU prepared fast path. + +use alloc::vec::Vec; +use core::mem::size_of; + +use j2k_core::{BatchInfrastructureError, DEFAULT_MAX_HOST_ALLOCATION_BYTES}; +use j2k_native::{ + HtCodeBlockPayloadRanges, J2kClassicCodeBlockPayload, J2kCodestreamRange, + J2kDirectCodeBlockIndex, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, + J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, +}; + +use super::{BatchCodecRoute, PreparedBatchGroup, PreparedImage}; +use crate::batch::allocation::J2K_BATCH_METADATA_ALLOWANCE_BYTES; + +mod plan; +use self::plan::{ + append_input_range, checked_metadata_bytes, classic_group_requirements, empty_range, ht_bucket, + ht_bucket_index, ht_group_requirements, reserve_reused, visit_classic_jobs, visit_ht_jobs, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CpuPayloadBucket { + Cleanup, + SigProp, + MagRef, + Classic, +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct CpuFlattenedPayloadJob { + pub(super) source_index: usize, + pub(super) image_slot: usize, + pub(super) payload_index: usize, + pub(super) destination_index: usize, + pub(super) block_index: J2kDirectCodeBlockIndex, + bucket: CpuPayloadBucket, + bucket_ordinal: usize, +} + +#[derive(Debug, Clone, Copy, Default)] +struct CpuImagePayloadSpan { + start: usize, +} + +#[derive(Debug, Clone, Copy, Default)] +pub(super) struct CpuFastWorkspaceStats { + pub(super) flattened_group_plans: u64, + pub(super) flattened_payload_jobs: u64, + pub(super) flattened_cleanup_jobs: u64, + pub(super) flattened_sigprop_jobs: u64, + pub(super) flattened_magref_jobs: u64, + pub(super) flattened_classic_jobs: u64, + pub(super) entropy_job_dispatches: u64, + pub(super) cross_image_entropy_windows: u64, + pub(super) compressed_arena_reuses: u64, + pub(super) retained_compressed_arena_bytes: usize, + pub(super) output_group_allocations: u64, + pub(super) output_compaction_copied_samples: u64, +} + +#[derive(Debug, Default)] +pub(super) struct CpuGroupFastWorkspace { + compressed_arena: Vec, + jobs: Vec, + image_spans: Vec, + ht_payloads: Vec, + classic_payloads: Vec, + classic_ranges: Vec, + route: Option, + stats: CpuFastWorkspaceStats, +} + +impl CpuGroupFastWorkspace { + pub(super) fn prepare_group( + &mut self, + group: &PreparedBatchGroup, + ) -> Result { + self.clear_active_plan(); + let flattened = match group.info.route { + BatchCodecRoute::Htj2k + if group + .images + .iter() + .all(|image| image.htj2k_plan().is_some()) => + { + self.prepare_htj2k(group)?; + true + } + BatchCodecRoute::Classic + if group + .images + .iter() + .all(|image| image.classic_plan().is_some()) => + { + self.prepare_classic(group)?; + true + } + BatchCodecRoute::Classic | BatchCodecRoute::Htj2k => false, + }; + if flattened { + self.stats.flattened_group_plans = self.stats.flattened_group_plans.saturating_add(1); + self.stats.flattened_payload_jobs = self + .stats + .flattened_payload_jobs + .saturating_add(self.jobs.len() as u64); + self.record_job_buckets(); + } + Ok(flattened) + } + + pub(super) fn jobs(&self) -> &[CpuFlattenedPayloadJob] { + &self.jobs + } + + pub(super) fn arena(&self) -> &[u8] { + &self.compressed_arena + } + + pub(super) fn route(&self) -> Option { + self.route + } + + pub(super) fn ht_payload( + &self, + job: CpuFlattenedPayloadJob, + ) -> Option { + self.ht_payloads.get(job.destination_index).copied() + } + + pub(super) fn classic_payload_range( + &self, + job: CpuFlattenedPayloadJob, + ) -> Option { + let descriptor = self.classic_payloads.get(job.destination_index)?; + if descriptor.range_count != 1 { + return None; + } + self.classic_ranges.get(job.destination_index).copied() + } + + pub(super) fn record_output_group( + &mut self, + copied_samples: usize, + ) -> Result<(), BatchInfrastructureError> { + self.stats.output_group_allocations = self.stats.output_group_allocations.saturating_add(1); + let copied_samples = u64::try_from(copied_samples).map_err(|_| { + BatchInfrastructureError::AllocationTooLarge { + what: "J2K CPU output compaction diagnostics", + requested: copied_samples, + cap: usize::MAX, + } + })?; + self.stats.output_compaction_copied_samples = self + .stats + .output_compaction_copied_samples + .saturating_add(copied_samples); + Ok(()) + } + + pub(super) fn record_entropy_dispatch(&mut self, jobs: usize, images: usize) { + self.stats.entropy_job_dispatches = self + .stats + .entropy_job_dispatches + .saturating_add(jobs as u64); + if jobs != 0 && images > 1 { + self.stats.cross_image_entropy_windows = + self.stats.cross_image_entropy_windows.saturating_add(1); + } + } + + pub(super) fn stats(&self) -> CpuFastWorkspaceStats { + CpuFastWorkspaceStats { + retained_compressed_arena_bytes: self.compressed_arena.capacity(), + ..self.stats + } + } + + fn prepare_htj2k( + &mut self, + group: &PreparedBatchGroup, + ) -> Result<(), BatchInfrastructureError> { + let (payload_count, payload_bytes) = ht_group_requirements(group)?; + self.prepare_storage::( + group.images.len(), + payload_count, + payload_bytes, + )?; + reserve_reused( + &mut self.ht_payloads, + payload_count, + "J2K CPU flattened HT payload ranges", + )?; + self.ht_payloads.resize( + payload_count, + HtCodeBlockPayloadRanges { + cleanup: empty_range(), + refinement: None, + }, + ); + self.assign_image_spans(group, |image| { + image + .htj2k_plan() + .map_or(0, super::PreparedHtj2kPlan::payload_count) + })?; + + for bucket in [ + CpuPayloadBucket::Cleanup, + CpuPayloadBucket::SigProp, + CpuPayloadBucket::MagRef, + ] { + for (image_slot, image) in group.images.iter().enumerate() { + let plan = image + .htj2k_plan() + .ok_or(BatchInfrastructureError::MissingResult { index: image_slot })?; + let span = self.image_spans[image_slot]; + let mut bucket_ordinals = [0_usize; 3]; + visit_ht_jobs( + plan.native_plan(), + |payload_index, block_index, coding_passes| { + if ht_bucket(coding_passes) == bucket { + let bucket_index = ht_bucket_index(bucket); + let bucket_ordinal = bucket_ordinals[bucket_index]; + bucket_ordinals[bucket_index] = bucket_ordinal.saturating_add(1); + self.jobs.push(CpuFlattenedPayloadJob { + source_index: group.source_indices[image_slot], + image_slot, + payload_index, + destination_index: span.start + payload_index, + block_index, + bucket, + bucket_ordinal, + }); + } + }, + ); + } + } + self.jobs.sort_unstable_by_key(|job| { + ( + ht_bucket_index(job.bucket), + job.bucket_ordinal, + job.image_slot, + ) + }); + if self.jobs.len() != payload_count { + return Err(BatchInfrastructureError::MissingResult { + index: self.jobs.len(), + }); + } + for job in &self.jobs { + if group.source_indices.get(job.image_slot).copied() != Some(job.source_index) { + return Err(BatchInfrastructureError::ResultIndexOutOfBounds { + index: job.source_index, + job_count: group.images.len(), + }); + } + let image = &group.images[job.image_slot]; + let payload = image + .htj2k_plan() + .and_then(|plan| plan.native_plan().payloads().get(job.payload_index)) + .copied() + .ok_or(BatchInfrastructureError::MissingResult { + index: job.source_index, + })?; + let cleanup = append_input_range( + &mut self.compressed_arena, + image, + payload.cleanup, + job.source_index, + )?; + let refinement = payload + .refinement + .map(|range| { + append_input_range(&mut self.compressed_arena, image, range, job.source_index) + }) + .transpose()?; + self.ht_payloads[job.destination_index] = HtCodeBlockPayloadRanges { + cleanup, + refinement, + }; + } + self.finish_group(BatchCodecRoute::Htj2k, payload_bytes) + } + + fn prepare_classic( + &mut self, + group: &PreparedBatchGroup, + ) -> Result<(), BatchInfrastructureError> { + let (payload_count, payload_bytes) = classic_group_requirements(group)?; + self.prepare_storage::<(J2kClassicCodeBlockPayload, J2kCodestreamRange)>( + group.images.len(), + payload_count, + payload_bytes, + )?; + reserve_reused( + &mut self.classic_payloads, + payload_count, + "J2K CPU flattened classic payloads", + )?; + reserve_reused( + &mut self.classic_ranges, + payload_count, + "J2K CPU flattened classic ranges", + )?; + self.classic_payloads.resize( + payload_count, + J2kClassicCodeBlockPayload { + first_range: 0, + range_count: 0, + combined_length: 0, + }, + ); + self.classic_ranges.resize(payload_count, empty_range()); + self.assign_image_spans(group, |image| { + image + .classic_plan() + .map_or(0, super::PreparedClassicPlan::payload_count) + })?; + + for (image_slot, image) in group.images.iter().enumerate() { + let plan = image + .classic_plan() + .ok_or(BatchInfrastructureError::MissingResult { index: image_slot })?; + let span = self.image_spans[image_slot]; + visit_classic_jobs(plan.native_plan(), |payload_index, block_index| { + self.jobs.push(CpuFlattenedPayloadJob { + source_index: group.source_indices[image_slot], + image_slot, + payload_index, + destination_index: span.start + payload_index, + block_index, + bucket: CpuPayloadBucket::Classic, + bucket_ordinal: payload_index, + }); + }); + } + self.jobs + .sort_unstable_by_key(|job| (job.bucket_ordinal, job.image_slot)); + for job in &self.jobs { + let image = &group.images[job.image_slot]; + let plan = image + .classic_plan() + .ok_or(BatchInfrastructureError::MissingResult { + index: job.source_index, + })?; + let payload = plan.native_plan().payloads().get(job.payload_index).ok_or( + BatchInfrastructureError::MissingResult { + index: job.source_index, + }, + )?; + let end_range = + payload + .end_range() + .ok_or(BatchInfrastructureError::ResultIndexOutOfBounds { + index: payload.first_range, + job_count: plan.native_plan().ranges().len(), + })?; + let fragments = plan + .native_plan() + .ranges() + .get(payload.first_range..end_range) + .ok_or(BatchInfrastructureError::ResultIndexOutOfBounds { + index: end_range, + job_count: plan.native_plan().ranges().len(), + })?; + let start = self.compressed_arena.len(); + for range in fragments { + append_input_range(&mut self.compressed_arena, image, *range, job.source_index)?; + } + let length = self.compressed_arena.len() - start; + if length != payload.combined_length { + return Err(BatchInfrastructureError::MissingResult { + index: job.source_index, + }); + } + self.classic_ranges[job.destination_index] = J2kCodestreamRange { + offset: start, + length, + }; + self.classic_payloads[job.destination_index] = J2kClassicCodeBlockPayload { + first_range: job.payload_index, + range_count: 1, + combined_length: length, + }; + } + self.finish_group(BatchCodecRoute::Classic, payload_bytes) + } + + fn prepare_storage( + &mut self, + image_count: usize, + payload_count: usize, + payload_bytes: usize, + ) -> Result<(), BatchInfrastructureError> { + let metadata_bytes = checked_metadata_bytes::(image_count, payload_count)?; + if metadata_bytes > J2K_BATCH_METADATA_ALLOWANCE_BYTES { + return Err(BatchInfrastructureError::AllocationTooLarge { + what: "J2K CPU flattened group metadata", + requested: metadata_bytes, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }); + } + if payload_bytes > DEFAULT_MAX_HOST_ALLOCATION_BYTES { + return Err(BatchInfrastructureError::AllocationTooLarge { + what: "J2K CPU flattened compressed arena", + requested: payload_bytes, + cap: DEFAULT_MAX_HOST_ALLOCATION_BYTES, + }); + } + if payload_bytes != 0 && self.compressed_arena.capacity() >= payload_bytes { + self.stats.compressed_arena_reuses = + self.stats.compressed_arena_reuses.saturating_add(1); + } + reserve_reused( + &mut self.compressed_arena, + payload_bytes, + "J2K CPU flattened compressed arena", + )?; + reserve_reused(&mut self.jobs, payload_count, "J2K CPU flattened jobs")?; + reserve_reused( + &mut self.image_spans, + image_count, + "J2K CPU flattened image spans", + )?; + Ok(()) + } + + fn assign_image_spans( + &mut self, + group: &PreparedBatchGroup, + payload_count: impl Fn(&PreparedImage) -> usize, + ) -> Result<(), BatchInfrastructureError> { + let mut start = 0usize; + for image in &group.images { + let len = payload_count(image); + self.image_spans.push(CpuImagePayloadSpan { start }); + start = start + .checked_add(len) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what: "J2K CPU flattened image spans", + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + })?; + } + Ok(()) + } + + fn finish_group( + &mut self, + route: BatchCodecRoute, + expected_payload_bytes: usize, + ) -> Result<(), BatchInfrastructureError> { + if self.compressed_arena.len() != expected_payload_bytes { + return Err(BatchInfrastructureError::MissingResult { + index: self.compressed_arena.len(), + }); + } + self.route = Some(route); + Ok(()) + } + + fn record_job_buckets(&mut self) { + let (cleanup, sigprop, magref, classic) = self.jobs.iter().fold( + (0_u64, 0_u64, 0_u64, 0_u64), + |(cleanup, sigprop, magref, classic), job| match job.bucket { + CpuPayloadBucket::Cleanup => (cleanup + 1, sigprop, magref, classic), + CpuPayloadBucket::SigProp => (cleanup, sigprop + 1, magref, classic), + CpuPayloadBucket::MagRef => (cleanup, sigprop, magref + 1, classic), + CpuPayloadBucket::Classic => (cleanup, sigprop, magref, classic + 1), + }, + ); + self.stats.flattened_cleanup_jobs = + self.stats.flattened_cleanup_jobs.saturating_add(cleanup); + self.stats.flattened_sigprop_jobs = + self.stats.flattened_sigprop_jobs.saturating_add(sigprop); + self.stats.flattened_magref_jobs = self.stats.flattened_magref_jobs.saturating_add(magref); + self.stats.flattened_classic_jobs = + self.stats.flattened_classic_jobs.saturating_add(classic); + } + + fn clear_active_plan(&mut self) { + self.compressed_arena.clear(); + self.jobs.clear(); + self.image_spans.clear(); + self.ht_payloads.clear(); + self.classic_payloads.clear(); + self.classic_ranges.clear(); + self.route = None; + } +} diff --git a/crates/j2k/src/owned_batch/cpu_fast/plan.rs b/crates/j2k/src/owned_batch/cpu_fast/plan.rs new file mode 100644 index 00000000..51daf4a1 --- /dev/null +++ b/crates/j2k/src/owned_batch/cpu_fast/plan.rs @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! HT/classic payload traversal and checked flattened-plan sizing. + +use super::{ + size_of, BatchInfrastructureError, CpuFlattenedPayloadJob, CpuImagePayloadSpan, + CpuPayloadBucket, J2kCodestreamRange, J2kDirectCodeBlockIndex, J2kDirectGrayscalePlan, + J2kDirectGrayscaleStep, J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, PreparedBatchGroup, + PreparedImage, Vec, DEFAULT_MAX_HOST_ALLOCATION_BYTES, J2K_BATCH_METADATA_ALLOWANCE_BYTES, +}; + +pub(super) fn ht_group_requirements( + group: &PreparedBatchGroup, +) -> Result<(usize, usize), BatchInfrastructureError> { + let mut payload_count = 0usize; + let mut payload_bytes = 0usize; + for (image_slot, image) in group.images.iter().enumerate() { + let plan = image + .htj2k_plan() + .ok_or(BatchInfrastructureError::MissingResult { index: image_slot })?; + let mut job_count = 0usize; + visit_ht_jobs(plan.native_plan(), |_, _, _| job_count += 1); + if job_count != plan.payload_count() { + return Err(BatchInfrastructureError::MissingResult { index: image_slot }); + } + payload_count = checked_add(payload_count, plan.payload_count(), "payload jobs")?; + for payload in plan.native_plan().payloads() { + payload_bytes = checked_add( + payload_bytes, + payload.cleanup.length, + "compressed payload bytes", + )?; + if let Some(refinement) = payload.refinement { + payload_bytes = + checked_add(payload_bytes, refinement.length, "compressed payload bytes")?; + } + } + } + Ok((payload_count, payload_bytes)) +} + +pub(super) fn classic_group_requirements( + group: &PreparedBatchGroup, +) -> Result<(usize, usize), BatchInfrastructureError> { + let mut payload_count = 0usize; + let mut payload_bytes = 0usize; + for (image_slot, image) in group.images.iter().enumerate() { + let plan = image + .classic_plan() + .ok_or(BatchInfrastructureError::MissingResult { index: image_slot })?; + payload_count = checked_add(payload_count, plan.payload_count(), "payload jobs")?; + for payload in plan.native_plan().payloads() { + payload_bytes = checked_add( + payload_bytes, + payload.combined_length, + "compressed payload bytes", + )?; + } + } + Ok((payload_count, payload_bytes)) +} + +pub(super) fn visit_ht_jobs( + plan: &J2kReferencedHtj2kPlan, + mut visit: impl FnMut(usize, J2kDirectCodeBlockIndex, u8), +) { + let mut payload_index = 0usize; + for (tile_index, tile) in plan.tiles().iter().enumerate() { + if let Some(geometry) = tile.grayscale_geometry() { + visit_ht_component_jobs(tile_index, 0, geometry, &mut payload_index, &mut visit); + } else if let Some(geometry) = tile.color_geometry() { + for (component_index, component) in geometry.component_plans.iter().enumerate() { + visit_ht_component_jobs( + tile_index, + component_index, + component, + &mut payload_index, + &mut visit, + ); + } + } else if let Some(geometry) = tile.rgba_geometry() { + for (component_index, component) in geometry.component_plans.iter().enumerate() { + visit_ht_component_jobs( + tile_index, + component_index, + component, + &mut payload_index, + &mut visit, + ); + } + } + } +} + +fn visit_ht_component_jobs( + tile_index: usize, + component_index: usize, + plan: &J2kDirectGrayscalePlan, + payload_index: &mut usize, + visit: &mut impl FnMut(usize, J2kDirectCodeBlockIndex, u8), +) { + for (step_index, step) in plan.steps.iter().enumerate() { + if let J2kDirectGrayscaleStep::HtSubBand(sub_band) = step { + for (code_block, job) in sub_band.jobs.iter().enumerate() { + visit( + *payload_index, + J2kDirectCodeBlockIndex { + tile: tile_index, + component: component_index, + step: step_index, + code_block, + }, + job.number_of_coding_passes, + ); + *payload_index = payload_index.saturating_add(1); + } + } + } +} + +pub(super) fn visit_classic_jobs( + plan: &J2kReferencedClassicPlan, + mut visit: impl FnMut(usize, J2kDirectCodeBlockIndex), +) { + let mut payload_index = 0usize; + for (tile_index, tile) in plan.tiles().iter().enumerate() { + if let Some(geometry) = tile.grayscale_geometry() { + visit_classic_component_jobs(tile_index, 0, geometry, &mut payload_index, &mut visit); + } else if let Some(geometry) = tile.color_geometry() { + for (component_index, component) in geometry.component_plans.iter().enumerate() { + visit_classic_component_jobs( + tile_index, + component_index, + component, + &mut payload_index, + &mut visit, + ); + } + } else if let Some(geometry) = tile.rgba_geometry() { + for (component_index, component) in geometry.component_plans.iter().enumerate() { + visit_classic_component_jobs( + tile_index, + component_index, + component, + &mut payload_index, + &mut visit, + ); + } + } + } +} + +fn visit_classic_component_jobs( + tile_index: usize, + component_index: usize, + plan: &J2kDirectGrayscalePlan, + payload_index: &mut usize, + visit: &mut impl FnMut(usize, J2kDirectCodeBlockIndex), +) { + for (step_index, step) in plan.steps.iter().enumerate() { + if let J2kDirectGrayscaleStep::ClassicSubBand(sub_band) = step { + for code_block in 0..sub_band.jobs.len() { + visit( + *payload_index, + J2kDirectCodeBlockIndex { + tile: tile_index, + component: component_index, + step: step_index, + code_block, + }, + ); + *payload_index = payload_index.saturating_add(1); + } + } + } +} + +pub(super) const fn ht_bucket(coding_passes: u8) -> CpuPayloadBucket { + match coding_passes { + 0 | 1 => CpuPayloadBucket::Cleanup, + 2 => CpuPayloadBucket::SigProp, + _ => CpuPayloadBucket::MagRef, + } +} + +pub(super) const fn ht_bucket_index(bucket: CpuPayloadBucket) -> usize { + match bucket { + CpuPayloadBucket::Cleanup => 0, + CpuPayloadBucket::SigProp => 1, + CpuPayloadBucket::MagRef => 2, + CpuPayloadBucket::Classic => 3, + } +} + +pub(super) fn append_input_range( + arena: &mut Vec, + image: &PreparedImage, + range: J2kCodestreamRange, + source_index: usize, +) -> Result { + let end = range.offset.checked_add(range.length).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: range.offset, + job_count: image.bytes().len(), + }, + )?; + let bytes = image.bytes().get(range.offset..end).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: source_index, + job_count: image.bytes().len(), + }, + )?; + let offset = arena.len(); + arena.extend_from_slice(bytes); + Ok(J2kCodestreamRange { + offset, + length: bytes.len(), + }) +} + +pub(super) fn checked_metadata_bytes( + image_count: usize, + payload_count: usize, +) -> Result { + image_count + .checked_mul(size_of::()) + .and_then(|bytes| { + payload_count + .checked_mul(size_of::() + size_of::()) + .and_then(|payload_bytes| bytes.checked_add(payload_bytes)) + }) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what: "J2K CPU flattened group metadata", + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }) +} + +pub(super) fn checked_add( + current: usize, + additional: usize, + what: &'static str, +) -> Result { + current + .checked_add(additional) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what, + requested: usize::MAX, + cap: DEFAULT_MAX_HOST_ALLOCATION_BYTES, + }) +} + +pub(super) fn reserve_reused( + values: &mut Vec, + required: usize, + what: &'static str, +) -> Result<(), BatchInfrastructureError> { + values.clear(); + if values.capacity() >= required { + return Ok(()); + } + let bytes = required.saturating_mul(size_of::()); + values + .try_reserve_exact(required) + .map_err(|_| BatchInfrastructureError::HostAllocationFailed { what, bytes }) +} + +pub(super) const fn empty_range() -> J2kCodestreamRange { + J2kCodestreamRange { + offset: 0, + length: 0, + } +} diff --git a/crates/j2k/src/owned_batch/cpu_materialize.rs b/crates/j2k/src/owned_batch/cpu_materialize.rs new file mode 100644 index 00000000..4ff72a9b --- /dev/null +++ b/crates/j2k/src/owned_batch/cpu_materialize.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! CPU output allocation, exact sample conversion, and fallback materialization. + +use super::{ + backend, decode_prepared_plan_samples, size_of, BatchDecodeOptions, BatchGroupInfo, + BatchInfrastructureError, BatchLayout, BatchWorker, CpuDecodeParallelism, Downscale, J2kError, + NativeSampleType, PreparedImage, Rect, Vec, +}; + +pub(super) fn ensure_batch_output_within_cap( + samples: usize, + sample_type: NativeSampleType, +) -> Result<(), BatchInfrastructureError> { + let width = match sample_type { + NativeSampleType::U8 => size_of::(), + NativeSampleType::U16 => size_of::(), + NativeSampleType::I16 => size_of::(), + _ => { + return Err(BatchInfrastructureError::UnsupportedContract { + what: "owned CPU batch sample width", + }); + } + }; + let bytes = samples + .checked_mul(width) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what: "J2K owned batch output", + requested: usize::MAX, + cap: j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES, + })?; + if bytes > j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES { + return Err(BatchInfrastructureError::AllocationTooLarge { + what: "J2K owned batch output", + requested: bytes, + cap: j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES, + }); + } + Ok(()) +} + +pub(super) fn try_zeroed_vec( + len: usize, + zero: T, +) -> Result, BatchInfrastructureError> { + let bytes = len.saturating_mul(size_of::()); + let mut values = Vec::new(); + values + .try_reserve_exact(len) + .map_err(|_| BatchInfrastructureError::HostAllocationFailed { + what: "J2K owned batch output", + bytes, + })?; + values.resize(len, zero); + Ok(values) +} + +fn image_components<'ctx, 'input>( + image: &'input PreparedImage, + options: BatchDecodeOptions, + parallelism: CpuDecodeParallelism, + context: &'ctx mut j2k_native::DecoderContext<'input>, +) -> Result, J2kError> { + context.set_cpu_decode_parallelism(parallelism.to_native()); + let plan = image.plan(); + let target_resolution = (plan.scale() != Downscale::None).then_some(( + plan.source_dims().0.div_ceil(plan.scale().denominator()), + plan.source_dims().1.div_ceil(plan.scale().denominator()), + )); + let decoded = backend::image(image.bytes(), options.settings, target_resolution)?; + let output_rect = plan.output_rect(); + let decoded_dims = (decoded.width(), decoded.height()); + if output_rect == Rect::full(decoded_dims) { + decoded + .decode_components_with_context(context) + .map_err(J2kError::from_native_decode_error) + } else { + decoded + .decode_region_components_with_context( + (output_rect.x, output_rect.y, output_rect.w, output_rect.h), + context, + ) + .map_err(J2kError::from_native_decode_error) + } +} + +pub(super) fn decode_image_u8<'image>( + image: &'image PreparedImage, + options: BatchDecodeOptions, + info: &BatchGroupInfo, + parallelism: CpuDecodeParallelism, + context: &mut j2k_native::DecoderContext<'image>, + worker: &mut BatchWorker, + out: &mut [u8], +) -> Result<(), J2kError> { + if decode_prepared_plan_samples(image, info, worker, out, |sample, precision| { + u8::try_from(round_unsigned(sample, precision)).unwrap_or(u8::MAX) + })? { + return Ok(()); + } + let components = image_components(image, options, parallelism, context)?; + validate_decoded_components(&components, info)?; + write_samples(&components, info, out, |sample, precision| { + u8::try_from(round_unsigned(sample, precision)).unwrap_or(u8::MAX) + }); + Ok(()) +} + +pub(super) fn decode_image_u16<'image>( + image: &'image PreparedImage, + options: BatchDecodeOptions, + info: &BatchGroupInfo, + parallelism: CpuDecodeParallelism, + context: &mut j2k_native::DecoderContext<'image>, + worker: &mut BatchWorker, + out: &mut [u16], +) -> Result<(), J2kError> { + if decode_prepared_plan_samples(image, info, worker, out, |sample, precision| { + u16::try_from(round_unsigned(sample, precision)).unwrap_or(u16::MAX) + })? { + return Ok(()); + } + let components = image_components(image, options, parallelism, context)?; + validate_decoded_components(&components, info)?; + write_samples(&components, info, out, |sample, precision| { + u16::try_from(round_unsigned(sample, precision)).unwrap_or(u16::MAX) + }); + Ok(()) +} + +pub(super) fn decode_image_i16<'image>( + image: &'image PreparedImage, + options: BatchDecodeOptions, + info: &BatchGroupInfo, + parallelism: CpuDecodeParallelism, + context: &mut j2k_native::DecoderContext<'image>, + worker: &mut BatchWorker, + out: &mut [i16], +) -> Result<(), J2kError> { + if decode_prepared_plan_samples(image, info, worker, out, round_signed)? { + return Ok(()); + } + let components = image_components(image, options, parallelism, context)?; + validate_decoded_components(&components, info)?; + write_samples(&components, info, out, round_signed); + Ok(()) +} + +fn validate_decoded_components( + components: &j2k_native::DecodedComponents<'_>, + info: &BatchGroupInfo, +) -> Result<(), J2kError> { + if components.dimensions() != info.dimensions + || components.planes().len() != info.color.channels() + { + return Err(J2kError::internal_backend( + "prepared batch output metadata changed during decode", + )); + } + let expected = (info.dimensions.0 as usize) + .checked_mul(info.dimensions.1 as usize) + .ok_or_else(|| J2kError::internal_backend("prepared batch sample count overflow"))?; + for (component, plane) in components.planes().iter().enumerate() { + if plane.dimensions() != info.dimensions || plane.samples().len() < expected { + return Err(J2kError::BackendComponentPlaneTooShort { + component, + samples: plane.samples().len(), + expected, + }); + } + } + Ok(()) +} + +fn write_samples( + components: &j2k_native::DecodedComponents<'_>, + info: &BatchGroupInfo, + out: &mut [T], + convert: impl Fn(f32, u8) -> T, +) { + let pixels = info.dimensions.0 as usize * info.dimensions.1 as usize; + let channels = info.color.channels(); + for pixel in 0..pixels { + for (channel, plane) in components.planes().iter().enumerate() { + let destination = match info.layout { + BatchLayout::Nchw => channel * pixels + pixel, + BatchLayout::Nhwc => pixel * channels + channel, + }; + out[destination] = convert(plane.samples()[pixel], info.precision); + } + } +} + +#[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "sample is rounded only after clamping to the declared unsigned precision" +)] +fn round_unsigned(sample: f32, precision: u8) -> u32 { + let max = (1_u32 << u32::from(precision)) - 1; + if sample.is_nan() || sample <= 0.0 { + 0 + } else if f64::from(sample) >= f64::from(max) { + max + } else { + (f64::from(sample) + 0.5) as u32 + } +} + +pub(super) fn convert_u8(sample: f32, precision: u8) -> u8 { + u8::try_from(round_unsigned(sample, precision)).unwrap_or(u8::MAX) +} + +pub(super) fn convert_u16(sample: f32, precision: u8) -> u16 { + u16::try_from(round_unsigned(sample, precision)).unwrap_or(u16::MAX) +} + +#[expect( + clippy::cast_possible_truncation, + reason = "sample is rounded only after clamping to the declared signed precision" +)] +pub(super) fn round_signed(sample: f32, precision: u8) -> i16 { + let magnitude_bits = u32::from(precision.saturating_sub(1)); + let min = -(1_i32 << magnitude_bits); + let max = (1_i32 << magnitude_bits) - 1; + let sample = f64::from(sample); + let rounded = if sample.is_nan() { + 0 + } else if sample <= f64::from(min) { + min + } else if sample >= f64::from(max) { + max + } else if sample >= 0.0 { + (sample + 0.5) as i32 + } else { + (sample - 0.5) as i32 + }; + i16::try_from(rounded).unwrap_or(if rounded.is_negative() { + i16::MIN + } else { + i16::MAX + }) +} diff --git a/crates/j2k/src/owned_batch/cpu_prepared.rs b/crates/j2k/src/owned_batch/cpu_prepared.rs new file mode 100644 index 00000000..5d3ab539 --- /dev/null +++ b/crates/j2k/src/owned_batch/cpu_prepared.rs @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Parse-free prepared-plan execution and dense CPU final store. + +use j2k_core::Rect; + +use super::cpu_fast::{CpuFlattenedPayloadJob, CpuGroupFastWorkspace}; +use super::{BatchCodecRoute, BatchGroupInfo, BatchLayout, PreparedCodecPlan, PreparedImage}; +use crate::{batch::worker::BatchWorker, J2kError}; + +pub(super) fn decode_prepared_plan_samples( + image: &PreparedImage, + info: &BatchGroupInfo, + worker: &mut BatchWorker, + out: &mut [T], + convert: impl Fn(f32, u8) -> T, +) -> Result { + let components = match image.codec_plan() { + PreparedCodecPlan::Htj2k(prepared_plan) => worker + .execute_prepared_htj2k_plan(prepared_plan.native_plan(), image.bytes(), info.signed) + .map_err(J2kError::from_native_decode_error)?, + PreparedCodecPlan::Classic(prepared_plan) => worker + .execute_prepared_classic_plan(prepared_plan.native_plan(), image.bytes(), info.signed) + .map_err(J2kError::from_native_decode_error)?, + PreparedCodecPlan::MetadataOnly => return Ok(false), + }; + let output_rect = image.plan().output_rect(); + validate_components(&components, output_rect, info, out.len())?; + write_samples(&components, output_rect, info, out, convert)?; + Ok(true) +} + +pub(super) fn prepare_staged_image( + image: &PreparedImage, + route: BatchCodecRoute, + worker: &mut BatchWorker, + scratch: &mut j2k_native::J2kDirectCpuScratch, +) -> Result<(), J2kError> { + match (route, image.codec_plan()) { + (BatchCodecRoute::Htj2k, PreparedCodecPlan::Htj2k(plan)) => worker + .prepare_staged_htj2k_plan(plan.native_plan(), scratch) + .map_err(J2kError::from_native_decode_error), + (BatchCodecRoute::Classic, PreparedCodecPlan::Classic(plan)) => worker + .prepare_staged_classic_plan(plan.native_plan(), scratch) + .map_err(J2kError::from_native_decode_error), + _ => Err(J2kError::internal_backend( + "staged CPU route does not match its prepared codec plan", + )), + } +} + +pub(super) fn execute_staged_entropy_job( + image: &PreparedImage, + job: CpuFlattenedPayloadJob, + flattened: &CpuGroupFastWorkspace, + worker: &mut BatchWorker, + scratch: &mut j2k_native::J2kDirectCpuScratch, +) -> Result<(), J2kError> { + match (flattened.route(), image.codec_plan()) { + (Some(BatchCodecRoute::Htj2k), PreparedCodecPlan::Htj2k(plan)) => { + let payload = flattened.ht_payload(job).ok_or_else(|| { + J2kError::internal_backend("staged HT payload descriptor is missing") + })?; + worker + .execute_staged_htj2k_job( + plan.native_plan(), + job.block_index, + flattened.arena(), + payload, + scratch, + ) + .map_err(J2kError::from_native_decode_error) + } + (Some(BatchCodecRoute::Classic), PreparedCodecPlan::Classic(plan)) => { + let payload = flattened.classic_payload_range(job).ok_or_else(|| { + J2kError::internal_backend("staged classic payload descriptor is missing") + })?; + worker + .execute_staged_classic_job( + plan.native_plan(), + job.block_index, + flattened.arena(), + payload, + scratch, + ) + .map_err(J2kError::from_native_decode_error) + } + _ => Err(J2kError::internal_backend( + "staged CPU route does not match its prepared codec plan", + )), + } +} + +pub(super) fn prepare_staged_entropy_worker( + image: &PreparedImage, + route: BatchCodecRoute, + worker: &mut BatchWorker, +) -> Result<(), J2kError> { + match (route, image.codec_plan()) { + (BatchCodecRoute::Htj2k, PreparedCodecPlan::Htj2k(plan)) => worker + .prepare_staged_htj2k_entropy_workspace(plan.native_plan()) + .map_err(J2kError::from_native_decode_error), + (BatchCodecRoute::Classic, PreparedCodecPlan::Classic(plan)) => worker + .prepare_staged_classic_entropy_workspace(plan.native_plan()) + .map_err(J2kError::from_native_decode_error), + _ => Err(J2kError::internal_backend( + "staged CPU worker route does not match its prepared codec plan", + )), + } +} + +pub(super) fn staged_tile_count( + image: &PreparedImage, + route: BatchCodecRoute, +) -> Result { + match (route, image.codec_plan()) { + (BatchCodecRoute::Htj2k, PreparedCodecPlan::Htj2k(plan)) => { + Ok(plan.native_plan().tiles().len()) + } + (BatchCodecRoute::Classic, PreparedCodecPlan::Classic(plan)) => { + Ok(plan.native_plan().tiles().len()) + } + _ => Err(J2kError::internal_backend( + "staged CPU route does not match its prepared codec plan", + )), + } +} + +pub(super) fn prepare_staged_tile( + image: &PreparedImage, + route: BatchCodecRoute, + tile_index: usize, + scratch: &mut j2k_native::J2kDirectCpuScratch, +) -> Result<(), J2kError> { + match (route, image.codec_plan()) { + (BatchCodecRoute::Htj2k, PreparedCodecPlan::Htj2k(plan)) => { + j2k_native::prepare_referenced_htj2k_tile_staged( + plan.native_plan(), + tile_index, + scratch, + ) + .map_err(J2kError::from_native_decode_error) + } + (BatchCodecRoute::Classic, PreparedCodecPlan::Classic(plan)) => { + j2k_native::prepare_referenced_classic_tile_staged( + plan.native_plan(), + tile_index, + scratch, + ) + .map_err(J2kError::from_native_decode_error) + } + _ => Err(J2kError::internal_backend( + "staged CPU route does not match its prepared codec plan", + )), + } +} + +pub(super) fn finish_staged_tile( + image: &PreparedImage, + route: BatchCodecRoute, + tile_index: usize, + signed: bool, + scratch: &mut j2k_native::J2kDirectCpuScratch, +) -> Result<(), J2kError> { + match (route, image.codec_plan()) { + (BatchCodecRoute::Htj2k, PreparedCodecPlan::Htj2k(plan)) => { + j2k_native::finish_referenced_htj2k_tile_staged( + plan.native_plan(), + tile_index, + signed, + scratch, + ) + .map_err(J2kError::from_native_decode_error) + } + (BatchCodecRoute::Classic, PreparedCodecPlan::Classic(plan)) => { + j2k_native::finish_referenced_classic_tile_staged( + plan.native_plan(), + tile_index, + signed, + scratch, + ) + .map_err(J2kError::from_native_decode_error) + } + _ => Err(J2kError::internal_backend( + "staged CPU route does not match its prepared codec plan", + )), + } +} + +pub(super) fn finish_staged_plan_samples( + image: &PreparedImage, + info: &BatchGroupInfo, + scratch: &mut j2k_native::J2kDirectCpuScratch, + out: &mut [T], + convert: impl Fn(f32, u8) -> T, +) -> Result<(), J2kError> { + let components = match image.codec_plan() { + PreparedCodecPlan::Htj2k(plan) => { + j2k_native::finish_referenced_htj2k_staged(plan.native_plan(), info.signed, scratch) + .map_err(J2kError::from_native_decode_error)? + } + PreparedCodecPlan::Classic(plan) => { + j2k_native::finish_referenced_classic_staged(plan.native_plan(), info.signed, scratch) + .map_err(J2kError::from_native_decode_error)? + } + PreparedCodecPlan::MetadataOnly => { + return Err(J2kError::internal_backend( + "staged CPU finish has no prepared codec plan", + )); + } + }; + let output_rect = image.plan().output_rect(); + validate_components(&components, output_rect, info, out.len())?; + write_samples(&components, output_rect, info, out, convert) +} + +fn validate_components( + components: &j2k_native::J2kDirectDecodedComponents<'_>, + output_rect: Rect, + info: &BatchGroupInfo, + output_len: usize, +) -> Result<(), J2kError> { + if components.component_count() != info.color.channels() + || (output_rect.w, output_rect.h) != info.dimensions + || components.dimensions() != info.dimensions + || info.samples_per_image() != Some(output_len) + { + return Err(J2kError::internal_backend( + "prepared codec plan output metadata changed during decode", + )); + } + let plane_len = (components.dimensions().0 as usize) + .checked_mul(components.dimensions().1 as usize) + .ok_or_else(|| J2kError::internal_backend("prepared HTJ2K plane size overflow"))?; + for component in 0..components.component_count() { + let plane = components.plane(component).ok_or_else(|| { + J2kError::internal_backend("prepared codec component plane is missing") + })?; + if plane.dimensions() != components.dimensions() + || plane.bit_depth() != info.precision + || plane.samples().len() < plane_len + { + return Err(J2kError::BackendComponentPlaneTooShort { + component, + samples: plane.samples().len(), + expected: plane_len, + }); + } + } + Ok(()) +} + +fn write_samples( + components: &j2k_native::J2kDirectDecodedComponents<'_>, + output_rect: Rect, + info: &BatchGroupInfo, + out: &mut [T], + convert: impl Fn(f32, u8) -> T, +) -> Result<(), J2kError> { + let output_width = output_rect.w as usize; + let output_height = output_rect.h as usize; + let source_width = components.dimensions().0 as usize; + let channels = components.component_count(); + let output_pixels = output_width * output_height; + for y in 0..output_height { + for x in 0..output_width { + let output_pixel = y * output_width + x; + let source_pixel = y * source_width + x; + for channel in 0..channels { + let plane = components.plane(channel).ok_or_else(|| { + J2kError::internal_backend("prepared codec component plane is missing") + })?; + let destination = match info.layout { + BatchLayout::Nchw => channel * output_pixels + output_pixel, + BatchLayout::Nhwc => output_pixel * channels + channel, + }; + out[destination] = convert(plane.samples()[source_pixel], plane.bit_depth()); + } + } + } + Ok(()) +} diff --git a/crates/j2k/src/owned_batch/cpu_staged.rs b/crates/j2k/src/owned_batch/cpu_staged.rs new file mode 100644 index 00000000..47a16a53 --- /dev/null +++ b/crates/j2k/src/owned_batch/cpu_staged.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Bounded retained owners for staged cross-image CPU entropy dispatch. + +use alloc::vec::Vec; +use core::{mem::size_of, ops::Range, sync::atomic::AtomicBool}; +use std::sync::Mutex; + +use j2k_core::BatchInfrastructureError; + +use super::cpu_fast::{CpuFlattenedPayloadJob, CpuGroupFastWorkspace}; +use crate::batch::allocation::{J2K_BATCH_METADATA_ALLOWANCE_BYTES, MAX_GENERIC_BATCH_WORKERS}; +use crate::J2kError; + +pub(super) const CPU_ENTROPY_IMAGES_PER_WINDOW: usize = MAX_GENERIC_BATCH_WORKERS * 2; + +#[derive(Debug, Default)] +pub(super) struct CpuStagedWorkspace { + image_scratch: Vec>, + failed: Vec, + entropy_jobs: Vec, + entropy_results: Vec>, +} + +#[derive(Debug)] +pub(super) enum CpuEntropyOutcome { + Complete, + Error(J2kError), +} + +pub(super) struct CpuStagedExecution<'a> { + pub(super) image_scratch: &'a [Mutex], + pub(super) failed: &'a [AtomicBool], + pub(super) jobs: &'a mut [CpuFlattenedPayloadJob], + pub(super) results: &'a mut [Option], +} + +impl CpuStagedWorkspace { + pub(super) fn prepare_window( + &mut self, + _flattened: &CpuGroupFastWorkspace, + image_range: Range, + ) -> Result<(), BatchInfrastructureError> { + let image_count = image_range.len(); + if image_count == 0 || image_count > CPU_ENTROPY_IMAGES_PER_WINDOW { + return Err(BatchInfrastructureError::ResultIndexOutOfBounds { + index: image_count, + job_count: CPU_ENTROPY_IMAGES_PER_WINDOW, + }); + } + ensure_metadata_capacity(image_count, 0)?; + reserve_for_len( + &mut self.image_scratch, + image_count, + "J2K staged CPU image scratch owners", + )?; + while self.image_scratch.len() < image_count { + self.image_scratch + .push(Mutex::new(j2k_native::J2kDirectCpuScratch::new())); + } + self.image_scratch.truncate(image_count); + + reserve_for_len( + &mut self.failed, + image_count, + "J2K staged CPU failure flags", + )?; + while self.failed.len() < image_count { + self.failed.push(AtomicBool::new(false)); + } + self.failed.truncate(image_count); + for failed in &self.failed { + failed.store(false, core::sync::atomic::Ordering::Relaxed); + } + + self.entropy_jobs.clear(); + self.entropy_results.clear(); + Ok(()) + } + + pub(super) fn prepare_tile_jobs( + &mut self, + flattened: &CpuGroupFastWorkspace, + image_range: Range, + tile_index: usize, + ) -> Result<(), BatchInfrastructureError> { + let entropy_count = flattened + .jobs() + .iter() + .filter(|job| { + image_range.contains(&job.image_slot) && job.block_index.tile == tile_index + }) + .count(); + ensure_metadata_capacity(image_range.len(), entropy_count)?; + reserve_for_len( + &mut self.entropy_jobs, + entropy_count, + "J2K staged CPU entropy jobs", + )?; + self.entropy_jobs.clear(); + self.entropy_jobs.extend( + flattened + .jobs() + .iter() + .filter(|job| { + image_range.contains(&job.image_slot) && job.block_index.tile == tile_index + }) + .copied(), + ); + + reserve_for_len( + &mut self.entropy_results, + entropy_count, + "J2K staged CPU entropy results", + )?; + self.entropy_results.clear(); + self.entropy_results.resize_with(entropy_count, || None); + Ok(()) + } + + pub(super) fn execution(&mut self) -> CpuStagedExecution<'_> { + CpuStagedExecution { + image_scratch: &self.image_scratch, + failed: &self.failed, + jobs: &mut self.entropy_jobs, + results: &mut self.entropy_results, + } + } +} + +fn ensure_metadata_capacity( + image_count: usize, + entropy_count: usize, +) -> Result<(), BatchInfrastructureError> { + let image_bytes = image_count + .checked_mul(size_of::>() + size_of::()) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what: "J2K staged CPU metadata", + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + })?; + let entropy_bytes = entropy_count + .checked_mul(size_of::() + size_of::>()) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what: "J2K staged CPU metadata", + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + })?; + let requested = image_bytes.checked_add(entropy_bytes).ok_or( + BatchInfrastructureError::AllocationTooLarge { + what: "J2K staged CPU metadata", + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }, + )?; + if requested > J2K_BATCH_METADATA_ALLOWANCE_BYTES { + return Err(BatchInfrastructureError::AllocationTooLarge { + what: "J2K staged CPU metadata", + requested, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }); + } + Ok(()) +} + +fn reserve_for_len( + values: &mut Vec, + required: usize, + what: &'static str, +) -> Result<(), BatchInfrastructureError> { + if values.capacity() >= required { + return Ok(()); + } + let additional = required.saturating_sub(values.len()); + let bytes = required.saturating_mul(size_of::()); + values + .try_reserve_exact(additional) + .map_err(|_| BatchInfrastructureError::HostAllocationFailed { what, bytes }) +} diff --git a/crates/j2k/src/owned_batch/cpu_staged_execute.rs b/crates/j2k/src/owned_batch/cpu_staged_execute.rs new file mode 100644 index 00000000..9de165c2 --- /dev/null +++ b/crates/j2k/src/owned_batch/cpu_staged_execute.rs @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Cross-image staged CPU entropy and tile execution. + +use super::cpu_execute::CpuTypedDecodeJob; +use super::{ + execute_staged_entropy_job, finish_staged_plan_samples, finish_staged_tile, + prepare_staged_entropy_worker, prepare_staged_image, prepare_staged_tile, run_retained_chunks, + staged_tile_count, BatchCodecRoute, BatchInfrastructureError, BatchWorker, CpuEntropyOutcome, + CpuGroupFastWorkspace, CpuStagedWorkspace, J2kError, Ordering, PreparedBatchGroup, Range, + TileBatchOptions, CPU_ENTROPY_IMAGES_PER_WINDOW, +}; + +#[expect( + clippy::too_many_arguments, + clippy::too_many_lines, + reason = "the staged scheduler keeps retained workers, image owners, flattened payloads, and dense output slots explicit" +)] +pub(super) fn run_staged_typed_group( + workers: &mut [BatchWorker], + group: &PreparedBatchGroup, + tile_options: TileBatchOptions, + image_jobs: &mut [CpuTypedDecodeJob<'_, '_, T>], + image_results: &mut [Option>], + flattened: &mut CpuGroupFastWorkspace, + staged: &mut CpuStagedWorkspace, + convert: fn(f32, u8) -> T, +) -> Result<(), BatchInfrastructureError> { + let route = flattened + .route() + .ok_or(BatchInfrastructureError::MissingResult { index: 0 })?; + for window_start in (0..image_jobs.len()).step_by(CPU_ENTROPY_IMAGES_PER_WINDOW) { + let window_end = window_start + .saturating_add(CPU_ENTROPY_IMAGES_PER_WINDOW) + .min(image_jobs.len()); + let image_range = Range { + start: window_start, + end: window_end, + }; + staged.prepare_window(flattened, image_range.clone())?; + + { + let execution = staged.execution(); + run_retained_chunks( + workers, + &mut image_jobs[image_range.clone()], + &mut image_results[image_range.clone()], + tile_options, + |worker, jobs, results| { + let _ = worker.prepare_owned_decode(); + for (job, result) in jobs.iter_mut().zip(results) { + let scratch_index = job.slot.checked_sub(window_start).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: job.slot, + job_count: image_range.len(), + }, + )?; + let scratch = execution.image_scratch.get(scratch_index).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: scratch_index, + job_count: execution.image_scratch.len(), + }, + )?; + let mut scratch = scratch + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *result = + Some(prepare_staged_image(job.image, route, worker, &mut scratch)); + } + Ok(()) + }, + )?; + for (local_index, result) in image_results[image_range.clone()].iter().enumerate() { + if result.as_ref().is_some_and(Result::is_err) { + execution.failed[local_index].store(true, Ordering::Release); + } + } + } + + let mut tile_count = 0usize; + { + let execution = staged.execution(); + for (local_index, job) in image_jobs[image_range.clone()].iter().enumerate() { + if image_results[job.slot].as_ref().is_some_and(Result::is_err) { + continue; + } + match staged_tile_count(job.image, route) { + Ok(count) => tile_count = tile_count.max(count), + Err(error) => { + image_results[job.slot] = Some(Err(error)); + execution.failed[local_index].store(true, Ordering::Release); + } + } + } + } + + for tile_index in 0..tile_count { + prepare_staged_tile_window( + workers, + group, + staged, + route, + tile_options, + window_start, + image_range.clone(), + tile_index, + image_jobs, + image_results, + )?; + staged.prepare_tile_jobs(flattened, image_range.clone(), tile_index)?; + let entropy_job_count = execute_staged_tile_entropy( + workers, + group, + staged, + flattened, + route, + tile_options, + window_start, + image_range.clone(), + image_results, + )?; + flattened.record_entropy_dispatch(entropy_job_count, image_range.len()); + finish_staged_tile_window( + workers, + group, + staged, + route, + tile_options, + window_start, + image_range.clone(), + tile_index, + image_jobs, + image_results, + )?; + } + + { + let execution = staged.execution(); + run_retained_chunks( + workers, + &mut image_jobs[image_range.clone()], + &mut image_results[image_range], + tile_options, + |_worker, jobs, results| { + for (job, result) in jobs.iter_mut().zip(results) { + if result.as_ref().is_some_and(Result::is_err) { + continue; + } + let scratch_index = job.slot.checked_sub(window_start).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: job.slot, + job_count: execution.image_scratch.len(), + }, + )?; + let scratch = execution.image_scratch.get(scratch_index).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: scratch_index, + job_count: execution.image_scratch.len(), + }, + )?; + let mut scratch = scratch + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *result = Some(finish_staged_plan_samples( + job.image, + &group.info, + &mut scratch, + job.output, + convert, + )); + } + Ok(()) + }, + )?; + } + } + Ok(()) +} + +#[expect( + clippy::too_many_arguments, + reason = "the tile-window preparation boundary keeps retained workers, indexed image outputs, and scratch owners explicit" +)] +fn prepare_staged_tile_window( + workers: &mut [BatchWorker], + _group: &PreparedBatchGroup, + staged: &mut CpuStagedWorkspace, + route: BatchCodecRoute, + tile_options: TileBatchOptions, + window_start: usize, + image_range: Range, + tile_index: usize, + image_jobs: &mut [CpuTypedDecodeJob<'_, '_, T>], + image_results: &mut [Option>], +) -> Result<(), BatchInfrastructureError> { + let execution = staged.execution(); + run_retained_chunks( + workers, + &mut image_jobs[image_range.clone()], + &mut image_results[image_range.clone()], + tile_options, + |_worker, jobs, results| { + for (job, result) in jobs.iter_mut().zip(results) { + if result.as_ref().is_some_and(Result::is_err) { + continue; + } + let scratch_index = job.slot.checked_sub(window_start).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: job.slot, + job_count: image_range.len(), + }, + )?; + let tile_count = match staged_tile_count(job.image, route) { + Ok(count) => count, + Err(error) => { + execution.failed[scratch_index].store(true, Ordering::Release); + *result = Some(Err(error)); + continue; + } + }; + if tile_index >= tile_count { + continue; + } + let scratch = execution.image_scratch.get(scratch_index).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: scratch_index, + job_count: execution.image_scratch.len(), + }, + )?; + let mut scratch = scratch + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Err(error) = prepare_staged_tile(job.image, route, tile_index, &mut scratch) + { + execution.failed[scratch_index].store(true, Ordering::Release); + *result = Some(Err(error)); + } + } + Ok(()) + }, + ) +} + +#[expect( + clippy::too_many_arguments, + reason = "the cross-image entropy boundary keeps retained workers, flattened payloads, failure flags, and indexed results explicit" +)] +fn execute_staged_tile_entropy( + workers: &mut [BatchWorker], + group: &PreparedBatchGroup, + staged: &mut CpuStagedWorkspace, + flattened: &CpuGroupFastWorkspace, + route: BatchCodecRoute, + tile_options: TileBatchOptions, + window_start: usize, + image_range: Range, + image_results: &mut [Option>], +) -> Result { + let execution = staged.execution(); + let entropy_job_count = execution.jobs.len(); + run_retained_chunks( + workers, + execution.jobs, + execution.results, + tile_options, + |worker, jobs, results| { + let _ = worker.prepare_owned_decode(); + let mut workspace_ready = false; + for (job, result) in jobs.iter_mut().zip(results) { + let scratch_index = job.image_slot.checked_sub(window_start).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: job.image_slot, + job_count: image_range.len(), + }, + )?; + let failed = execution.failed.get(scratch_index).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: scratch_index, + job_count: execution.failed.len(), + }, + )?; + if failed.load(Ordering::Acquire) { + *result = Some(CpuEntropyOutcome::Complete); + continue; + } + let image = group.images.get(job.image_slot).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: job.image_slot, + job_count: group.images.len(), + }, + )?; + if !workspace_ready { + if let Err(error) = prepare_staged_entropy_worker(image, route, worker) { + failed.store(true, Ordering::Release); + *result = Some(CpuEntropyOutcome::Error(error)); + continue; + } + workspace_ready = true; + } + let scratch = execution.image_scratch.get(scratch_index).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: scratch_index, + job_count: execution.image_scratch.len(), + }, + )?; + let mut scratch = scratch + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let outcome = + execute_staged_entropy_job(image, *job, flattened, worker, &mut scratch); + if outcome.is_err() { + failed.store(true, Ordering::Release); + } + *result = Some(match outcome { + Ok(()) => CpuEntropyOutcome::Complete, + Err(error) => CpuEntropyOutcome::Error(error), + }); + } + Ok(()) + }, + )?; + let image_result_count = image_results.len(); + for (job, outcome) in execution.jobs.iter().zip(execution.results.iter_mut()) { + let Some(CpuEntropyOutcome::Error(error)) = outcome.take() else { + continue; + }; + let image_result = image_results.get_mut(job.image_slot).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: job.image_slot, + job_count: image_result_count, + }, + )?; + if image_result.as_ref().is_some_and(Result::is_ok) { + *image_result = Some(Err(error)); + } + } + Ok(entropy_job_count) +} + +#[expect( + clippy::too_many_arguments, + reason = "the tile Store boundary keeps retained workers, dense output metadata, indexed results, and scratch retirement explicit" +)] +fn finish_staged_tile_window( + workers: &mut [BatchWorker], + group: &PreparedBatchGroup, + staged: &mut CpuStagedWorkspace, + route: BatchCodecRoute, + tile_options: TileBatchOptions, + window_start: usize, + image_range: Range, + tile_index: usize, + image_jobs: &mut [CpuTypedDecodeJob<'_, '_, T>], + image_results: &mut [Option>], +) -> Result<(), BatchInfrastructureError> { + let execution = staged.execution(); + run_retained_chunks( + workers, + &mut image_jobs[image_range.clone()], + &mut image_results[image_range.clone()], + tile_options, + |_worker, jobs, results| { + for (job, result) in jobs.iter_mut().zip(results) { + if result.as_ref().is_some_and(Result::is_err) { + continue; + } + let scratch_index = job.slot.checked_sub(window_start).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: job.slot, + job_count: execution.image_scratch.len(), + }, + )?; + let tile_count = match staged_tile_count(job.image, route) { + Ok(count) => count, + Err(error) => { + execution.failed[scratch_index].store(true, Ordering::Release); + *result = Some(Err(error)); + continue; + } + }; + if tile_index >= tile_count { + continue; + } + let scratch = execution.image_scratch.get(scratch_index).ok_or( + BatchInfrastructureError::ResultIndexOutOfBounds { + index: scratch_index, + job_count: execution.image_scratch.len(), + }, + )?; + let mut scratch = scratch + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Err(error) = finish_staged_tile( + job.image, + route, + tile_index, + group.info.signed, + &mut scratch, + ) { + execution.failed[scratch_index].store(true, Ordering::Release); + *result = Some(Err(error)); + } + } + Ok(()) + }, + ) +} diff --git a/crates/j2k/src/owned_batch/errors.rs b/crates/j2k/src/owned_batch/errors.rs new file mode 100644 index 00000000..d05e73e6 --- /dev/null +++ b/crates/j2k/src/owned_batch/errors.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use alloc::sync::Arc; +use core::fmt; + +use crate::{DecodeSettings, J2kError}; + +/// Stage at which an indexed image failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum BatchErrorStage { + /// Header parsing, request planning, or representability validation. + Prepare, + /// Pixel reconstruction or output packing. + Decode, +} + +impl fmt::Display for BatchErrorStage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Prepare => f.write_str("prepare"), + Self::Decode => f.write_str("decode"), + } + } +} + +/// Reason an otherwise valid image cannot be represented by the fast batch API. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum NonRepresentableReason { + /// Output requires a component count other than Gray, RGB, or RGBA. + UnsupportedComponentCount, + /// Components use different significant precisions. + MixedPrecision, + /// Components mix signed and unsigned sample domains. + MixedSignedness, + /// At least one component is subsampled. + ComponentSubsampling, + /// At least one component uses more than sixteen significant bits. + PrecisionAboveSixteen, + /// Tile or component coding-style overrides select different wavelet transforms. + MixedWaveletTransform, + /// Parsed color metadata is not representable as Gray, RGB, or RGBA. + UnsupportedColor, +} + +impl fmt::Display for NonRepresentableReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnsupportedComponentCount => f.write_str("unsupported component count"), + Self::MixedPrecision => f.write_str("mixed component precision"), + Self::MixedSignedness => f.write_str("mixed component signedness"), + Self::ComponentSubsampling => f.write_str("component subsampling"), + Self::PrecisionAboveSixteen => f.write_str("component precision above sixteen bits"), + Self::MixedWaveletTransform => { + f.write_str("mixed tile or component wavelet transforms") + } + Self::UnsupportedColor => f.write_str("unsupported color interpretation"), + } + } +} + +/// Structured error for one submitted image. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum BatchItemError { + /// Codec parsing or decoding failed. + #[error("batch image {stage} failed: {source}")] + Codec { + /// Operation phase. + stage: BatchErrorStage, + /// Source-preserving codec error. + #[source] + source: Arc, + }, + /// Valid codec output cannot be represented by one dense native-width batch. + #[error("non-representable batch output: {reason}")] + NonRepresentableBatchOutput { + /// Stable representability class. + reason: NonRepresentableReason, + }, + /// A prepared image was submitted under a different codec validation policy. + #[error( + "prepared image decode settings {prepared:?} do not match requested settings {requested:?}" + )] + PreparedDecodeSettingsMismatch { + /// Validation policy used to parse and build the retained execution plan. + prepared: DecodeSettings, + /// Validation policy requested for the new batch. + requested: DecodeSettings, + }, +} + +/// One batch item error retaining the caller's input index. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("image {index}: {source}")] +pub struct IndexedBatchError { + /// Index in the original submitted input slice. + pub index: usize, + /// Structured preparation or decode failure. + #[source] + pub source: BatchItemError, +} diff --git a/crates/j2k/src/owned_batch/prepare.rs b/crates/j2k/src/owned_batch/prepare.rs new file mode 100644 index 00000000..7028e328 --- /dev/null +++ b/crates/j2k/src/owned_batch/prepare.rs @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Concurrent inspection, representability validation, and homogeneous grouping. + +use super::{ + run_retained_chunks, size_of, Arc, BatchDecodeOptions, BatchInfrastructureError, + BatchItemError, BatchWorker, EncodedImage, IndexedBatchError, NonZeroUsize, PrepareImageResult, + PrepareJob, PreparedBatch, PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, + J2K_BATCH_METADATA_ALLOWANCE_BYTES, MAX_GENERIC_BATCH_WORKERS, +}; + +mod image; +use self::image::{ + batch_execution_shape, batch_group_info, prepare_image, reconcile_codec_plan_metadata, +}; + +/// Inspect, validate, and group owned encoded images without decoding pixels. +/// +/// Per-image failures are retained in [`PreparedBatch::errors`]. The outer +/// error is reserved for batch infrastructure and allocation failures. +pub fn prepare_batch( + inputs: Vec, + options: BatchDecodeOptions, +) -> Result { + let input_count = inputs.len(); + let available = std::thread::available_parallelism().map_or(1, NonZeroUsize::get); + let worker_count = options + .workers + .map_or(available, NonZeroUsize::get) + .clamp(1, MAX_GENERIC_BATCH_WORKERS) + .min(input_count.max(1)); + let mut workers = try_prepare_vec(worker_count, "J2K preparation workers")?; + for _ in 0..worker_count { + workers.push(BatchWorker::new_owned(input_count.max(2))); + } + prepare_batch_with_workers(inputs, options, &mut workers) +} + +pub(super) fn prepare_batch_with_workers( + inputs: Vec, + options: BatchDecodeOptions, + workers: &mut [BatchWorker], +) -> Result { + prepare_staging_bytes(inputs.len())?; + let mut groups: Vec = Vec::new(); + let mut errors = Vec::new(); + let input_count = inputs.len(); + let mut jobs = try_prepare_vec(input_count, "J2K preparation jobs")?; + for (source_index, input) in inputs.into_iter().enumerate() { + jobs.push(PrepareJob { + source_index, + input: Some(input), + }); + } + let mut prepared = try_prepare_vec(input_count, "J2K preparation result slots")?; + prepared.resize_with(input_count, || None); + run_retained_chunks( + workers, + &mut jobs, + &mut prepared, + TileBatchOptions::new(options.workers), + |worker, jobs, results| { + worker.prepare_owned_decode(); + for (job, slot) in jobs.iter_mut().zip(results) { + let input = job + .input + .take() + .ok_or(BatchInfrastructureError::MissingResult { + index: job.source_index, + })?; + *slot = Some(prepare_image(input, job.source_index, options, worker)); + } + Ok(()) + }, + )?; + drop(jobs); + + for (source_index, result) in prepared.into_iter().enumerate() { + let Some(result) = result else { + return Err(BatchInfrastructureError::MissingResult { + index: source_index, + }); + }; + push_prepared_result(&mut groups, &mut errors, source_index, result, options)?; + } + + Ok(PreparedBatch { + groups: Arc::from(groups), + errors: Arc::from(errors), + options, + }) +} + +/// Regroup caller-supplied prepared images without parsing or copying codestreams. +/// +/// Returned source indices are positions in `images`, not +/// [`PreparedImage::source_index`], which remains the index from the image's +/// original preparation call. Group order follows first occurrence and image +/// order is stable within each group. A strict/lenient policy mismatch is an +/// indexed preflight error; output layout and CPU worker policy may change. +/// Errors from earlier batches are not carried because only the supplied +/// successful prepared images participate in this new batch. +pub fn prepare_batch_from_images( + images: Vec, + options: BatchDecodeOptions, +) -> Result { + prepare_staging_bytes(images.len())?; + let mut groups = Vec::new(); + let mut errors = Vec::new(); + for (source_index, image) in images.into_iter().enumerate() { + let result = regroup_prepared_image(image, options); + push_prepared_result(&mut groups, &mut errors, source_index, result, options)?; + } + Ok(PreparedBatch { + groups: Arc::from(groups), + errors: Arc::from(errors), + options, + }) +} + +fn regroup_prepared_image(image: PreparedImage, options: BatchDecodeOptions) -> PrepareImageResult { + if image.decode_settings() != options.settings { + return Err(BatchItemError::PreparedDecodeSettingsMismatch { + prepared: image.decode_settings(), + requested: options.settings, + }); + } + let mut info = batch_group_info(image.support(), image.plan(), options.layout)?; + reconcile_codec_plan_metadata(&mut info, image.codec_plan())?; + let execution_shape = + batch_execution_shape(image.support(), image.plan(), image.preparation_depth()); + Ok((image, info, execution_shape)) +} + +fn push_prepared_result( + groups: &mut Vec, + errors: &mut Vec, + source_index: usize, + result: PrepareImageResult, + options: BatchDecodeOptions, +) -> Result<(), BatchInfrastructureError> { + match result { + Ok((image, info, execution_shape)) => { + if let Some(group) = groups + .iter_mut() + .find(|group| group.info == info && group.execution_shape == execution_shape) + { + try_reserve_one( + &mut group.source_indices, + "J2K prepared group source indices", + )?; + try_reserve_one(&mut group.images, "J2K prepared group images")?; + group.source_indices.push(source_index); + group.images.push(image); + } else { + let mut images = try_prepare_vec(1, "J2K prepared group images")?; + images.push(image); + let mut source_indices = try_prepare_vec(1, "J2K prepared group source indices")?; + source_indices.push(source_index); + try_reserve_one(groups, "J2K prepared groups")?; + groups.push(PreparedBatchGroup { + info, + options, + execution_shape, + images, + source_indices, + }); + } + } + Err(source) => { + try_reserve_one(errors, "J2K prepared indexed errors")?; + errors.push(IndexedBatchError { + index: source_index, + source, + }); + } + } + Ok(()) +} + +pub(super) fn prepare_staging_bytes(input_count: usize) -> Result { + let execution_item_bytes = size_of::() + .checked_add(size_of::>()) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what: "J2K preparation staging", + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + })?; + let successful_output_bytes = size_of::() + .checked_add(size_of::()) + .and_then(|bytes| bytes.checked_add(size_of::())) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what: "J2K preparation staging", + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + })?; + let output_item_bytes = size_of::>() + .checked_add(successful_output_bytes.max(size_of::())) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what: "J2K preparation staging", + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + })?; + let item_bytes = execution_item_bytes.max(output_item_bytes); + let bytes = input_count.checked_mul(item_bytes).ok_or( + BatchInfrastructureError::AllocationTooLarge { + what: "J2K preparation staging", + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }, + )?; + if bytes > J2K_BATCH_METADATA_ALLOWANCE_BYTES { + return Err(BatchInfrastructureError::AllocationTooLarge { + what: "J2K preparation staging", + requested: bytes, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }); + } + Ok(bytes) +} + +fn try_prepare_vec( + capacity: usize, + what: &'static str, +) -> Result, BatchInfrastructureError> { + let bytes = capacity.checked_mul(size_of::()).ok_or( + BatchInfrastructureError::AllocationTooLarge { + what, + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }, + )?; + if bytes > J2K_BATCH_METADATA_ALLOWANCE_BYTES { + return Err(BatchInfrastructureError::AllocationTooLarge { + what, + requested: bytes, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }); + } + j2k_core::try_host_vec_with_capacity(capacity).map_err(|error| { + BatchInfrastructureError::HostAllocationFailed { + what, + bytes: error.requested_bytes(), + } + }) +} + +fn try_reserve_one( + values: &mut Vec, + what: &'static str, +) -> Result<(), BatchInfrastructureError> { + if values.len() < values.capacity() { + return Ok(()); + } + let requested_items = + values + .len() + .checked_add(1) + .ok_or(BatchInfrastructureError::AllocationTooLarge { + what, + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + })?; + let requested = requested_items.checked_mul(size_of::()).ok_or( + BatchInfrastructureError::AllocationTooLarge { + what, + requested: usize::MAX, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }, + )?; + if requested > J2K_BATCH_METADATA_ALLOWANCE_BYTES { + return Err(BatchInfrastructureError::AllocationTooLarge { + what, + requested, + cap: J2K_BATCH_METADATA_ALLOWANCE_BYTES, + }); + } + values + .try_reserve_exact(1) + .map_err(|_| BatchInfrastructureError::HostAllocationFailed { + what, + bytes: requested, + }) +} diff --git a/crates/j2k/src/owned_batch/prepare/image.rs b/crates/j2k/src/owned_batch/prepare/image.rs new file mode 100644 index 00000000..2edb9e5e --- /dev/null +++ b/crates/j2k/src/owned_batch/prepare/image.rs @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Per-image metadata validation and retained native plan construction. + +use super::super::{ + extract_j2k_codestream_payload, parse_image_info, Arc, BatchAlpha, BatchCodecRoute, BatchColor, + BatchDecodeOptions, BatchErrorStage, BatchExecutionShape, BatchGroupInfo, BatchItemError, + BatchLayout, BatchWaveletTransform, BatchWorker, Colorspace, CompressedTransferSyntax, + DecodeSettings, DeviceDecodePlan, Downscale, EncodedImage, J2kCodestreamRange, J2kError, + J2kSupportInfo, NativeSampleType, NonRepresentableReason, PreparationDepth, PrepareImageResult, + PreparedClassicPlan, PreparedCodecPlan, PreparedHtj2kPlan, PreparedImage, PreparedImageInner, +}; + +pub(super) fn prepare_image( + input: EncodedImage, + source_index: usize, + options: BatchDecodeOptions, + worker: &mut BatchWorker, +) -> PrepareImageResult { + let parsed = parse_image_info(&input.bytes).map_err(|source| BatchItemError::Codec { + stage: BatchErrorStage::Prepare, + source: Arc::new(source), + })?; + let support = parsed.into_support_info(); + let plan = DeviceDecodePlan::for_image(support.info.dimensions, input.request.device_request()) + .map_err(|source| BatchItemError::Codec { + stage: BatchErrorStage::Prepare, + source: Arc::new(source), + })?; + let mut info = batch_group_info(&support, plan, options.layout)?; + let codestream = + extract_j2k_codestream_payload(&input.bytes).map_err(|source| BatchItemError::Codec { + stage: BatchErrorStage::Prepare, + source: Arc::new(source), + })?; + let codestream_range = J2kCodestreamRange { + offset: codestream.codestream_offset(), + length: codestream.codestream().len(), + }; + let codec_plan = match info.route { + BatchCodecRoute::Classic => { + prepare_classic_offset_plan(&input, &info, plan, options.settings, worker)? + .map_or(PreparedCodecPlan::MetadataOnly, PreparedCodecPlan::Classic) + } + BatchCodecRoute::Htj2k => { + prepare_htj2k_offset_plan(&input, &info, plan, options.settings, worker)? + .map_or(PreparedCodecPlan::MetadataOnly, PreparedCodecPlan::Htj2k) + } + }; + reconcile_codec_plan_metadata(&mut info, &codec_plan)?; + let preparation_depth = codec_plan.preparation_depth(); + let execution_shape = batch_execution_shape(&support, plan, preparation_depth); + let image = PreparedImage { + inner: Arc::new(PreparedImageInner { + bytes: input.bytes, + request: input.request, + source_index, + decode_settings: options.settings, + support: Arc::new(support), + plan, + codestream_range, + codec_plan, + }), + }; + Ok((image, info, execution_shape)) +} + +pub(super) fn batch_execution_shape( + support: &J2kSupportInfo, + plan: DeviceDecodePlan, + preparation_depth: PreparationDepth, +) -> BatchExecutionShape { + let source_rect = plan.source_rect(); + BatchExecutionShape { + source_dimensions: plan.source_dims(), + source_rect_dimensions: (source_rect.w, source_rect.h), + scale: plan.scale(), + tile_layout: support.info.tile_layout, + resolution_levels: support.info.resolution_levels, + preparation_depth, + } +} + +fn prepare_classic_offset_plan( + input: &EncodedImage, + info: &BatchGroupInfo, + plan: DeviceDecodePlan, + decode_settings: DecodeSettings, + worker: &mut BatchWorker, +) -> Result, BatchItemError> { + if info.route != BatchCodecRoute::Classic + || !matches!( + info.color, + BatchColor::Gray | BatchColor::Rgb | BatchColor::Rgba + ) + { + return Ok(None); + } + let target_resolution = (plan.scale() != Downscale::None).then_some(( + plan.source_dims().0.div_ceil(plan.scale().denominator()), + plan.source_dims().1.div_ceil(plan.scale().denominator()), + )); + let native_settings = decode_settings.to_native(target_resolution); + let image = j2k_native::Image::new(&input.bytes, &native_settings).map_err(|source| { + BatchItemError::Codec { + stage: BatchErrorStage::Prepare, + source: Arc::new(J2kError::from_native_decode_error(source)), + } + })?; + let output = plan.output_rect(); + match worker.build_prepared_classic_plan(&image, (output.x, output.y, output.w, output.h)) { + Ok(plan) => Ok(Some(PreparedClassicPlan::from_native(plan))), + Err(j2k_native::DecodeError::Decoding( + j2k_native::DecodingError::DirectPlanUnsupported(_) + | j2k_native::DecodingError::UnsupportedFeature(_), + )) => Ok(None), + Err(source) => Err(BatchItemError::Codec { + stage: BatchErrorStage::Prepare, + source: Arc::new(J2kError::from_native_decode_error(source)), + }), + } +} + +fn prepare_htj2k_offset_plan( + input: &EncodedImage, + info: &BatchGroupInfo, + plan: DeviceDecodePlan, + decode_settings: DecodeSettings, + worker: &mut BatchWorker, +) -> Result, BatchItemError> { + if info.route != BatchCodecRoute::Htj2k + || !matches!( + info.color, + BatchColor::Gray | BatchColor::Rgb | BatchColor::Rgba + ) + { + return Ok(None); + } + let target_resolution = (plan.scale() != Downscale::None).then_some(( + plan.source_dims().0.div_ceil(plan.scale().denominator()), + plan.source_dims().1.div_ceil(plan.scale().denominator()), + )); + let native_settings = decode_settings.to_native(target_resolution); + let image = j2k_native::Image::new(&input.bytes, &native_settings).map_err(|source| { + BatchItemError::Codec { + stage: BatchErrorStage::Prepare, + source: Arc::new(J2kError::from_native_decode_error(source)), + } + })?; + let output = plan.output_rect(); + match worker.build_prepared_htj2k_plan(&image, (output.x, output.y, output.w, output.h)) { + Ok(plan) => Ok(Some(PreparedHtj2kPlan::from_native(plan))), + Err(j2k_native::DecodeError::Decoding( + j2k_native::DecodingError::DirectPlanUnsupported(_), + )) => Ok(None), + Err(source) => Err(BatchItemError::Codec { + stage: BatchErrorStage::Prepare, + source: Arc::new(J2kError::from_native_decode_error(source)), + }), + } +} + +pub(super) fn batch_group_info( + support: &J2kSupportInfo, + plan: DeviceDecodePlan, + layout: BatchLayout, +) -> Result { + let Some(first) = support.components.first().copied() else { + return Err(nonrepresentable( + NonRepresentableReason::UnsupportedComponentCount, + )); + }; + if support + .components + .iter() + .any(|component| component.bit_depth != first.bit_depth) + { + return Err(nonrepresentable(NonRepresentableReason::MixedPrecision)); + } + if support + .components + .iter() + .any(|component| component.signed != first.signed) + { + return Err(nonrepresentable(NonRepresentableReason::MixedSignedness)); + } + if support.has_component_subsampling() { + return Err(nonrepresentable( + NonRepresentableReason::ComponentSubsampling, + )); + } + if first.bit_depth > 16 { + return Err(nonrepresentable( + NonRepresentableReason::PrecisionAboveSixteen, + )); + } + + let (color, alpha) = representable_batch_color(support)?; + let sample_type = if first.signed { + NativeSampleType::I16 + } else if first.bit_depth <= 8 { + NativeSampleType::U8 + } else { + NativeSampleType::U16 + }; + let (route, transform) = match support.transfer_syntax { + CompressedTransferSyntax::Jpeg2000Lossless => ( + BatchCodecRoute::Classic, + BatchWaveletTransform::Reversible53, + ), + CompressedTransferSyntax::Jpeg2000Lossy => ( + BatchCodecRoute::Classic, + BatchWaveletTransform::Irreversible97, + ), + CompressedTransferSyntax::HtJpeg2000Lossless => { + (BatchCodecRoute::Htj2k, BatchWaveletTransform::Reversible53) + } + CompressedTransferSyntax::HtJpeg2000Lossy => ( + BatchCodecRoute::Htj2k, + BatchWaveletTransform::Irreversible97, + ), + _ => return Err(nonrepresentable(NonRepresentableReason::UnsupportedColor)), + }; + + Ok(BatchGroupInfo { + dimensions: plan.output_dims(), + color, + alpha, + precision: first.bit_depth, + signed: first.signed, + sample_type, + layout, + colorspace: support.info.colorspace, + route, + transform, + transfer_syntax: support.transfer_syntax, + payload_kind: support.payload_kind, + }) +} + +pub(super) fn reconcile_codec_plan_metadata( + info: &mut BatchGroupInfo, + codec_plan: &PreparedCodecPlan, +) -> Result<(), BatchItemError> { + let (route, transform) = match codec_plan { + PreparedCodecPlan::MetadataOnly => return Ok(()), + PreparedCodecPlan::Classic(plan) => { + (BatchCodecRoute::Classic, plan.uniform_wavelet_transform()) + } + PreparedCodecPlan::Htj2k(plan) => { + (BatchCodecRoute::Htj2k, plan.uniform_wavelet_transform()) + } + }; + let transform = + transform.ok_or_else(|| nonrepresentable(NonRepresentableReason::MixedWaveletTransform))?; + let transform = match transform { + j2k_native::J2kWaveletTransform::Reversible53 => BatchWaveletTransform::Reversible53, + j2k_native::J2kWaveletTransform::Irreversible97 => BatchWaveletTransform::Irreversible97, + }; + info.route = route; + info.transform = transform; + info.transfer_syntax = match (route, transform) { + (BatchCodecRoute::Classic, BatchWaveletTransform::Reversible53) => { + CompressedTransferSyntax::Jpeg2000Lossless + } + (BatchCodecRoute::Classic, BatchWaveletTransform::Irreversible97) => { + CompressedTransferSyntax::Jpeg2000Lossy + } + (BatchCodecRoute::Htj2k, BatchWaveletTransform::Reversible53) => { + CompressedTransferSyntax::HtJpeg2000Lossless + } + (BatchCodecRoute::Htj2k, BatchWaveletTransform::Irreversible97) => { + CompressedTransferSyntax::HtJpeg2000Lossy + } + }; + Ok(()) +} + +fn representable_batch_color( + support: &J2kSupportInfo, +) -> Result<(BatchColor, BatchAlpha), BatchItemError> { + let file_metadata = support.file_metadata.as_ref(); + if file_metadata.is_some_and(|metadata| { + metadata.has_palette + || metadata.has_component_mapping + || metadata.palette.is_some() + || !metadata.component_mappings.is_empty() + || metadata.has_icc_profile() + }) || support.info.colorspace == Colorspace::IccTagged + { + return Err(nonrepresentable(NonRepresentableReason::UnsupportedColor)); + } + + match (support.component_count(), support.info.colorspace) { + (1, Colorspace::Grayscale | Colorspace::SGray) => Ok((BatchColor::Gray, BatchAlpha::None)), + (3, Colorspace::Rgb | Colorspace::SRgb | Colorspace::Rct | Colorspace::Ict) + if file_metadata.is_none_or(|metadata| { + metadata.channel_definitions.is_empty() + || has_identity_rgb_channel_definitions(&metadata.channel_definitions) + }) => + { + Ok((BatchColor::Rgb, BatchAlpha::None)) + } + (4, Colorspace::Rgb | Colorspace::SRgb | Colorspace::Rct | Colorspace::Ict) => { + file_metadata + .and_then(|metadata| identity_rgba_alpha(&metadata.channel_definitions)) + .map(|alpha| (BatchColor::Rgba, alpha)) + .ok_or_else(|| nonrepresentable(NonRepresentableReason::UnsupportedColor)) + } + (1 | 3 | 4, _) => Err(nonrepresentable(NonRepresentableReason::UnsupportedColor)), + _ => Err(nonrepresentable( + NonRepresentableReason::UnsupportedComponentCount, + )), + } +} + +fn has_identity_rgb_channel_definitions(definitions: &[crate::J2kChannelDefinition]) -> bool { + definitions.len() == 3 + && definitions.iter().enumerate().all(|(index, definition)| { + definition.channel_index == u16::try_from(index).unwrap_or(u16::MAX) + && definition.channel_type == crate::J2kChannelType::Color + && definition.association + == crate::J2kChannelAssociation::Color { + index: u16::try_from(index + 1).unwrap_or(u16::MAX), + } + }) +} + +fn identity_rgba_alpha(definitions: &[crate::J2kChannelDefinition]) -> Option { + let (alpha, rgb) = definitions.split_last()?; + let alpha_interpretation = match alpha.channel_type { + crate::J2kChannelType::Opacity => BatchAlpha::Straight, + crate::J2kChannelType::PremultipliedOpacity => BatchAlpha::Premultiplied, + _ => return None, + }; + (has_identity_rgb_channel_definitions(rgb) + && alpha.channel_index == 3 + && alpha.association == crate::J2kChannelAssociation::WholeImage) + .then_some(alpha_interpretation) +} + +fn nonrepresentable(reason: NonRepresentableReason) -> BatchItemError { + BatchItemError::NonRepresentableBatchOutput { reason } +} diff --git a/crates/j2k/src/owned_batch/prepared.rs b/crates/j2k/src/owned_batch/prepared.rs new file mode 100644 index 00000000..6bfa1cd2 --- /dev/null +++ b/crates/j2k/src/owned_batch/prepared.rs @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use alloc::{sync::Arc, vec::Vec}; + +use j2k_core::{BatchInfrastructureError, Downscale, TileLayout}; + +use super::{ + prepare_batch, prepare_batch_from_images, BatchDecodeOptions, BatchGroupInfo, DecodeRequest, + EncodedImage, IndexedBatchError, J2kCodestreamRange, PreparationDepth, PreparedClassicPlan, + PreparedHtj2kPlan, +}; +use crate::{DecodeSettings, DeviceDecodePlan, J2kSupportInfo}; + +/// Cheaply cloneable preparation result for one image. +#[derive(Debug, Clone)] +pub struct PreparedImage { + pub(super) inner: Arc, +} + +#[derive(Debug)] +pub(super) enum PreparedCodecPlan { + MetadataOnly, + Htj2k(PreparedHtj2kPlan), + Classic(PreparedClassicPlan), +} + +impl PreparedCodecPlan { + pub(super) const fn preparation_depth(&self) -> PreparationDepth { + match self { + Self::MetadataOnly => PreparationDepth::MetadataOnly, + Self::Htj2k(_) => PreparationDepth::Htj2kOffsetPlan, + Self::Classic(_) => PreparationDepth::ClassicOffsetPlan, + } + } +} + +#[derive(Debug)] +pub(super) struct PreparedImageInner { + pub(super) bytes: Arc<[u8]>, + pub(super) request: DecodeRequest, + pub(super) source_index: usize, + pub(super) decode_settings: DecodeSettings, + pub(super) support: Arc, + pub(super) plan: DeviceDecodePlan, + pub(super) codestream_range: J2kCodestreamRange, + pub(super) codec_plan: PreparedCodecPlan, +} + +impl PreparedImage { + /// Original encoded bytes. + #[must_use] + pub fn bytes(&self) -> &Arc<[u8]> { + &self.inner.bytes + } + + /// Caller decode request. + #[must_use] + pub fn request(&self) -> DecodeRequest { + self.inner.request + } + + /// Original caller input index. + #[must_use] + pub fn source_index(&self) -> usize { + self.inner.source_index + } + + /// Validation policy used to parse and build this retained execution plan. + #[must_use] + pub fn decode_settings(&self) -> DecodeSettings { + self.inner.decode_settings + } + + /// Parsed codestream and wrapper metadata. + #[must_use] + pub fn support(&self) -> &J2kSupportInfo { + &self.inner.support + } + + /// Normalized source and output geometry. + #[must_use] + pub fn plan(&self) -> DeviceDecodePlan { + self.inner.plan + } + + /// Raw codestream range inside [`Self::bytes`]. + #[must_use] + pub fn codestream_range(&self) -> J2kCodestreamRange { + self.inner.codestream_range + } + + /// Whether this image retains metadata only or a parse-free codec offset plan. + #[must_use] + pub fn preparation_depth(&self) -> PreparationDepth { + self.inner.codec_plan.preparation_depth() + } + + /// Reusable HTJ2K geometry and payload references, when supported. + /// Payload offsets are absolute byte ranges inside [`Self::bytes`], + /// including any JP2/JPH container prefix. + #[must_use] + pub fn htj2k_plan(&self) -> Option<&PreparedHtj2kPlan> { + match &self.inner.codec_plan { + PreparedCodecPlan::Htj2k(plan) => Some(plan), + PreparedCodecPlan::MetadataOnly | PreparedCodecPlan::Classic(_) => None, + } + } + + /// Reusable classic JPEG 2000 geometry and payload-fragment references, + /// when supported. Fragment offsets are absolute ranges inside + /// [`Self::bytes`], including any JP2 container prefix. + #[must_use] + pub fn classic_plan(&self) -> Option<&PreparedClassicPlan> { + match &self.inner.codec_plan { + PreparedCodecPlan::Classic(plan) => Some(plan), + PreparedCodecPlan::MetadataOnly | PreparedCodecPlan::Htj2k(_) => None, + } + } + + pub(super) fn codec_plan(&self) -> &PreparedCodecPlan { + &self.inner.codec_plan + } +} + +/// One homogeneous set of prepared images. +#[derive(Debug)] +pub struct PreparedBatchGroup { + pub(super) info: BatchGroupInfo, + pub(super) options: BatchDecodeOptions, + pub(super) execution_shape: BatchExecutionShape, + pub(super) images: Vec, + pub(super) source_indices: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct BatchExecutionShape { + pub(super) source_dimensions: (u32, u32), + pub(super) source_rect_dimensions: (u32, u32), + pub(super) scale: Downscale, + pub(super) tile_layout: Option, + pub(super) resolution_levels: u8, + pub(super) preparation_depth: PreparationDepth, +} + +impl PreparedBatchGroup { + /// Shared output metadata and grouping key. + #[must_use] + pub fn info(&self) -> &BatchGroupInfo { + &self.info + } + + /// Decode policy captured when every image in this group was prepared. + /// + /// Backend sessions use this value instead of their current defaults so a + /// reusable group cannot silently drift between strict and lenient decode. + #[must_use] + pub const fn options(&self) -> BatchDecodeOptions { + self.options + } + + /// Prepared images in caller order. + #[must_use] + pub fn images(&self) -> &[PreparedImage] { + &self.images + } + + /// Original caller indices in group order. + #[must_use] + pub fn source_indices(&self) -> &[usize] { + &self.source_indices + } +} + +/// Cheaply cloneable parsed and grouped batch reusable across decode calls. +#[derive(Debug, Clone)] +pub struct PreparedBatch { + pub(super) groups: Arc<[PreparedBatchGroup]>, + pub(super) errors: Arc<[IndexedBatchError]>, + pub(super) options: BatchDecodeOptions, +} + +/// Common synchronous boundary implemented by persistent codec batch sessions. +/// +/// GPU implementations may choose a resident or pending associated output; +/// synchronization and submission details remain backend-specific. +pub trait BatchDecoder { + /// Backend-specific successful output. + type Output; + /// Backend-specific infrastructure or execution error. + type Error: From; + + /// Preparation and output policy retained by this persistent session. + fn options(&self) -> BatchDecodeOptions; + + /// Inspect and group owned inputs without consuming their encoded byte owners. + fn prepare_batch(&self, inputs: Vec) -> Result { + prepare_batch(inputs, self.options()).map_err(Self::Error::from) + } + + /// Regroup caller-supplied prepared images without reparsing their encoded bytes. + fn prepare_prepared_images( + &self, + images: Vec, + ) -> Result { + prepare_batch_from_images(images, self.options()).map_err(Self::Error::from) + } + + /// Prepare and decode one owned batch through the common session boundary. + fn decode_batch(&mut self, inputs: Vec) -> Result { + let prepared = self.prepare_batch(inputs)?; + self.decode_prepared(&prepared) + } + + /// Regroup and decode caller-supplied prepared images without reparsing them. + fn decode_prepared_images( + &mut self, + images: Vec, + ) -> Result { + let prepared = self.prepare_prepared_images(images)?; + self.decode_prepared(&prepared) + } + + /// Decode a reusable prepared batch without consuming its encoded inputs or plans. + fn decode_prepared(&mut self, prepared: &PreparedBatch) -> Result; +} + +impl PreparedBatch { + /// Homogeneous prepared groups in first-occurrence order. + #[must_use] + pub fn groups(&self) -> &[PreparedBatchGroup] { + &self.groups + } + + /// Indexed preflight failures. + #[must_use] + pub fn errors(&self) -> &[IndexedBatchError] { + &self.errors + } + + /// Options captured when this batch was prepared. + #[must_use] + pub const fn options(&self) -> BatchDecodeOptions { + self.options + } + + /// Consume this handle into its shared group and error owners. + #[must_use] + pub fn into_parts( + self, + ) -> ( + Arc<[PreparedBatchGroup]>, + Arc<[IndexedBatchError]>, + BatchDecodeOptions, + ) { + (self.groups, self.errors, self.options) + } +} diff --git a/crates/j2k/src/owned_batch/prepared_plan.rs b/crates/j2k/src/owned_batch/prepared_plan.rs new file mode 100644 index 00000000..3aea7fbb --- /dev/null +++ b/crates/j2k/src/owned_batch/prepared_plan.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Facade-owned views over retained HTJ2K preparation metadata. + +use alloc::sync::Arc; +use core::any::Any; + +pub use j2k_types::{ + HtCodeBlockPayloadRanges as Htj2kPayloadRanges, + J2kClassicCodeBlockPayload as ClassicCodeBlockPayload, J2kCodestreamRange, +}; + +/// Opaque retained classic JPEG 2000 execution plan for one prepared image. +/// +/// Public callers inspect fragment byte ranges without depending on native +/// implementation types. Backend crates use [`Self::adapter_view`] to borrow +/// the immutable native geometry and segment metadata. +#[derive(Debug, Clone)] +pub struct PreparedClassicPlan { + plan: Arc, +} + +impl PreparedClassicPlan { + pub(super) fn from_native(plan: j2k_native::J2kReferencedClassicPlan) -> Self { + Self { + plan: Arc::new(plan), + } + } + + pub(crate) fn native_plan(&self) -> &j2k_native::J2kReferencedClassicPlan { + &self.plan + } + + /// Whether the retained geometry decodes one grayscale component. + #[must_use] + pub fn is_grayscale(&self) -> bool { + !self.plan.tiles().is_empty() + && self + .plan + .tiles() + .iter() + .all(|tile| tile.grayscale_geometry().is_some()) + } + + /// Whether the retained geometry decodes three color components. + #[must_use] + pub fn is_color(&self) -> bool { + !self.plan.tiles().is_empty() + && self + .plan + .tiles() + .iter() + .all(|tile| tile.color_geometry().is_some()) + } + + /// Whether the retained geometry decodes four components in R, G, B, A order. + #[must_use] + pub fn is_rgba(&self) -> bool { + !self.plan.tiles().is_empty() + && self + .plan + .tiles() + .iter() + .all(|tile| tile.rgba_geometry().is_some()) + } + + /// Number of referenced classic code-block payloads. + #[must_use] + pub fn payload_count(&self) -> usize { + self.plan.payloads().len() + } + + /// Number of encoded-input fragments across every code-block payload. + #[must_use] + pub fn range_count(&self) -> usize { + self.plan.ranges().len() + } + + /// Whether the plan has no referenced code-block payloads. + #[must_use] + pub fn is_empty(&self) -> bool { + self.plan.payloads().is_empty() + } + + pub(super) fn uniform_wavelet_transform(&self) -> Option { + uniform_wavelet_transform(self.plan.tiles()) + } + + /// Return one code-block payload descriptor by traversal index. + #[must_use] + pub fn payload(&self, index: usize) -> Option { + self.plan.payloads().get(index).copied() + } + + /// Iterate over code-block payload descriptors in component/step/job order. + pub fn payloads(&self) -> impl ExactSizeIterator + '_ { + self.plan.payloads().iter().copied() + } + + /// Return one original-input fragment range by flat index. + #[must_use] + pub fn range(&self, index: usize) -> Option { + self.plan.ranges().get(index).copied() + } + + /// Iterate over original-input fragment ranges in payload concatenation order. + pub fn ranges(&self) -> impl ExactSizeIterator + '_ { + self.plan.ranges().iter().copied() + } + + /// Borrow immutable backend-specific geometry for adapter downcasting. + #[doc(hidden)] + #[must_use] + pub fn adapter_view(&self) -> &(dyn Any + Send + Sync) { + self.plan.as_ref() + } +} + +/// Opaque retained HTJ2K execution plan for one prepared image. +/// +/// Public callers can inspect payload byte ranges without depending on native +/// implementation types. Device backend crates use [`Self::adapter_view`] to +/// borrow the immutable native plan for the lifetime of this value. +#[derive(Debug, Clone)] +pub struct PreparedHtj2kPlan { + plan: Arc, +} + +impl PreparedHtj2kPlan { + pub(super) fn from_native(plan: j2k_native::J2kReferencedHtj2kPlan) -> Self { + Self { + plan: Arc::new(plan), + } + } + + pub(crate) fn native_plan(&self) -> &j2k_native::J2kReferencedHtj2kPlan { + &self.plan + } + + /// Whether the retained geometry decodes one grayscale component. + #[must_use] + pub fn is_grayscale(&self) -> bool { + !self.plan.tiles().is_empty() + && self + .plan + .tiles() + .iter() + .all(|tile| tile.grayscale_geometry().is_some()) + } + + /// Whether the retained geometry decodes three color components. + #[must_use] + pub fn is_color(&self) -> bool { + !self.plan.tiles().is_empty() + && self + .plan + .tiles() + .iter() + .all(|tile| tile.color_geometry().is_some()) + } + + /// Whether the retained geometry decodes four components in R, G, B, A order. + #[must_use] + pub fn is_rgba(&self) -> bool { + !self.plan.tiles().is_empty() + && self + .plan + .tiles() + .iter() + .all(|tile| tile.rgba_geometry().is_some()) + } + + /// Number of referenced HTJ2K code-block payloads. + #[must_use] + pub fn payload_count(&self) -> usize { + self.plan.payloads().len() + } + + /// Whether the plan has no referenced HTJ2K code-block payloads. + #[must_use] + pub fn is_empty(&self) -> bool { + self.plan.payloads().is_empty() + } + + pub(super) fn uniform_wavelet_transform(&self) -> Option { + uniform_wavelet_transform(self.plan.tiles()) + } + + /// Return one referenced code-block payload range by traversal index. + #[must_use] + pub fn payload(&self, index: usize) -> Option { + self.plan.payloads().get(index).copied() + } + + /// Iterate over referenced code-block payload ranges in geometry order. + pub fn payloads(&self) -> impl ExactSizeIterator + '_ { + self.plan.payloads().iter().copied() + } + + /// Borrow the immutable backend-specific plan for adapter downcasting. + /// + /// The returned view is tied to `self`, cannot be mutated, and does not + /// expose an owning or raw handle. Backend crates that depend directly on + /// `j2k-native` may downcast it to their supported plan implementation. + #[doc(hidden)] + #[must_use] + pub fn adapter_view(&self) -> &(dyn Any + Send + Sync) { + self.plan.as_ref() + } +} + +fn uniform_wavelet_transform( + tiles: &[j2k_native::J2kReferencedTilePlan], +) -> Option { + let first = tiles.first()?.wavelet_transform(); + tiles + .iter() + .all(|tile| tile.wavelet_transform() == first) + .then_some(first) +} diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index 84a7c30b..d7deb922 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -1,9 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use crate::{ - backend::{ - image as backend_image, inspect_info, inspect_info_from_image, DecodeSettings, Image, - }, + backend::{image as backend_image, inspect_info, inspect_info_from_image, Image}, decode::{ decode_image_into_with_native_context, decode_image_region_into_with_native_context, decode_warnings_for_settings, validate_buffer, validate_region, J2kDecodeOutcome, @@ -11,7 +9,7 @@ use crate::{ }, parse::{parse_image_info, parse_info}, scratch::J2kScratchPool, - CpuDecodeParallelism, J2kError, J2kSupportInfo, + CpuDecodeParallelism, DecodeSettings, J2kError, J2kSupportInfo, }; use j2k_core::{ CompressedPayloadKind, CompressedTransferSyntax, Downscale, Info, PassthroughCandidate, @@ -49,7 +47,7 @@ impl<'a> J2kView<'a> { Err(error) if should_retry_with_backend(&error) => (inspect_info(input)?, None, None), Err(error) => return Err(error), }; - let image = Some(backend_image(input, DecodeSettings::default())?); + let image = Some(backend_image(input, DecodeSettings::default(), None)?); Ok(Self { bytes: input, info, @@ -361,12 +359,9 @@ impl<'a> J2kDecoder<'a> { if scale == Downscale::None { return self.decode_into_with_scratch(pool, out, stride, fmt); } - let settings = DecodeSettings { - target_resolution: Some(self.scaled_target_dims(scale)), - ..DecodeSettings::default() - }; + let settings = DecodeSettings::default(); let warnings = decode_warnings_for_settings(settings); - let image = backend_image(self.bytes, settings)?; + let image = backend_image(self.bytes, settings, Some(self.scaled_target_dims(scale)))?; let image_dims = (image.width(), image.height()); validate_buffer(image_dims, out.len(), stride, fmt)?; let mut native_context = self.scaled_decode_native_context(); @@ -401,12 +396,9 @@ impl<'a> J2kDecoder<'a> { validate_region(roi, self.info.dimensions)?; let scaled_roi = roi.scaled_covering(scale); validate_buffer((scaled_roi.w, scaled_roi.h), out.len(), stride, fmt)?; - let settings = DecodeSettings { - target_resolution: Some(self.scaled_target_dims(scale)), - ..DecodeSettings::default() - }; + let settings = DecodeSettings::default(); let warnings = decode_warnings_for_settings(settings); - let image = backend_image(self.bytes, settings)?; + let image = backend_image(self.bytes, settings, Some(self.scaled_target_dims(scale)))?; let image_dims = (image.width(), image.height()); validate_region(scaled_roi, image_dims)?; let mut native_context = self.scaled_decode_native_context(); @@ -423,7 +415,7 @@ impl<'a> J2kDecoder<'a> { fn ensure_image(&mut self) -> Result<(), J2kError> { if self.image.is_none() { - self.image = Some(backend_image(self.bytes, DecodeSettings::default())?); + self.image = Some(backend_image(self.bytes, DecodeSettings::default(), None)?); if self.info.tile_layout.is_none() { self.info = inspect_info_from_image(self.cached_image()?); } diff --git a/crates/j2k/tests/device_plan.rs b/crates/j2k/tests/device_plan.rs index f62daec3..edd3882b 100644 --- a/crates/j2k/tests/device_plan.rs +++ b/crates/j2k/tests/device_plan.rs @@ -72,3 +72,37 @@ fn invalid_region_is_rejected() { assert!(result.is_err(), "out-of-bounds ROI must be rejected"); } + +#[test] +fn empty_regions_are_rejected_before_decode() { + for roi in [ + Rect { + x: 8, + y: 8, + w: 0, + h: 16, + }, + Rect { + x: 8, + y: 8, + w: 16, + h: 0, + }, + ] { + let full_resolution = + DeviceDecodePlan::for_image((64, 64), DeviceDecodeRequest::Region { roi }); + assert!( + full_resolution.is_err(), + "zero-area full-resolution ROI must be rejected" + ); + + let reduced = DeviceDecodePlan::for_image( + (64, 64), + DeviceDecodeRequest::RegionScaled { + roi, + scale: Downscale::Half, + }, + ); + assert!(reduced.is_err(), "zero-area reduced ROI must be rejected"); + } +} diff --git a/crates/j2k/tests/owned_batch.rs b/crates/j2k/tests/owned_batch.rs new file mode 100644 index 00000000..d7c8c108 --- /dev/null +++ b/crates/j2k/tests/owned_batch.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[path = "owned_batch/fixtures.rs"] +mod fixtures; +#[path = "owned_batch/oracles.rs"] +mod oracles; +#[path = "owned_batch/payload_plan.rs"] +mod payload_plan; + +#[path = "owned_batch/cpu_reuse.rs"] +mod cpu_reuse; +#[path = "owned_batch/errors_and_requests.rs"] +mod errors_and_requests; +#[path = "owned_batch/grouping.rs"] +mod grouping; +#[path = "owned_batch/native_types_and_requests.rs"] +mod native_types_and_requests; +#[path = "owned_batch/payload_ranges.rs"] +mod payload_ranges; +#[path = "owned_batch/reuse.rs"] +mod reuse; +#[path = "owned_batch/rgba.rs"] +mod rgba; diff --git a/crates/j2k/tests/owned_batch/cpu_reuse.rs b/crates/j2k/tests/owned_batch/cpu_reuse.rs new file mode 100644 index 00000000..055cb2db --- /dev/null +++ b/crates/j2k/tests/owned_batch/cpu_reuse.rs @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{num::NonZeroUsize, sync::Arc}; + +use j2k::{ + prepare_batch, BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, + DecodeRequest, Downscale, EncodedImage, PreparationDepth, Rect, +}; +use j2k_native::{encode, EncodeOptions}; + +use super::fixtures::{ + classic_gray8_fixture, classic_gray8_fixture_with_tile_size, htj2k_gray8_fixture, +}; +use super::oracles::{decoded_samples_for_source, native_request_oracle}; +use super::payload_plan::native_prepared_classic_plan; + +#[test] +fn cpu_fast_group_uses_one_flattened_arena_and_one_direct_output_allocation() { + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + ..BatchDecodeOptions::default() + }; + let encoded = Arc::<[u8]>::from(htj2k_gray8_fixture(16, 16)); + let prepared = prepare_batch( + vec![ + EncodedImage::full(Arc::clone(&encoded)), + EncodedImage::full(encoded), + ], + options, + ) + .expect("prepare repeated fast group"); + let payloads_per_image = prepared.groups()[0].images()[0] + .htj2k_plan() + .expect("HT offset plan") + .payload_count(); + let mut session = CpuBatchDecoder::new(options); + + let first = session + .decode_prepared(&prepared) + .expect("first flattened decode"); + let first_stats = session.workspace_stats(); + let second = session + .decode_prepared(&prepared) + .expect("second flattened decode"); + let second_stats = session.workspace_stats(); + + assert!( + first.errors().is_empty(), + "flattened decode errors: {:?}", + first.errors() + ); + assert_eq!(first.groups(), second.groups()); + assert_eq!(first_stats.flattened_group_plans(), 1); + assert_eq!( + first_stats.flattened_payload_jobs(), + 2 * payloads_per_image as u64 + ); + assert_eq!( + first_stats.entropy_job_dispatches(), + 2 * payloads_per_image as u64, + "every flattened code block must be dispatched as scheduler work" + ); + assert_eq!( + first_stats.cross_image_entropy_windows(), + 1, + "the two images must share one entropy-dispatch window" + ); + assert_eq!(first_stats.output_group_allocations(), 1); + assert_eq!(first_stats.output_compaction_copied_samples(), 0); + assert!(first_stats.retained_compressed_arena_bytes() > 0); + assert_eq!(second_stats.flattened_group_plans(), 2); + assert_eq!( + second_stats.entropy_job_dispatches(), + 4 * payloads_per_image as u64 + ); + assert_eq!(second_stats.cross_image_entropy_windows(), 2); + assert_eq!(second_stats.output_group_allocations(), 2); + assert_eq!(second_stats.output_compaction_copied_samples(), 0); + assert_eq!(second_stats.compressed_arena_reuses(), 1); + assert_eq!( + second_stats.retained_compressed_arena_bytes(), + first_stats.retained_compressed_arena_bytes() + ); +} + +#[test] +fn cpu_prepared_ht_plan_decode_avoids_native_reparse() { + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(htj2k_gray8_fixture(16, 16)))], + options, + ) + .expect("prepare parse-free CPU batch"); + assert_eq!( + prepared.groups()[0].images()[0].preparation_depth(), + PreparationDepth::Htj2kOffsetPlan + ); + let mut session = CpuBatchDecoder::new(options); + + let first = session + .decode_prepared(&prepared) + .expect("first parse-free prepared decode"); + let first_stats = session.workspace_stats(); + let second = session + .decode_prepared(&prepared) + .expect("second parse-free prepared decode"); + let second_stats = session.workspace_stats(); + + assert!(first.errors().is_empty()); + assert_eq!(first.groups(), second.groups()); + assert_eq!(first_stats.prepared_plan_decode_calls(), 1); + assert_eq!(second_stats.prepared_plan_decode_calls(), 2); + assert_eq!(first_stats.decode_calls(), 0); + assert_eq!(second_stats.decode_calls(), 0); + assert!(first_stats.retained_prepared_plan_ht_workspace_bytes() > 0); + assert_eq!( + second_stats.retained_prepared_plan_ht_workspace_bytes(), + first_stats.retained_prepared_plan_ht_workspace_bytes() + ); +} + +#[test] +fn cpu_prepared_classic_plan_decode_avoids_native_reparse() { + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(classic_gray8_fixture(16, 16)))], + options, + ) + .expect("prepare parse-free classic CPU batch"); + let image = &prepared.groups()[0].images()[0]; + assert_eq!( + image.preparation_depth(), + PreparationDepth::ClassicOffsetPlan + ); + assert!(image + .classic_plan() + .is_some_and(j2k::PreparedClassicPlan::is_grayscale)); + let mut session = CpuBatchDecoder::new(options); + + let first = session + .decode_prepared(&prepared) + .expect("first parse-free classic prepared decode"); + let first_stats = session.workspace_stats(); + let second = session + .decode_prepared(&prepared) + .expect("second parse-free classic prepared decode"); + let second_stats = session.workspace_stats(); + + assert!( + first.errors().is_empty(), + "classic decode errors: {:?}", + first.errors() + ); + assert_eq!(first.groups(), second.groups()); + assert_eq!(first_stats.prepared_plan_decode_calls(), 1); + assert_eq!(second_stats.prepared_plan_decode_calls(), 2); + assert_eq!(first_stats.decode_calls(), 0); + assert_eq!(second_stats.decode_calls(), 0); + assert!(first_stats.retained_prepared_plan_classic_workspace_bytes() > 0); + assert_eq!( + second_stats.retained_prepared_plan_classic_workspace_bytes(), + first_stats.retained_prepared_plan_classic_workspace_bytes() + ); +} + +#[test] +fn prepared_classic_plan_supports_duplicate_full_roi_and_reduced_requests() { + let encoded = Arc::<[u8]>::from(classic_gray8_fixture(16, 16)); + let roi = Rect { + x: 3, + y: 2, + w: 8, + h: 6, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + + for layout in [BatchLayout::Nchw, BatchLayout::Nhwc] { + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + layout, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + requests + .into_iter() + .map(|request| EncodedImage::new(Arc::clone(&encoded), request)) + .collect(), + options, + ) + .expect("prepare classic request matrix"); + assert!(prepared.errors().is_empty()); + assert!(prepared + .groups() + .iter() + .all(|group| group.images().iter().all(|image| { + image.preparation_depth() == PreparationDepth::ClassicOffsetPlan + && image.classic_plan().is_some() + && Arc::ptr_eq(image.bytes(), &encoded) + }))); + + let mut decoder = CpuBatchDecoder::new(options); + let first = decoder + .decode_prepared(&prepared) + .expect("first classic request matrix decode"); + let second = decoder + .decode_prepared(&prepared) + .expect("second classic request matrix decode"); + assert!(first.errors().is_empty()); + assert_eq!(first.groups(), second.groups()); + assert_eq!(decoder.workspace_stats().decode_calls(), 0); + assert_eq!(decoder.workspace_stats().prepared_plan_decode_calls(), 10); + + for source_index in 0..requests.len() { + let prepared_image = prepared + .groups() + .iter() + .flat_map(j2k::PreparedBatchGroup::images) + .find(|image| image.source_index() == source_index) + .expect("prepared classic request source"); + assert_eq!( + decoded_samples_for_source(&first, source_index), + native_request_oracle(prepared_image, layout), + "classic {layout:?} source={source_index}" + ); + } + } +} + +#[test] +fn referenced_classic_whole_plan_executes_all_gray_and_rgb_tiles_exactly() { + const WIDTH: u32 = 19; + const HEIGHT: u32 = 13; + let gray = Arc::<[u8]>::from(classic_gray8_fixture_with_tile_size( + WIDTH, + HEIGHT, + Some((11, 7)), + )); + let rgb_samples = (0..WIDTH * HEIGHT * 3) + .map(|index| ((index * 37 + index / 7) & 0xff) as u8) + .collect::>(); + let rgb = Arc::<[u8]>::from( + encode( + &rgb_samples, + WIDTH, + HEIGHT, + 3, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + tile_size: Some((11, 7)), + use_mct: false, + ..EncodeOptions::default() + }, + ) + .expect("encode multi-tile classic RGB8"), + ); + + for (encoded, component_count) in [(gray, 1usize), (rgb, 3)] { + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::clone(&encoded))], + BatchDecodeOptions::default(), + ) + .expect("prepare multi-tile classic whole-plan fixture"); + assert!(prepared.errors().is_empty()); + let image = &prepared.groups()[0].images()[0]; + let oracle = native_request_oracle(image, BatchLayout::Nhwc); + let CpuBatchSamples::U8(oracle) = oracle else { + panic!("classic Gray/RGB8 oracle must contain u8 samples") + }; + let plan = + native_prepared_classic_plan(image.classic_plan().expect("referenced classic plan")); + assert!(plan.tiles().len() > 1); + let mut scratch = j2k_native::J2kDirectCpuScratch::new(); + let decoded = + j2k_native::execute_referenced_classic_plan(plan, &encoded, false, &mut scratch) + .expect("execute every referenced classic tile"); + assert_eq!(decoded.dimensions(), (WIDTH, HEIGHT)); + assert_eq!(decoded.component_count(), component_count); + for pixel in 0..(WIDTH * HEIGHT) as usize { + for component in 0..component_count { + let expected = if component_count == 1 { + oracle[pixel] + } else { + oracle[pixel * component_count + component] + }; + let actual = decoded.plane(component).expect("decoded plane").samples()[pixel]; + assert_eq!( + actual.round().clamp(0.0, 255.0).to_bits(), + f32::from(expected).to_bits(), + "pixel {pixel}, component {component}" + ); + } + } + } +} + +#[test] +fn cpu_session_retained_workspace_stabilizes_during_thousand_batch_soak() { + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from( + classic_gray8_fixture_with_tile_size(16, 16, Some((8, 8))), + ))], + options, + ) + .expect("prepare CPU soak batch"); + assert_eq!( + prepared.groups()[0].images()[0].preparation_depth(), + PreparationDepth::ClassicOffsetPlan + ); + let mut session = CpuBatchDecoder::new(options); + + session + .decode_prepared(&prepared) + .expect("warm CPU workspace"); + let warmed = session.workspace_stats(); + for _ in 0..1_000 { + let decoded = session + .decode_prepared(&prepared) + .expect("decode CPU soak batch"); + assert!(decoded.errors().is_empty()); + } + let soaked = session.workspace_stats(); + + assert_eq!(warmed.prepared_plan_decode_calls(), 1); + assert_eq!(soaked.prepared_plan_decode_calls(), 1_001); + assert_eq!(warmed.decode_calls(), 0); + assert_eq!(soaked.decode_calls(), 0); + assert_eq!(warmed.flattened_group_plans(), 1); + assert_eq!(soaked.flattened_group_plans(), 1_001); + assert!(warmed.retained_prepared_plan_classic_workspace_bytes() > 0); + assert_eq!(soaked.scratch_capacity_retries(), 0); + assert_eq!( + soaked.retained_prepared_plan_classic_workspace_bytes(), + warmed.retained_prepared_plan_classic_workspace_bytes() + ); +} diff --git a/crates/j2k/tests/owned_batch/errors_and_requests.rs b/crates/j2k/tests/owned_batch/errors_and_requests.rs new file mode 100644 index 00000000..f20707ab --- /dev/null +++ b/crates/j2k/tests/owned_batch/errors_and_requests.rs @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{num::NonZeroUsize, sync::Arc}; + +use j2k::{ + prepare_batch, wrap_j2k_codestream, BatchDecodeOptions, BatchDecoder, BatchItemError, + BatchLayout, CpuBatchDecoder, CpuBatchSamples, DecodeRequest, Downscale, EncodedImage, + J2kComponentMapping, J2kComponentMappingType, J2kFileBoxMetadata, J2kFileColorSpec, + J2kFileWrapOptions, J2kPaletteColumn, J2kPaletteMetadata, NonRepresentableReason, Rect, +}; +use j2k_core::Colorspace; +use j2k_native::{encode, EncodeOptions}; + +use super::fixtures::{ + four_component_fixture, htj2k_gray8_fixture, rewrite_component_descriptor, rgb8_fixture, + signed_gray16_fixture, +}; + +#[test] +fn shared_batch_decoder_interface_prepares_and_decodes_owned_inputs() { + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + workers: NonZeroUsize::new(1), + ..BatchDecodeOptions::default() + }; + let input = EncodedImage::full(Arc::from(htj2k_gray8_fixture(3, 2))); + let mut decoder = CpuBatchDecoder::new(options); + + let prepared = ::prepare_batch(&decoder, vec![input.clone()]) + .expect("prepare through shared interface"); + assert_eq!(prepared.groups()[0].options().layout, BatchLayout::Nhwc); + + let prepared_output = + ::decode_prepared(&mut decoder, &prepared) + .expect("decode prepared through shared interface"); + let one_shot_output = + ::decode_batch(&mut decoder, vec![input]) + .expect("decode owned input through shared interface"); + assert_eq!(prepared_output.groups(), one_shot_output.groups()); +} + +#[test] +fn prepared_batch_keeps_indexed_preflight_errors_and_decodes_other_inputs() { + let valid = Arc::<[u8]>::from(htj2k_gray8_fixture(3, 2)); + let inputs = vec![ + EncodedImage::full(Arc::clone(&valid)), + EncodedImage::full(Arc::<[u8]>::from([0_u8, 1, 2, 3])), + EncodedImage::new( + valid, + DecodeRequest::Region { + roi: Rect { + x: 1, + y: 0, + w: 2, + h: 2, + }, + }, + ), + ]; + let options = BatchDecodeOptions::default(); + let prepared = prepare_batch(inputs, options).expect("prepare partial batch"); + + assert_eq!(prepared.errors().len(), 1); + assert_eq!(prepared.errors()[0].index, 1); + assert!(matches!( + prepared.errors()[0].source, + BatchItemError::Codec { .. } + )); + + let mut session = CpuBatchDecoder::new(options); + let result = session + .decode_prepared(&prepared) + .expect("decode valid prepared inputs"); + assert_eq!(result.groups().len(), 2); + assert_eq!(result.errors().len(), 1); + assert_eq!(result.errors()[0].index, 1); +} + +#[test] +fn cpu_execution_failure_compacts_and_preserves_successful_group_members() { + let valid = htj2k_gray8_fixture(4, 4); + let valid_prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(valid.clone()))], + BatchDecodeOptions::default(), + ) + .expect("prepare valid HT corruption source"); + let cleanup = valid_prepared.groups()[0].images()[0] + .htj2k_plan() + .and_then(|plan| plan.payload(0)) + .expect("first HT cleanup payload") + .cleanup; + let cleanup_end = cleanup.end().expect("cleanup payload end"); + let mut corrupted = valid.clone(); + corrupted[cleanup.offset..cleanup_end].fill(0xff); + let options = BatchDecodeOptions::default(); + let prepared = prepare_batch( + vec![ + EncodedImage::full(Arc::<[u8]>::from(valid.clone())), + EncodedImage::full(Arc::<[u8]>::from(corrupted)), + EncodedImage::full(Arc::<[u8]>::from(valid)), + ], + options, + ) + .expect("prepare header-valid group"); + + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 1); + let mut session = CpuBatchDecoder::new(options); + let result = session + .decode_prepared(&prepared) + .expect("complete batch execution"); + + assert_eq!(result.groups().len(), 1); + assert_eq!(result.groups()[0].source_indices(), &[0, 2]); + let CpuBatchSamples::U8(samples) = result.groups()[0].samples() else { + panic!("expected Gray8 samples") + }; + assert_eq!(samples.len(), 2 * 4 * 4); + let expected = (0_u8..16).collect::>(); + assert_eq!(&samples[..expected.len()], &expected); + assert_eq!(&samples[expected.len()..], &expected); + assert_eq!(result.errors().len(), 1); + assert_eq!(result.errors()[0].index, 1); +} + +#[test] +fn cpu_batch_preserves_signed_i16_samples_in_nchw_layout() { + let first = [-300_i16, -1, 0, 300]; + let second = [511_i16, -512, 12, -12]; + let inputs = vec![ + EncodedImage::full(Arc::<[u8]>::from(signed_gray16_fixture(&first, 2, 2))), + EncodedImage::full(Arc::<[u8]>::from(signed_gray16_fixture(&second, 2, 2))), + ]; + let options = BatchDecodeOptions { + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let mut session = CpuBatchDecoder::new(options); + let result = session.decode(inputs).expect("decode signed batch"); + + assert!(result.errors().is_empty()); + assert_eq!(result.groups().len(), 1); + assert_eq!(result.groups()[0].source_indices(), &[0, 1]); + let CpuBatchSamples::I16(samples) = result.groups()[0].samples() else { + panic!("expected i16 samples") + }; + assert_eq!(samples, &[first, second].concat()); +} + +#[test] +fn reduced_and_region_reduced_requests_group_by_decoded_geometry() { + let bytes = Arc::<[u8]>::from(htj2k_gray8_fixture(7, 5)); + let prepared = prepare_batch( + vec![ + EncodedImage::new( + Arc::clone(&bytes), + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + ), + EncodedImage::new( + bytes, + DecodeRequest::RegionReduced { + roi: Rect { + x: 1, + y: 1, + w: 5, + h: 3, + }, + scale: Downscale::Half, + }, + ), + ], + BatchDecodeOptions::default(), + ) + .expect("prepare reduced requests"); + + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 2); + assert_eq!(prepared.groups()[0].info().dimensions, (4, 3)); + assert_eq!(prepared.groups()[1].info().dimensions, (3, 2)); + + let mut session = CpuBatchDecoder::new(BatchDecodeOptions::default()); + let result = session + .decode_prepared(&prepared) + .expect("decode reduced requests"); + assert!(result.errors().is_empty()); + assert_eq!(result.groups()[0].samples().len(), 12); + assert_eq!(result.groups()[1].samples().len(), 6); +} + +#[test] +fn prepare_reports_nonrepresentable_subsampled_mixed_and_high_precision_inputs() { + let mut subsampled = rgb8_fixture(); + let siz_marker = subsampled + .windows(2) + .position(|marker| marker == [0xff, 0x51]) + .expect("SIZ marker"); + subsampled[siz_marker + 44] = 2; + + let mut mixed = rgb8_fixture(); + rewrite_component_descriptor(&mut mixed, 1, 11); + + let mut high_precision = rgb8_fixture(); + for component in 0..3 { + rewrite_component_descriptor(&mut high_precision, component, 16); + } + + let prepared = prepare_batch( + vec![subsampled, mixed, high_precision] + .into_iter() + .map(|bytes| EncodedImage::full(Arc::<[u8]>::from(bytes))) + .collect(), + BatchDecodeOptions::default(), + ) + .expect("prepare rejected profiles"); + + assert!(prepared.groups().is_empty()); + assert_eq!(prepared.errors().len(), 3); + let reasons = prepared + .errors() + .iter() + .map(|error| match error.source { + BatchItemError::NonRepresentableBatchOutput { reason } => reason, + _ => panic!("expected representability error"), + }) + .collect::>(); + assert_eq!( + reasons, + [ + NonRepresentableReason::ComponentSubsampling, + NonRepresentableReason::MixedPrecision, + NonRepresentableReason::PrecisionAboveSixteen, + ] + ); +} + +#[test] +fn prepare_rejects_raw_four_component_data_without_explicit_alpha_metadata() { + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(four_component_fixture()))], + BatchDecodeOptions::default(), + ) + .expect("prepare raw four-component input"); + + assert!(prepared.groups().is_empty()); + assert_eq!(prepared.errors().len(), 1); + assert!(matches!( + prepared.errors()[0].source, + BatchItemError::NonRepresentableBatchOutput { + reason: NonRepresentableReason::UnsupportedColor + } + )); +} + +#[test] +fn prepare_rejects_palette_mapped_wrapper_before_pixel_decode() { + let codestream = encode( + &[0_u8, 1, 1, 0], + 2, + 2, + 1, + 8, + false, + &EncodeOptions::default(), + ) + .expect("encode palette indices"); + let palette = J2kPaletteMetadata { + columns: vec![ + J2kPaletteColumn { + bit_depth: 8, + signed: false, + }, + J2kPaletteColumn { + bit_depth: 8, + signed: false, + }, + J2kPaletteColumn { + bit_depth: 8, + signed: false, + }, + ], + entries: vec![vec![10, 20, 30], vec![200, 210, 220]], + }; + let mappings = [0_u8, 1, 2].map(|column| J2kComponentMapping { + component_index: 0, + mapping_type: J2kComponentMappingType::Palette { column }, + }); + let wrapped = wrap_j2k_codestream( + &codestream, + J2kFileWrapOptions::jp2() + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: Some(&palette), + component_mappings: &mappings, + channel_definitions: &[], + }), + ) + .expect("wrap palette image"); + + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(wrapped))], + BatchDecodeOptions::default(), + ) + .expect("prepare palette image"); + + assert!(prepared.groups().is_empty()); + assert!(matches!( + prepared.errors()[0].source, + BatchItemError::NonRepresentableBatchOutput { + reason: NonRepresentableReason::UnsupportedColor + } + )); +} diff --git a/crates/j2k/tests/owned_batch/fixtures.rs b/crates/j2k/tests/owned_batch/fixtures.rs new file mode 100644 index 00000000..739d3b49 --- /dev/null +++ b/crates/j2k/tests/owned_batch/fixtures.rs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::{ + wrap_j2k_codestream, J2kChannelAssociation, J2kChannelDefinition, J2kChannelType, + J2kFileBoxMetadata, J2kFileColorSpec, J2kFileWrapOptions, +}; +use j2k_core::Colorspace; +use j2k_native::{encode, encode_htj2k, EncodeOptions}; +use j2k_test_support::Htj2kRgbaAlpha; + +pub(super) fn htj2k_gray8_fixture(width: u32, height: u32) -> Vec { + htj2k_gray8_fixture_with_levels(width, height, 1) +} + +pub(super) fn classic_gray8_fixture(width: u32, height: u32) -> Vec { + classic_gray8_fixture_with_tile_size(width, height, None) +} + +pub(super) fn classic_gray8_fixture_with_tile_size( + width: u32, + height: u32, + tile_size: Option<(u32, u32)>, +) -> Vec { + let pixels = (0..width * height) + .map(|index| (index & 0xff) as u8) + .collect::>(); + encode( + &pixels, + width, + height, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + tile_size, + ..EncodeOptions::default() + }, + ) + .expect("encode classic J2K gray8") +} + +pub(super) fn htj2k_gray8_fixture_with_levels( + width: u32, + height: u32, + num_decomposition_levels: u8, +) -> Vec { + let pixels = (0..width * height) + .map(|index| (index & 0xff) as u8) + .collect::>(); + encode_htj2k( + &pixels, + width, + height, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels, + ..EncodeOptions::default() + }, + ) + .expect("encode HTJ2K gray8") +} + +pub(super) fn rewrite_component_descriptor(bytes: &mut [u8], component: usize, descriptor: u8) { + let siz_marker = bytes + .windows(2) + .position(|marker| marker == [0xff, 0x51]) + .expect("SIZ marker"); + bytes[siz_marker + 40 + component * 3] = descriptor; +} + +pub(super) fn rgb8_fixture() -> Vec { + let pixels = (0_u8..48).collect::>(); + encode( + &pixels, + 4, + 4, + 3, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + .expect("encode RGB8") +} + +pub(super) fn four_component_fixture() -> Vec { + let pixels = (0_u8..64).collect::>(); + encode( + &pixels, + 4, + 4, + 4, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + use_mct: false, + ..EncodeOptions::default() + }, + ) + .expect("encode four-component image") +} + +fn rgba_channel_definitions(alpha_type: J2kChannelType) -> [J2kChannelDefinition; 4] { + [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 3, + channel_type: alpha_type, + association: J2kChannelAssociation::WholeImage, + }, + ] +} + +pub(super) fn wrap_rgba_jph(codestream: &[u8], alpha: Htj2kRgbaAlpha) -> Vec { + let alpha_type = match alpha { + Htj2kRgbaAlpha::Straight => J2kChannelType::Opacity, + Htj2kRgbaAlpha::Premultiplied => J2kChannelType::PremultipliedOpacity, + }; + let channel_definitions = rgba_channel_definitions(alpha_type); + wrap_j2k_codestream( + codestream, + J2kFileWrapOptions::jph() + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &channel_definitions, + }), + ) + .expect("wrap explicit HTJ2K RGBA image") +} + +pub(super) fn signed_gray16_fixture(samples: &[i16], width: u32, height: u32) -> Vec { + let bytes = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + encode( + &bytes, + width, + height, + 1, + 16, + true, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + use_mct: false, + ..EncodeOptions::default() + }, + ) + .expect("encode signed gray16") +} + +pub(super) fn htj2k_native_fixture( + components: u16, + precision: u8, + signed: bool, + width: u32, + height: u32, +) -> Vec { + let sample_count = width as usize * height as usize * components as usize; + let bytes: Vec = if signed { + let magnitude = 1_i32 << u32::from(precision - 1); + (0..sample_count) + .flat_map(|index| { + let value = (i32::try_from(index).expect("small fixture") * 193 + 17) + % (magnitude * 2) + - magnitude; + i16::try_from(value) + .expect("signed fixture value") + .to_le_bytes() + }) + .collect() + } else if precision <= 8 { + (0..sample_count) + .map(|index| { + u8::try_from((index * 47 + 13) & 0xff).expect("masked 8-bit fixture value") + }) + .collect() + } else { + let modulus = 1_u32 << u32::from(precision); + (0..sample_count) + .flat_map(|index| { + u16::try_from((u32::try_from(index).expect("small fixture") * 977 + 31) % modulus) + .expect("unsigned fixture value") + .to_le_bytes() + }) + .collect() + }; + encode_htj2k( + &bytes, + width, + height, + components, + precision, + signed, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + use_mct: false, + ..EncodeOptions::default() + }, + ) + .expect("encode native HTJ2K request fixture") +} diff --git a/crates/j2k/tests/owned_batch/grouping.rs b/crates/j2k/tests/owned_batch/grouping.rs new file mode 100644 index 00000000..5f62931e --- /dev/null +++ b/crates/j2k/tests/owned_batch/grouping.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{prepare_batch, BatchDecodeOptions, EncodedImage}; + +use super::fixtures::htj2k_gray8_fixture_with_levels; + +#[test] +fn grouping_separates_incompatible_backend_execution_shapes() { + let prepared = prepare_batch( + vec![ + EncodedImage::full(Arc::from(htj2k_gray8_fixture_with_levels(16, 16, 1))), + EncodedImage::full(Arc::from(htj2k_gray8_fixture_with_levels(16, 16, 2))), + ], + BatchDecodeOptions::default(), + ) + .expect("prepare differing execution shapes"); + + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 2); + assert_eq!(prepared.groups()[0].source_indices(), &[0]); + assert_eq!(prepared.groups()[1].source_indices(), &[1]); +} diff --git a/crates/j2k/tests/owned_batch/native_types_and_requests.rs b/crates/j2k/tests/owned_batch/native_types_and_requests.rs new file mode 100644 index 00000000..32ba5b74 --- /dev/null +++ b/crates/j2k/tests/owned_batch/native_types_and_requests.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{num::NonZeroUsize, sync::Arc}; + +use j2k::{ + BatchDecodeOptions, BatchLayout, CpuBatchDecoder, DecodeRequest, Downscale, EncodedImage, + NativeSampleType, PreparationDepth, Rect, +}; + +use super::fixtures::htj2k_native_fixture; +use super::oracles::{decoded_samples_for_source, native_request_oracle}; + +#[test] +fn prepared_htj2k_gray_and_rgb_support_native_types_and_requests_exactly() { + const WIDTH: u32 = 8; + const HEIGHT: u32 = 6; + let roi = Rect { + x: 1, + y: 1, + w: 5, + h: 3, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + + for components in [1_u16, 3] { + for (precision, signed, sample_type) in [ + (8, false, NativeSampleType::U8), + (12, false, NativeSampleType::U16), + (16, true, NativeSampleType::I16), + ] { + let encoded = Arc::<[u8]>::from(htj2k_native_fixture( + components, precision, signed, WIDTH, HEIGHT, + )); + for layout in [BatchLayout::Nchw, BatchLayout::Nhwc] { + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + layout, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBatchDecoder::new(options); + let prepared = decoder + .prepare( + requests + .into_iter() + .map(|request| EncodedImage::new(Arc::clone(&encoded), request)) + .collect(), + ) + .expect("prepare Gray/RGB request matrix"); + assert!(prepared.errors().is_empty()); + assert!(prepared.groups().iter().all(|group| { + group.info().sample_type == sample_type + && group.info().color.channels() == components as usize + && group.images().iter().all(|image| { + image.preparation_depth() == PreparationDepth::Htj2kOffsetPlan + }) + })); + + let result = decoder + .decode_prepared(&prepared) + .expect("decode Gray/RGB request matrix"); + assert!(result.errors().is_empty()); + for source_index in 0..requests.len() { + let prepared_image = prepared + .groups() + .iter() + .flat_map(j2k::PreparedBatchGroup::images) + .find(|image| image.source_index() == source_index) + .expect("prepared Gray/RGB source"); + assert_eq!( + decoded_samples_for_source(&result, source_index), + native_request_oracle(prepared_image, layout), + "components={components} precision={precision} signed={signed} {layout:?} source={source_index}" + ); + } + let stats = decoder.workspace_stats(); + assert_eq!(stats.preparation_calls(), requests.len() as u64); + assert_eq!(stats.prepared_plan_decode_calls(), requests.len() as u64); + assert_eq!(stats.flattened_group_plans(), requests.len() as u64); + assert_eq!(stats.output_group_allocations(), requests.len() as u64); + assert_eq!(stats.output_compaction_copied_samples(), 0); + } + } + } +} diff --git a/crates/j2k/tests/owned_batch/oracles.rs b/crates/j2k/tests/owned_batch/oracles.rs new file mode 100644 index 00000000..74bfd607 --- /dev/null +++ b/crates/j2k/tests/owned_batch/oracles.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::{BatchLayout, CpuBatchSamples, Downscale, Rect}; + +pub(super) fn native_request_oracle( + image: &j2k::PreparedImage, + layout: BatchLayout, +) -> CpuBatchSamples { + let plan = image.plan(); + let target_resolution = (plan.scale() != Downscale::None).then_some(( + plan.source_dims().0.div_ceil(plan.scale().denominator()), + plan.source_dims().1.div_ceil(plan.scale().denominator()), + )); + let decoded = j2k_native::Image::new( + image.bytes(), + &j2k_native::DecodeSettings { + target_resolution, + ..j2k_native::DecodeSettings::strict() + }, + ) + .expect("parse broad native RGBA oracle"); + let output = plan.output_rect(); + let raw = if output == Rect::full((decoded.width(), decoded.height())) { + decoded.decode_native().expect("decode broad native oracle") + } else { + decoded + .decode_native_region((output.x, output.y, output.w, output.h)) + .expect("decode broad native region oracle") + }; + let pixel_count = output.w as usize * output.h as usize; + let channels = raw.num_components as usize; + if raw.signed { + CpuBatchSamples::I16(apply_batch_layout( + raw.data + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]])) + .collect(), + pixel_count, + channels, + layout, + )) + } else if raw.bit_depth <= 8 { + CpuBatchSamples::U8(apply_batch_layout(raw.data, pixel_count, channels, layout)) + } else { + CpuBatchSamples::U16(apply_batch_layout( + raw.data + .chunks_exact(2) + .map(|sample| u16::from_le_bytes([sample[0], sample[1]])) + .collect(), + pixel_count, + channels, + layout, + )) + } +} + +pub(super) fn apply_batch_layout( + samples: Vec, + pixel_count: usize, + channels: usize, + layout: BatchLayout, +) -> Vec { + match layout { + BatchLayout::Nhwc => samples, + BatchLayout::Nchw => (0..channels) + .flat_map(|channel| { + let samples = &samples; + (0..pixel_count).map(move |pixel| samples[pixel * channels + channel]) + }) + .collect(), + _ => panic!("unsupported test batch layout"), + } +} + +pub(super) fn decoded_samples_for_source( + result: &j2k::CpuBatchDecodeResult, + source_index: usize, +) -> CpuBatchSamples { + let group = result + .groups() + .iter() + .find(|group| group.source_indices().contains(&source_index)) + .expect("decoded source group"); + let image_position = group + .source_indices() + .iter() + .position(|index| *index == source_index) + .expect("source position inside decoded group"); + let image_count = group.source_indices().len(); + match group.samples() { + CpuBatchSamples::U8(samples) => { + let samples_per_image = samples.len() / image_count; + let start = image_position * samples_per_image; + CpuBatchSamples::U8(samples[start..start + samples_per_image].to_vec()) + } + CpuBatchSamples::U16(samples) => { + let samples_per_image = samples.len() / image_count; + let start = image_position * samples_per_image; + CpuBatchSamples::U16(samples[start..start + samples_per_image].to_vec()) + } + CpuBatchSamples::I16(samples) => { + let samples_per_image = samples.len() / image_count; + let start = image_position * samples_per_image; + CpuBatchSamples::I16(samples[start..start + samples_per_image].to_vec()) + } + _ => panic!("unsupported test batch sample type"), + } +} diff --git a/crates/j2k/tests/owned_batch/payload_plan.rs b/crates/j2k/tests/owned_batch/payload_plan.rs new file mode 100644 index 00000000..5d4821eb --- /dev/null +++ b/crates/j2k/tests/owned_batch/payload_plan.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{prepare_batch, BatchDecodeOptions, EncodedImage}; + +pub(super) fn native_prepared_plan( + plan: &j2k::PreparedHtj2kPlan, +) -> &j2k_native::J2kReferencedHtj2kPlan { + plan.adapter_view() + .downcast_ref::() + .expect("j2k-native prepared-plan adapter") +} + +pub(super) fn native_prepared_classic_plan( + plan: &j2k::PreparedClassicPlan, +) -> &j2k_native::J2kReferencedClassicPlan { + plan.adapter_view() + .downcast_ref::() + .expect("j2k-native prepared classic-plan adapter") +} + +pub(super) fn assert_prepared_ht_payload_ranges_reconstruct_owned_bytes(bytes: Vec) { + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::<[u8]>::from(bytes))], + BatchDecodeOptions::default(), + ) + .expect("prepare HTJ2K offset plan"); + let prepared_image = &prepared.groups()[0].images()[0]; + let referenced = prepared_image.htj2k_plan().expect("referenced HTJ2K plan"); + let geometry = native_prepared_plan(referenced) + .grayscale_geometry() + .expect("grayscale referenced geometry"); + let native_image = j2k_native::Image::new( + prepared_image.bytes(), + &j2k_native::DecodeSettings::strict(), + ) + .expect("parse owned HTJ2K plan source"); + let mut context = j2k_native::DecoderContext::default(); + let owned = native_image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("build owned HTJ2K direct plan"); + let mut payload_cursor = 0usize; + + for (owned_step, referenced_step) in owned.steps.iter().zip(&geometry.steps) { + let ( + j2k_native::J2kDirectGrayscaleStep::HtSubBand(owned_sub_band), + j2k_native::J2kDirectGrayscaleStep::HtSubBand(referenced_sub_band), + ) = (owned_step, referenced_step) + else { + continue; + }; + for (owned_job, referenced_job) in owned_sub_band.jobs.iter().zip(&referenced_sub_band.jobs) + { + assert!(referenced_job.data.is_empty()); + let payload = referenced + .payload(payload_cursor) + .expect("payload range for referenced HT job"); + payload_cursor += 1; + let cleanup_end = payload.cleanup.end().expect("cleanup range end"); + let mut reconstructed = prepared_image + .bytes() + .get(payload.cleanup.offset..cleanup_end) + .expect("cleanup range inside retained encoded owner") + .to_vec(); + if let Some(refinement) = payload.refinement { + let refinement_end = refinement.end().expect("refinement range end"); + reconstructed.extend_from_slice( + prepared_image + .bytes() + .get(refinement.offset..refinement_end) + .expect("refinement range inside retained encoded owner"), + ); + } + assert_eq!(reconstructed, owned_job.data); + } + } + assert_eq!(payload_cursor, referenced.payload_count()); +} + +pub(super) fn assert_prepared_classic_payload_ranges_reconstruct_owned_bytes(bytes: Vec) { + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::<[u8]>::from(bytes))], + BatchDecodeOptions::default(), + ) + .expect("prepare classic offset plan"); + let prepared_image = &prepared.groups()[0].images()[0]; + let referenced = prepared_image + .classic_plan() + .expect("referenced classic plan"); + let geometry = native_prepared_classic_plan(referenced) + .grayscale_geometry() + .expect("grayscale referenced classic geometry"); + let native_image = j2k_native::Image::new( + prepared_image.bytes(), + &j2k_native::DecodeSettings::strict(), + ) + .expect("parse owned classic plan source"); + let mut context = j2k_native::DecoderContext::default(); + let owned = native_image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("build owned classic direct plan"); + let mut payload_cursor = 0usize; + + for (owned_step, referenced_step) in owned.steps.iter().zip(&geometry.steps) { + let ( + j2k_native::J2kDirectGrayscaleStep::ClassicSubBand(owned_sub_band), + j2k_native::J2kDirectGrayscaleStep::ClassicSubBand(referenced_sub_band), + ) = (owned_step, referenced_step) + else { + continue; + }; + for (owned_job, referenced_job) in owned_sub_band.jobs.iter().zip(&referenced_sub_band.jobs) + { + assert_eq!(referenced_job.data.capacity(), 0); + assert_eq!(referenced_job.segments, owned_job.segments); + let payload = referenced + .payload(payload_cursor) + .expect("payload descriptor for referenced classic job"); + payload_cursor += 1; + let end_range = payload.end_range().expect("classic fragment span end"); + let mut reconstructed = Vec::with_capacity(payload.combined_length); + for range_index in payload.first_range..end_range { + let range = referenced + .range(range_index) + .expect("classic fragment range"); + let range_end = range.end().expect("classic fragment range end"); + reconstructed.extend_from_slice( + prepared_image + .bytes() + .get(range.offset..range_end) + .expect("classic fragment inside retained encoded owner"), + ); + } + assert_eq!(reconstructed.len(), payload.combined_length); + assert_eq!(reconstructed, owned_job.data); + } + } + assert_eq!(payload_cursor, referenced.payload_count()); + assert_eq!( + referenced + .payloads() + .map(|payload| payload.range_count) + .sum::(), + referenced.range_count() + ); +} diff --git a/crates/j2k/tests/owned_batch/payload_ranges.rs b/crates/j2k/tests/owned_batch/payload_ranges.rs new file mode 100644 index 00000000..b211a244 --- /dev/null +++ b/crates/j2k/tests/owned_batch/payload_ranges.rs @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::sync::Arc; + +use j2k::{ + prepare_batch, wrap_j2k_codestream, BatchDecodeOptions, EncodedImage, J2kFileWrapOptions, + PreparationDepth, +}; +use j2k_native::{encode, EncodeOptions}; + +use super::fixtures::{classic_gray8_fixture, htj2k_gray8_fixture}; +use super::payload_plan::{ + assert_prepared_classic_payload_ranges_reconstruct_owned_bytes, + assert_prepared_ht_payload_ranges_reconstruct_owned_bytes, native_prepared_plan, +}; + +#[test] +fn prepared_htj2k_uses_codestream_ranges_without_retaining_payload_copies() { + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::<[u8]>::from(htj2k_gray8_fixture( + 8, 8, + )))], + BatchDecodeOptions::default(), + ) + .expect("prepare HTJ2K offset plan"); + let image = &prepared.groups()[0].images()[0]; + + assert_eq!(image.preparation_depth(), PreparationDepth::Htj2kOffsetPlan); + let codestream = image.codestream_range(); + assert_eq!(codestream.offset, 0); + assert_eq!(codestream.length, image.bytes().len()); + let plan = image.htj2k_plan().expect("HTJ2K reference plan"); + assert!(!plan.is_empty()); + assert!(plan.payloads().all(|payload| { + payload + .cleanup + .end() + .is_some_and(|end| end <= codestream.length) + && payload + .refinement + .is_none_or(|range| range.end().is_some_and(|end| end <= codestream.length)) + })); + let geometry = native_prepared_plan(plan) + .grayscale_geometry() + .expect("grayscale geometry"); + let retained_payload_bytes = geometry + .steps + .iter() + .filter_map(|step| match step { + j2k_native::J2kDirectGrayscaleStep::HtSubBand(sub_band) => Some( + sub_band + .jobs + .iter() + .map(|job| job.data.capacity()) + .sum::(), + ), + _ => None, + }) + .sum::(); + assert_eq!(retained_payload_bytes, 0); +} + +#[test] +fn prepared_raw_htj2k_payload_ranges_reconstruct_original_job_bytes() { + assert_prepared_ht_payload_ranges_reconstruct_owned_bytes(htj2k_gray8_fixture(8, 8)); +} + +#[test] +fn prepared_jph_payload_ranges_reconstruct_original_job_bytes() { + let codestream = htj2k_gray8_fixture(8, 8); + let wrapped = wrap_j2k_codestream(&codestream, J2kFileWrapOptions::jph()).expect("wrap JPH"); + assert_prepared_ht_payload_ranges_reconstruct_owned_bytes(wrapped); +} + +#[test] +fn prepared_raw_classic_payload_ranges_reconstruct_original_job_bytes() { + assert_prepared_classic_payload_ranges_reconstruct_owned_bytes(classic_gray8_fixture(8, 8)); +} + +#[test] +fn prepared_jp2_classic_payload_ranges_reconstruct_original_job_bytes() { + let codestream = classic_gray8_fixture(8, 8); + let wrapped = wrap_j2k_codestream(&codestream, J2kFileWrapOptions::jp2()).expect("wrap JP2"); + assert_prepared_classic_payload_ranges_reconstruct_owned_bytes(wrapped); +} + +#[test] +fn prepared_raw_jp2_and_jph_codestream_ranges_reconstruct_original_codestreams() { + let classic = encode( + &(0_u8..64).collect::>(), + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + .expect("encode classic J2K fixture"); + let ht = htj2k_gray8_fixture(8, 8); + let inputs = [ + classic.clone(), + wrap_j2k_codestream(&classic, J2kFileWrapOptions::jp2()).expect("wrap JP2"), + wrap_j2k_codestream(&ht, J2kFileWrapOptions::jph()).expect("wrap JPH"), + ]; + let expected = [&classic[..], &classic[..], &ht[..]]; + + for (bytes, expected_codestream) in inputs.into_iter().zip(expected) { + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::<[u8]>::from(bytes))], + BatchDecodeOptions::default(), + ) + .expect("prepare encoded owner"); + let image = &prepared.groups()[0].images()[0]; + let range = image.codestream_range(); + let end = range.end().expect("codestream range end"); + assert_eq!( + image + .bytes() + .get(range.offset..end) + .expect("codestream range inside retained encoded owner"), + expected_codestream + ); + } +} + +#[test] +fn prepared_jph_payload_ranges_resolve_inside_the_original_owned_bytes() { + let codestream = htj2k_gray8_fixture(8, 8); + let wrapped = wrap_j2k_codestream(&codestream, J2kFileWrapOptions::jph()).expect("wrap JPH"); + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::<[u8]>::from(wrapped))], + BatchDecodeOptions::default(), + ) + .expect("prepare JPH offset plan"); + let image = &prepared.groups()[0].images()[0]; + let codestream_range = image.codestream_range(); + let plan = image.htj2k_plan().expect("JPH HTJ2K reference plan"); + + assert!(codestream_range.offset > 0); + assert!(plan.payloads().all(|payload| { + payload + .cleanup + .end() + .is_some_and(|absolute_end| absolute_end <= image.bytes().len()) + })); +} diff --git a/crates/j2k/tests/owned_batch/reuse.rs b/crates/j2k/tests/owned_batch/reuse.rs new file mode 100644 index 00000000..e3899b3c --- /dev/null +++ b/crates/j2k/tests/owned_batch/reuse.rs @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{num::NonZeroUsize, sync::Arc}; + +use j2k::{ + prepare_batch, BatchCodecRoute, BatchDecodeOptions, BatchItemError, BatchLayout, + CpuBatchDecoder, CpuBatchSamples, DecodeSettings, EncodedImage, NativeSampleType, + PreparationDepth, +}; + +use super::fixtures::{classic_gray8_fixture_with_tile_size, htj2k_gray8_fixture}; + +#[test] +fn prepared_batch_is_reusable_and_groups_heterogeneous_images_without_padding() { + let gray_4x3 = Arc::<[u8]>::from(htj2k_gray8_fixture(4, 3)); + let gray_2x2 = Arc::<[u8]>::from(htj2k_gray8_fixture(2, 2)); + let inputs = vec![ + EncodedImage::full(Arc::clone(&gray_4x3)), + EncodedImage::full(gray_2x2), + EncodedImage::full(gray_4x3), + ]; + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + workers: NonZeroUsize::new(2), + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch(inputs, options).expect("prepare batch"); + + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 2); + assert_eq!(prepared.groups()[0].source_indices(), &[0, 2]); + assert_eq!(prepared.groups()[1].source_indices(), &[1]); + assert_eq!(prepared.groups()[0].options().layout, options.layout); + assert_eq!( + prepared.groups()[0] + .options() + .settings + .lenient_tolerance_enabled(), + options.settings.lenient_tolerance_enabled() + ); + assert_eq!( + prepared.groups()[0].info().sample_type, + NativeSampleType::U8 + ); + assert_eq!(prepared.groups()[0].info().route, BatchCodecRoute::Htj2k); + + let mut session = CpuBatchDecoder::new(options); + let first = session + .decode_prepared(&prepared) + .expect("first prepared decode"); + let second = session + .decode_prepared(&prepared) + .expect("second prepared decode"); + + assert!( + first.errors().is_empty(), + "decode errors: {:?}", + first.errors() + ); + assert_eq!(first.groups(), second.groups()); + let CpuBatchSamples::U8(samples) = first.groups()[0].samples() else { + panic!("expected u8 samples") + }; + let expected = (0_u8..12).collect::>(); + assert_eq!(&samples[..expected.len()], expected); + assert_eq!(&samples[expected.len()..], expected); +} + +#[test] +fn caller_supplied_prepared_images_regroup_in_new_submission_order_without_reparse() { + let preparation_options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + ..BatchDecodeOptions::default() + }; + let larger_bytes = Arc::<[u8]>::from(htj2k_gray8_fixture(4, 3)); + let larger_preparation = prepare_batch( + vec![ + EncodedImage::full(Arc::from([0_u8, 1, 2, 3])), + EncodedImage::full(Arc::clone(&larger_bytes)), + ], + preparation_options, + ) + .expect("prepare larger image after an indexed failure"); + let larger = larger_preparation.groups()[0].images()[0].clone(); + assert_eq!(larger.source_index(), 1); + + let smaller_preparation = prepare_batch( + vec![EncodedImage::full(Arc::from(htj2k_gray8_fixture(2, 2)))], + preparation_options, + ) + .expect("prepare smaller image"); + let smaller = smaller_preparation.groups()[0].images()[0].clone(); + + let decode_options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + workers: NonZeroUsize::new(2), + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBatchDecoder::new(decode_options); + let regrouped = decoder + .prepare_prepared_images(vec![larger.clone(), smaller, larger]) + .expect("regroup caller-supplied prepared images"); + + assert!(regrouped.errors().is_empty()); + assert_eq!(regrouped.groups().len(), 2); + assert_eq!(regrouped.groups()[0].source_indices(), [0, 2]); + assert_eq!(regrouped.groups()[1].source_indices(), [1]); + assert_eq!(regrouped.groups()[0].info().layout, BatchLayout::Nhwc); + assert_eq!( + regrouped.groups()[0] + .images() + .iter() + .map(j2k::PreparedImage::source_index) + .collect::>(), + [1, 1] + ); + assert!(regrouped.groups()[0] + .images() + .iter() + .all(|image| Arc::ptr_eq(image.bytes(), &larger_bytes))); + + let batch_output = decoder + .decode_prepared(®rouped) + .expect("decode regrouped prepared images"); + assert!(batch_output.errors().is_empty()); + assert_eq!(batch_output.groups()[0].source_indices(), [0, 2]); + assert_eq!(batch_output.groups()[1].source_indices(), [1]); + let CpuBatchSamples::U8(samples) = batch_output.groups()[0].samples() else { + panic!("expected native u8 samples") + }; + let expected = (0_u8..12).collect::>(); + assert_eq!(samples, &[expected.clone(), expected].concat()); +} + +#[test] +fn regrouping_prepared_images_reports_settings_mismatches_at_submission_indices() { + let lenient_options = BatchDecodeOptions { + settings: DecodeSettings::lenient(), + ..BatchDecodeOptions::default() + }; + let lenient = prepare_batch( + vec![EncodedImage::full(Arc::from(htj2k_gray8_fixture(3, 2)))], + lenient_options, + ) + .expect("prepare lenient image") + .groups()[0] + .images()[0] + .clone(); + let strict = prepare_batch( + vec![EncodedImage::full(Arc::from(htj2k_gray8_fixture(2, 2)))], + BatchDecodeOptions::default(), + ) + .expect("prepare strict image") + .groups()[0] + .images()[0] + .clone(); + let mut decoder = CpuBatchDecoder::new(BatchDecodeOptions::default()); + + let regrouped = decoder + .prepare_prepared_images(vec![lenient, strict]) + .expect("regroup remains an infrastructure success"); + + assert_eq!(regrouped.errors().len(), 1); + assert_eq!(regrouped.errors()[0].index, 0); + assert!(matches!( + regrouped.errors()[0].source, + BatchItemError::PreparedDecodeSettingsMismatch { + prepared, + requested, + } if prepared == DecodeSettings::lenient() && requested == DecodeSettings::strict() + )); + assert_eq!(regrouped.groups().len(), 1); + assert_eq!(regrouped.groups()[0].source_indices(), [1]); + + let batch_output = decoder + .decode_prepared(®rouped) + .expect("decode compatible regrouped image"); + assert_eq!(batch_output.errors().len(), 1); + assert_eq!(batch_output.errors()[0].index, 0); + assert_eq!(batch_output.groups()[0].source_indices(), [1]); +} + +#[test] +fn cpu_session_reuses_multitile_classic_prepared_workspace_across_decodes() { + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from( + classic_gray8_fixture_with_tile_size(16, 16, Some((8, 8))), + ))], + options, + ) + .expect("prepare reusable CPU batch"); + assert_eq!( + prepared.groups()[0].images()[0].preparation_depth(), + PreparationDepth::ClassicOffsetPlan + ); + let mut session = CpuBatchDecoder::new(options); + + let first_output = session + .decode_prepared(&prepared) + .expect("first prepared decode"); + let first = session.workspace_stats(); + let second_output = session + .decode_prepared(&prepared) + .expect("second prepared decode"); + let second = session.workspace_stats(); + + assert!(first_output.errors().is_empty()); + assert_eq!(first_output.groups(), second_output.groups()); + assert_eq!(first.prepared_plan_decode_calls(), 1); + assert_eq!(second.prepared_plan_decode_calls(), 2); + assert_eq!(first.decode_calls(), 0); + assert_eq!(second.decode_calls(), 0); + assert_eq!(second.scratch_capacity_retries(), 0); + assert!(first.retained_prepared_plan_classic_workspace_bytes() > 0); + assert_eq!( + second.retained_prepared_plan_classic_workspace_bytes(), + first.retained_prepared_plan_classic_workspace_bytes() + ); +} + +#[test] +fn cpu_session_reuses_preparation_workers_across_one_shot_batches() { + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + ..BatchDecodeOptions::default() + }; + let encoded = Arc::<[u8]>::from(htj2k_gray8_fixture(16, 16)); + let session = CpuBatchDecoder::new(options); + + let first = session + .prepare(vec![EncodedImage::full(Arc::clone(&encoded))]) + .expect("first retained preparation"); + assert!(first.errors().is_empty()); + let first_stats = session.workspace_stats(); + + let second = session + .prepare(vec![EncodedImage::full(encoded)]) + .expect("second retained preparation"); + assert!(second.errors().is_empty()); + let second_stats = session.workspace_stats(); + + assert_eq!(first_stats.preparation_calls(), 1); + assert_eq!(first_stats.preparation_worker_reuses(), 0); + assert_eq!(second_stats.preparation_calls(), 2); + assert_eq!(second_stats.preparation_worker_reuses(), 1); +} diff --git a/crates/j2k/tests/owned_batch/rgba.rs b/crates/j2k/tests/owned_batch/rgba.rs new file mode 100644 index 00000000..0414be46 --- /dev/null +++ b/crates/j2k/tests/owned_batch/rgba.rs @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{num::NonZeroUsize, sync::Arc}; + +use j2k::{ + prepare_batch, wrap_j2k_codestream, BatchAlpha, BatchDecodeOptions, BatchItemError, + BatchLayout, CpuBatchDecoder, CpuBatchSamples, DecodeRequest, Downscale, EncodedImage, + J2kChannelAssociation, J2kChannelDefinition, J2kChannelType, J2kFileBoxMetadata, + J2kFileColorSpec, J2kFileWrapOptions, NativeSampleType, NonRepresentableReason, + PreparationDepth, Rect, +}; +use j2k_core::Colorspace; +use j2k_test_support::{ + generated_htj2k_rgba_fixture, Htj2kRgbaAlpha, Htj2kRgbaSampleProfile, Htj2kRgbaSamples, +}; + +use super::fixtures::{four_component_fixture, rgb8_fixture, wrap_rgba_jph}; +use super::oracles::{apply_batch_layout, native_request_oracle}; +use super::payload_plan::native_prepared_plan; + +#[test] +fn prepare_accepts_identity_rgb_cdef_with_explicit_alpha() { + let codestream = four_component_fixture(); + let channels = [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 3, + channel_type: J2kChannelType::Opacity, + association: J2kChannelAssociation::WholeImage, + }, + ]; + let wrapped = wrap_j2k_codestream( + &codestream, + J2kFileWrapOptions::jp2() + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &channels, + }), + ) + .expect("wrap explicit RGBA image"); + + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(wrapped))], + BatchDecodeOptions::default(), + ) + .expect("prepare explicit RGBA image"); + + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups()[0].info().color.channels(), 4); + assert_eq!(prepared.groups()[0].info().alpha, BatchAlpha::Straight); + let prepared_image = &prepared.groups()[0].images()[0]; + assert_eq!( + prepared_image.preparation_depth(), + PreparationDepth::ClassicOffsetPlan + ); + assert!(prepared_image + .classic_plan() + .is_some_and(j2k::PreparedClassicPlan::is_rgba)); + + let mut session = CpuBatchDecoder::new(BatchDecodeOptions::default()); + let output = session + .decode_prepared(&prepared) + .expect("decode explicit RGBA image"); + assert!(output.errors().is_empty()); + let CpuBatchSamples::U8(samples) = output.groups()[0].samples() else { + panic!("expected native RGBA8 samples") + }; + let expected = (0_u8..4) + .flat_map(|channel| (0_u8..16).map(move |pixel| pixel * 4 + channel)) + .collect::>(); + assert_eq!(samples, &expected); + assert_eq!(session.workspace_stats().decode_calls(), 0); + assert_eq!(session.workspace_stats().prepared_plan_decode_calls(), 1); +} + +#[test] +fn prepare_preserves_and_groups_straight_and_premultiplied_alpha_separately() { + let codestream = four_component_fixture(); + let rgba_definitions = |alpha_type| { + [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 3, + channel_type: alpha_type, + association: J2kChannelAssociation::WholeImage, + }, + ] + }; + let wrap = |alpha_type| { + let definitions = rgba_definitions(alpha_type); + wrap_j2k_codestream( + &codestream, + J2kFileWrapOptions::jp2() + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &definitions, + }), + ) + .expect("wrap RGBA image") + }; + + let prepared = prepare_batch( + vec![ + EncodedImage::full(Arc::from(wrap(J2kChannelType::Opacity))), + EncodedImage::full(Arc::from(wrap(J2kChannelType::PremultipliedOpacity))), + ], + BatchDecodeOptions::default(), + ) + .expect("prepare alpha interpretations"); + + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 2); + assert_eq!(prepared.groups()[0].source_indices(), &[0]); + assert_eq!(prepared.groups()[0].info().alpha, BatchAlpha::Straight); + assert_eq!(prepared.groups()[1].source_indices(), &[1]); + assert_eq!(prepared.groups()[1].info().alpha, BatchAlpha::Premultiplied); +} + +#[test] +fn prepared_htj2k_rgba_preserves_alpha_semantics_and_avoids_reparse() { + let fixture = + generated_htj2k_rgba_fixture(Htj2kRgbaSampleProfile::U8Rct, Htj2kRgbaAlpha::Straight); + let codestream = fixture.encoded; + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + layout: BatchLayout::Nchw, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![ + EncodedImage::full(Arc::from(wrap_rgba_jph( + &codestream, + Htj2kRgbaAlpha::Straight, + ))), + EncodedImage::full(Arc::from(wrap_rgba_jph( + &codestream, + Htj2kRgbaAlpha::Premultiplied, + ))), + ], + options, + ) + .expect("prepare retained RGBA HTJ2K plans"); + + assert!(prepared.errors().is_empty()); + assert_eq!(prepared.groups().len(), 2); + assert_eq!(prepared.groups()[0].info().alpha, BatchAlpha::Straight); + assert_eq!(prepared.groups()[1].info().alpha, BatchAlpha::Premultiplied); + assert!(prepared.groups().iter().all(|group| { + let image = &group.images()[0]; + image.preparation_depth() == PreparationDepth::Htj2kOffsetPlan + && image.htj2k_plan().is_some_and(|plan| { + plan.is_rgba() + && !plan.is_grayscale() + && !plan.is_color() + && native_prepared_plan(plan) + .rgba_geometry() + .is_some_and(|geometry| { + geometry.component_plans.len() == 4 + && geometry.bit_depths == [8; 4] + && geometry.mct + }) + }) + })); + + let mut decoder = CpuBatchDecoder::new(options); + let first = decoder + .decode_prepared(&prepared) + .expect("first parse-free RGBA decode"); + let first_stats = decoder.workspace_stats(); + let second = decoder + .decode_prepared(&prepared) + .expect("second parse-free RGBA decode"); + let second_stats = decoder.workspace_stats(); + + assert!(first.errors().is_empty()); + assert_eq!(first.groups(), second.groups()); + let Htj2kRgbaSamples::U8(source_samples) = fixture.samples else { + panic!("shared U8 RGBA fixture must retain U8 source samples") + }; + let expected_source = CpuBatchSamples::U8(apply_batch_layout( + source_samples, + fixture.width as usize * fixture.height as usize, + 4, + options.layout, + )); + for group in first.groups() { + let source_index = group.source_indices()[0]; + let prepared_image = prepared + .groups() + .iter() + .flat_map(j2k::PreparedBatchGroup::images) + .find(|image| image.source_index() == source_index) + .expect("prepared RGBA source"); + assert_eq!( + group.samples(), + &native_request_oracle(prepared_image, options.layout) + ); + assert_eq!(group.samples(), &expected_source); + } + assert_eq!(first_stats.prepared_plan_decode_calls(), 2); + assert_eq!(second_stats.prepared_plan_decode_calls(), 4); + assert_eq!(first_stats.decode_calls(), 0); + assert_eq!(second_stats.decode_calls(), 0); + assert!(first_stats.retained_prepared_plan_ht_workspace_bytes() > 0); + assert_eq!( + second_stats.retained_prepared_plan_ht_workspace_bytes(), + first_stats.retained_prepared_plan_ht_workspace_bytes() + ); +} + +#[test] +fn prepared_htj2k_rgba_supports_native_types_requests_and_layouts_exactly() { + let profiles = [ + (Htj2kRgbaSampleProfile::U8Rct, NativeSampleType::U8), + (Htj2kRgbaSampleProfile::U12, NativeSampleType::U16), + (Htj2kRgbaSampleProfile::I16, NativeSampleType::I16), + ]; + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let requests = [ + DecodeRequest::Full, + DecodeRequest::Region { roi }, + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ]; + + for (profile, sample_type) in profiles { + let fixture = generated_htj2k_rgba_fixture(profile, Htj2kRgbaAlpha::Straight); + let bit_depth = fixture.bit_depth; + let signed = fixture.signed; + let encoded = Arc::<[u8]>::from(wrap_rgba_jph(&fixture.encoded, fixture.alpha)); + for layout in [BatchLayout::Nchw, BatchLayout::Nhwc] { + let options = BatchDecodeOptions { + workers: NonZeroUsize::new(1), + layout, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + requests + .into_iter() + .map(|request| EncodedImage::new(Arc::clone(&encoded), request)) + .collect(), + options, + ) + .expect("prepare RGBA request matrix"); + assert!( + prepared.errors().is_empty(), + "{bit_depth}-bit signed={signed} {layout:?}: {:?}", + prepared.errors() + ); + assert!(prepared.groups().iter().all(|group| { + group.info().sample_type == sample_type + && group.info().color.channels() == 4 + && group.images().iter().all(|image| { + image.preparation_depth() == PreparationDepth::Htj2kOffsetPlan + && image.htj2k_plan().is_some() + }) + })); + + let mut decoder = CpuBatchDecoder::new(options); + let result = decoder + .decode_prepared(&prepared) + .expect("decode RGBA request matrix"); + assert!(result.errors().is_empty()); + assert_eq!(decoder.workspace_stats().decode_calls(), 0); + assert_eq!(decoder.workspace_stats().prepared_plan_decode_calls(), 4); + + for source_index in 0..requests.len() { + let prepared_image = prepared + .groups() + .iter() + .flat_map(j2k::PreparedBatchGroup::images) + .find(|image| image.source_index() == source_index) + .expect("prepared RGBA request source"); + let group = result + .groups() + .iter() + .find(|group| group.source_indices() == [source_index]) + .expect("decoded RGBA request group"); + assert_eq!(group.samples().sample_type(), sample_type); + assert_eq!( + group.samples(), + &native_request_oracle(prepared_image, layout), + "{bit_depth}-bit signed={signed} {layout:?} source={source_index}" + ); + } + } + } +} + +#[test] +fn prepare_rejects_icc_tagged_and_reordering_cdef_outputs() { + let rgb = rgb8_fixture(); + let icc = wrap_j2k_codestream( + &rgb, + J2kFileWrapOptions::jp2().with_color(J2kFileColorSpec::IccProfile(b"test-profile")), + ) + .expect("wrap ICC-tagged RGB"); + let reordered = [ + J2kChannelDefinition { + channel_index: 0, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 3 }, + }, + J2kChannelDefinition { + channel_index: 1, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 2 }, + }, + J2kChannelDefinition { + channel_index: 2, + channel_type: J2kChannelType::Color, + association: J2kChannelAssociation::Color { index: 1 }, + }, + ]; + let cdef = wrap_j2k_codestream( + &rgb, + J2kFileWrapOptions::jp2() + .with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)) + .with_metadata(J2kFileBoxMetadata { + palette: None, + component_mappings: &[], + channel_definitions: &reordered, + }), + ) + .expect("wrap reordered CDEF RGB"); + + let prepared = prepare_batch( + vec![icc, cdef] + .into_iter() + .map(|bytes| EncodedImage::full(Arc::from(bytes))) + .collect(), + BatchDecodeOptions::default(), + ) + .expect("prepare unsupported color metadata"); + + assert!(prepared.groups().is_empty()); + assert_eq!(prepared.errors().len(), 2); + assert!(prepared.errors().iter().all(|error| matches!( + error.source, + BatchItemError::NonRepresentableBatchOutput { + reason: NonRepresentableReason::UnsupportedColor + } + ))); +} diff --git a/crates/j2k/tests/owned_batch_fixtures.rs b/crates/j2k/tests/owned_batch_fixtures.rs new file mode 100644 index 00000000..0c331de3 --- /dev/null +++ b/crates/j2k/tests/owned_batch_fixtures.rs @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::collections::BTreeSet; +use std::sync::Arc; + +use j2k::{ + encode_j2k_lossless, prepare_batch, wrap_j2k_codestream, BatchCodecRoute, BatchDecodeOptions, + BatchLayout, BatchWaveletTransform, CompressedPayloadKind, CompressedTransferSyntax, + CpuBatchDecoder, CpuBatchGroup, CpuBatchSamples, DecodeRequest, Downscale, EncodedImage, + J2kDecoder, J2kFileWrapOptions, J2kLosslessEncodeOptions, J2kLosslessSamples, J2kScratchPool, + NativeSampleType, PixelFormat, Rect, +}; +use j2k_native::{ + encode, encode_htj2k, EncodeOptions, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, +}; +use j2k_test_support::{ + openhtj2k_refinement_fixture, openhtj2k_refinement_odd_fixture, + openhtj2k_refinement_odd_pixels, openhtj2k_refinement_pixels, openhtj2k_sigprop_fixture, + openhtj2k_sigprop_overlap_fixture, openhtj2k_sigprop_overlap_pixels, + openhtj2k_sigprop_pixels_le, openjph_batch_fixtures, OpenJphBatchFixture, +}; + +#[derive(Clone, Copy, Debug)] +enum CodingRoute { + Classic, + Htj2k, +} + +#[derive(Debug)] +enum NativeOracle { + U8(Vec), + U16(Vec), + I16(Vec), +} + +fn fixture_oracle(fixture: OpenJphBatchFixture) -> NativeOracle { + if fixture.signed { + if fixture.precision <= 8 { + NativeOracle::I16( + fixture + .oracle + .iter() + .map(|sample| i16::from(i8::from_ne_bytes([*sample]))) + .collect(), + ) + } else { + NativeOracle::I16( + fixture + .oracle + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]])) + .collect(), + ) + } + } else if fixture.precision <= 8 { + NativeOracle::U8(fixture.oracle.to_vec()) + } else { + NativeOracle::U16( + fixture + .oracle + .chunks_exact(2) + .map(|sample| u16::from_le_bytes([sample[0], sample[1]])) + .collect(), + ) + } +} + +#[test] +fn independent_openjph_batch_matrix_preserves_native_samples_and_metadata() { + let fixtures = openjph_batch_fixtures(); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBatchDecoder::new(options); + let result = decoder + .decode( + fixtures + .iter() + .map(|fixture| EncodedImage::full(Arc::from(fixture.encoded))) + .collect(), + ) + .expect("decode independent OpenJPH batch matrix"); + + assert!( + result.errors().is_empty(), + "OpenJPH matrix errors: {:?}", + result.errors() + ); + assert_eq!(result.groups().len(), fixtures.len()); + for (source_index, fixture) in fixtures.iter().copied().enumerate() { + let group = result + .groups() + .iter() + .find(|group| group.source_indices() == [source_index]) + .unwrap_or_else(|| panic!("{}: output group", fixture.name)); + assert_eq!( + group.info().route, + BatchCodecRoute::Htj2k, + "{}", + fixture.name + ); + assert_eq!( + group.info().dimensions, + (fixture.width, fixture.height), + "{}", + fixture.name + ); + assert_eq!( + group.info().precision, + fixture.precision, + "{}", + fixture.name + ); + assert_eq!(group.info().signed, fixture.signed, "{}", fixture.name); + assert_eq!( + group.info().color.channels(), + fixture.components, + "{}", + fixture.name + ); + assert_eq!( + group.info().transform, + if fixture.reversible { + BatchWaveletTransform::Reversible53 + } else { + BatchWaveletTransform::Irreversible97 + }, + "{}", + fixture.name + ); + assert_eq!( + group.info().payload_kind, + if fixture.jph { + CompressedPayloadKind::JphFile + } else { + CompressedPayloadKind::Jpeg2000Codestream + }, + "{}", + fixture.name + ); + + let expected = fixture_oracle(fixture); + if fixture.reversible { + expected.assert_samples(group.samples(), fixture.name); + } else { + expected.assert_within_one_lsb(group.samples(), fixture.name); + } + } +} + +#[test] +fn independent_openjph_signed_rgb_component_planes_preserve_negative_samples() { + let fixture = openjph_batch_fixtures() + .iter() + .find(|fixture| fixture.name == "openjph-rgb-s8-53-raw") + .expect("signed RGB8 OpenJPH fixture"); + let NativeOracle::I16(expected) = fixture_oracle(*fixture) else { + panic!("signed RGB8 fixture oracle must be i16") + }; + let mut decoder = J2kDecoder::new(fixture.encoded).expect("OpenJPH component decoder"); + let components = decoder + .decode_components() + .expect("decode OpenJPH component planes"); + + assert_eq!(components.planes().len(), 3); + for (channel, plane) in components.planes().iter().enumerate() { + assert!(plane.signed()); + let expected = expected + .iter() + .skip(channel) + .step_by(3) + .copied() + .collect::>(); + #[expect( + clippy::cast_possible_truncation, + reason = "the decoded fixture plane is bounded by its asserted signed 8-bit metadata" + )] + let actual = plane + .samples() + .iter() + .map(|sample| sample.round() as i16) + .collect::>(); + assert_eq!(actual, expected, "component {channel}"); + } +} + +impl NativeOracle { + fn assert_samples(&self, actual: &CpuBatchSamples, name: &str) { + match (self, actual) { + (Self::U8(expected), CpuBatchSamples::U8(actual)) => { + assert_eq!(actual, expected, "{name}: native u8 samples"); + } + (Self::U16(expected), CpuBatchSamples::U16(actual)) => { + assert_eq!(actual, expected, "{name}: native u16 samples"); + } + (Self::I16(expected), CpuBatchSamples::I16(actual)) => { + assert_eq!(actual, expected, "{name}: native i16 samples"); + } + _ => panic!("{name}: batch returned the wrong native sample owner"), + } + } + + fn assert_within_one_lsb(&self, actual: &CpuBatchSamples, name: &str) { + let (lengths_match, (max_difference, max_index, actual_at_max, expected_at_max)) = + match (self, actual) { + (Self::U8(expected), CpuBatchSamples::U8(actual)) => ( + actual.len() == expected.len(), + actual + .iter() + .zip(expected) + .enumerate() + .map(|(index, (actual, expected))| { + ( + u16::from(actual.abs_diff(*expected)), + index, + i32::from(*actual), + i32::from(*expected), + ) + }) + .max() + .unwrap_or((0, 0, 0, 0)), + ), + (Self::U16(expected), CpuBatchSamples::U16(actual)) => ( + actual.len() == expected.len(), + actual + .iter() + .zip(expected) + .enumerate() + .map(|(index, (actual, expected))| { + ( + actual.abs_diff(*expected), + index, + i32::from(*actual), + i32::from(*expected), + ) + }) + .max() + .unwrap_or((0, 0, 0, 0)), + ), + (Self::I16(expected), CpuBatchSamples::I16(actual)) => ( + actual.len() == expected.len(), + actual + .iter() + .zip(expected) + .enumerate() + .map(|(index, (actual, expected))| { + ( + actual.abs_diff(*expected), + index, + i32::from(*actual), + i32::from(*expected), + ) + }) + .max() + .unwrap_or((0, 0, 0, 0)), + ), + _ => (false, (u16::MAX, 0, 0, 0)), + }; + assert!( + lengths_match && max_difference <= 1, + "{name}: irreversible reconstruction has max OpenJPH difference {max_difference} LSB at sample {max_index}: codec={actual_at_max}, OpenJPH={expected_at_max}; or returned the wrong native sample owner" + ); + } +} + +#[derive(Debug)] +struct NativeCase { + name: String, + encoded: Arc<[u8]>, + components: usize, + precision: u8, + signed: bool, + route: BatchCodecRoute, + oracle: NativeOracle, +} + +fn encode_case(route: CodingRoute, components: usize, precision: u8, signed: bool) -> NativeCase { + const WIDTH: u32 = 5; + const HEIGHT: u32 = 3; + let sample_count = WIDTH as usize * HEIGHT as usize * components; + let (bytes, oracle) = if signed { + let modulus = 1_i32 << precision; + let midpoint = modulus / 2; + let samples = (0..sample_count) + .map(|index| { + let raw = (i32::try_from(index).expect("small fixture index") * 83 + 19) % modulus; + i16::try_from(raw - midpoint).expect("fixture stays in i16") + }) + .collect::>(); + let bytes = if precision <= 8 { + samples + .iter() + .map(|sample| sample.to_le_bytes()[0]) + .collect() + } else { + samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect() + }; + (bytes, NativeOracle::I16(samples)) + } else if precision <= 8 { + let modulus = 1_u32 << precision; + let samples = (0..sample_count) + .map(|index| { + u8::try_from( + (u32::try_from(index).expect("small fixture index") * 37 + 11) % modulus, + ) + .expect("8-bit fixture sample") + }) + .collect::>(); + (samples.clone(), NativeOracle::U8(samples)) + } else { + let modulus = 1_u32 << precision; + let samples = (0..sample_count) + .map(|index| { + u16::try_from( + (u32::try_from(index).expect("small fixture index") * 977 + 31) % modulus, + ) + .expect("16-bit fixture sample") + }) + .collect::>(); + let bytes = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect(); + (bytes, NativeOracle::U16(samples)) + }; + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + use_mct: false, + ..EncodeOptions::default() + }; + let components_u16 = u16::try_from(components).expect("fixture component count"); + let encoded = match route { + CodingRoute::Classic => encode( + &bytes, + WIDTH, + HEIGHT, + components_u16, + precision, + signed, + &options, + ) + .expect("encode classic matrix fixture"), + CodingRoute::Htj2k => encode_htj2k( + &bytes, + WIDTH, + HEIGHT, + components_u16, + precision, + signed, + &options, + ) + .expect("encode HTJ2K matrix fixture"), + }; + let route_name = match route { + CodingRoute::Classic => "classic", + CodingRoute::Htj2k => "htj2k", + }; + NativeCase { + name: format!( + "{route_name}-{}-{sign}{precision}", + if components == 1 { "gray" } else { "rgb" }, + sign = if signed { "s" } else { "u" }, + ), + encoded: Arc::from(encoded), + components, + precision, + signed, + route: match route { + CodingRoute::Classic => BatchCodecRoute::Classic, + CodingRoute::Htj2k => BatchCodecRoute::Htj2k, + }, + oracle, + } +} + +#[path = "owned_batch_fixtures/classic.rs"] +mod classic; +#[path = "owned_batch_fixtures/ht_matrix.rs"] +mod ht_matrix; +#[path = "owned_batch_fixtures/irreversible.rs"] +mod irreversible; diff --git a/crates/j2k/tests/owned_batch_fixtures/classic.rs b/crates/j2k/tests/owned_batch_fixtures/classic.rs new file mode 100644 index 00000000..66b4ae00 --- /dev/null +++ b/crates/j2k/tests/owned_batch_fixtures/classic.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn classic_raw_and_jp2_regression_outputs_are_bit_exact() { + const WIDTH: u32 = 11; + const HEIGHT: u32 = 7; + let pixels = (0..WIDTH * HEIGHT) + .map(|index| ((index * 29 + 7) & 0xff) as u8) + .collect::>(); + let raw = encode( + &pixels, + WIDTH, + HEIGHT, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }, + ) + .expect("encode classic raw fixture"); + let jp2 = wrap_j2k_codestream(&raw, J2kFileWrapOptions::jp2()).expect("wrap classic JP2"); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBatchDecoder::new(options); + let result = decoder + .decode(vec![ + EncodedImage::full(Arc::from(raw)), + EncodedImage::full(Arc::from(jp2)), + ]) + .expect("decode classic raw/JP2 batch"); + + assert!(result.errors().is_empty()); + assert_eq!(result.groups().len(), 2); + for (index, payload_kind) in [ + (0, CompressedPayloadKind::Jpeg2000Codestream), + (1, CompressedPayloadKind::Jp2File), + ] { + let group = result + .groups() + .iter() + .find(|group| group.source_indices() == [index]) + .expect("classic wrapper group"); + assert_eq!(group.info().payload_kind, payload_kind); + assert_eq!(group.info().route, BatchCodecRoute::Classic); + assert_eq!(group.info().transform, BatchWaveletTransform::Reversible53); + let CpuBatchSamples::U8(samples) = group.samples() else { + panic!("classic fixture must retain u8 samples") + }; + assert_eq!(samples, &pixels); + } +} + +fn u16_from_le_bytes(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(2) + .map(|bytes| u16::from_le_bytes([bytes[0], bytes[1]])) + .collect() +} + +fn assert_full_and_roi_u16( + groups: &[CpuBatchGroup], + samples: &[u16], + dimensions: (u32, u32), + roi: Rect, +) { + let (width, height) = dimensions; + let full = groups + .iter() + .find(|group| group.source_indices() == [0]) + .expect("full group"); + assert_eq!( + full.decoded_rects(), + [Rect { + x: 0, + y: 0, + w: width, + h: height, + }] + ); + let CpuBatchSamples::U16(full_samples) = full.samples() else { + panic!("full Gray12 group must retain u16 samples") + }; + assert_eq!(full_samples, samples); + + let region = groups + .iter() + .find(|group| group.source_indices() == [1]) + .expect("ROI group"); + assert_eq!(region.decoded_rects(), [roi]); + let CpuBatchSamples::U16(region_samples) = region.samples() else { + panic!("ROI Gray12 group must retain u16 samples") + }; + let expected_region = (roi.y..roi.y + roi.h) + .flat_map(|y| { + let start = (y * width + roi.x) as usize; + samples[start..start + roi.w as usize].iter().copied() + }) + .collect::>(); + assert_eq!(region_samples, &expected_region); +} + +fn assert_reduced_u16(groups: &[CpuBatchGroup], encoded: &[u8], dimensions: (u32, u32), roi: Rect) { + let reduced_dimensions = (dimensions.0.div_ceil(2), dimensions.1.div_ceil(2)); + let mut reduced_bytes = + vec![0_u8; reduced_dimensions.0 as usize * reduced_dimensions.1 as usize * 2]; + J2kDecoder::new(encoded) + .expect("scalar reduced decoder") + .decode_scaled_into( + &mut J2kScratchPool::new(), + &mut reduced_bytes, + reduced_dimensions.0 as usize * 2, + PixelFormat::Gray16, + Downscale::Half, + ) + .expect("scalar reduced oracle"); + let reduced = groups + .iter() + .find(|group| group.source_indices() == [2]) + .expect("reduced group"); + let CpuBatchSamples::U16(reduced_samples) = reduced.samples() else { + panic!("reduced Gray12 group must retain u16 samples") + }; + assert_eq!(reduced_samples, &u16_from_le_bytes(&reduced_bytes)); + + let scaled_roi = roi.scaled_covering(Downscale::Half); + let mut region_reduced_bytes = vec![0_u8; scaled_roi.w as usize * scaled_roi.h as usize * 2]; + let outcome = J2kDecoder::new(encoded) + .expect("scalar region-reduced decoder") + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut region_reduced_bytes, + scaled_roi.w as usize * 2, + PixelFormat::Gray16, + roi, + Downscale::Half, + ) + .expect("scalar region-reduced oracle"); + assert_eq!(outcome.decoded, scaled_roi); + let region_reduced = groups + .iter() + .find(|group| group.source_indices() == [3]) + .expect("region-reduced group"); + assert_eq!(region_reduced.decoded_rects(), [scaled_roi]); + let CpuBatchSamples::U16(region_reduced_samples) = region_reduced.samples() else { + panic!("region-reduced Gray12 group must retain u16 samples") + }; + assert_eq!( + region_reduced_samples, + &u16_from_le_bytes(®ion_reduced_bytes) + ); +} + +#[test] +fn multitile_multilevel_classic_batch_supports_full_roi_and_reduction() { + const WIDTH: u32 = 96; + const HEIGHT: u32 = 80; + let samples = (0..WIDTH * HEIGHT) + .map(|index| ((index * 613 + index / 11) & 0x0fff) as u16) + .collect::>(); + let bytes = samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect::>(); + let source = J2kLosslessSamples::new(&bytes, WIDTH, HEIGHT, 1, 12, false) + .expect("multi-tile Gray12 source"); + let encoded = Arc::<[u8]>::from( + encode_j2k_lossless( + source, + &J2kLosslessEncodeOptions::default() + .with_cpu_only_backend() + .with_max_decomposition_levels(Some(3)) + .with_tile_size(Some((48, 40))), + ) + .expect("encode odd multi-tile classic fixture") + .codestream, + ); + let roi = Rect { + x: 5, + y: 7, + w: 41, + h: 29, + }; + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBatchDecoder::new(options); + let result = decoder + .decode(vec![ + EncodedImage::full(Arc::clone(&encoded)), + EncodedImage::new(Arc::clone(&encoded), DecodeRequest::Region { roi }), + EncodedImage::new( + Arc::clone(&encoded), + DecodeRequest::Reduced { + scale: Downscale::Half, + }, + ), + EncodedImage::new( + Arc::clone(&encoded), + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ), + ]) + .expect("decode odd classic request matrix"); + + assert!( + result.errors().is_empty(), + "request errors: {:?}", + result.errors() + ); + assert_eq!(result.groups().len(), 4); + assert!(result + .groups() + .iter() + .all(|group| group.info().route == BatchCodecRoute::Classic)); + assert_full_and_roi_u16(result.groups(), &samples, (WIDTH, HEIGHT), roi); + assert_reduced_u16(result.groups(), &encoded, (WIDTH, HEIGHT), roi); +} diff --git a/crates/j2k/tests/owned_batch_fixtures/ht_matrix.rs b/crates/j2k/tests/owned_batch_fixtures/ht_matrix.rs new file mode 100644 index 00000000..5bcbe2f2 --- /dev/null +++ b/crates/j2k/tests/owned_batch_fixtures/ht_matrix.rs @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn reversible_batch_matrix_preserves_native_gray_and_rgb_samples() { + let mut cases = Vec::new(); + for route in [CodingRoute::Classic, CodingRoute::Htj2k] { + for components in [1, 3] { + for signed in [false, true] { + for precision in [8, 12, 16] { + cases.push(encode_case(route, components, precision, signed)); + } + } + } + } + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBatchDecoder::new(options); + let result = decoder + .decode( + cases + .iter() + .map(|case| EncodedImage::full(Arc::clone(&case.encoded))) + .collect(), + ) + .expect("decode native sample matrix"); + + assert!( + result.errors().is_empty(), + "matrix errors: {:?}", + result.errors() + ); + assert_eq!(result.groups().len(), cases.len()); + for (source_index, case) in cases.iter().enumerate() { + let group = result + .groups() + .iter() + .find(|group| group.source_indices() == [source_index]) + .unwrap_or_else(|| panic!("{}: output group", case.name)); + assert_eq!(group.info().precision, case.precision, "{}", case.name); + assert_eq!(group.info().signed, case.signed, "{}", case.name); + assert_eq!( + group.info().color.channels(), + case.components, + "{}", + case.name + ); + assert_eq!(group.info().route, case.route, "{}", case.name); + assert_eq!( + group.info().transform, + BatchWaveletTransform::Reversible53, + "{}", + case.name + ); + assert_eq!( + group.info().sample_type, + if case.signed { + NativeSampleType::I16 + } else if case.precision <= 8 { + NativeSampleType::U8 + } else { + NativeSampleType::U16 + }, + "{}", + case.name + ); + case.oracle.assert_samples(group.samples(), &case.name); + } +} + +#[test] +fn independent_openhtj2k_raw_and_derived_jph_outputs_are_exact_and_indexed() { + let raw = openhtj2k_refinement_fixture(); + let expected = openhtj2k_refinement_pixels(); + // The codestream and oracle are independent OpenHT/OpenJPH artifacts; only + // the JPH container around those bytes is constructed by this repository. + let jph = wrap_j2k_codestream(raw, J2kFileWrapOptions::jph()).expect("wrap OpenHT fixture"); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBatchDecoder::new(options); + let result = decoder + .decode(vec![ + EncodedImage::full(Arc::from(raw)), + EncodedImage::full(Arc::from([0_u8, 1, 2, 3])), + EncodedImage::full(Arc::from(raw)), + EncodedImage::full(Arc::from(jph)), + ]) + .expect("decode raw Part 15 and JPH batch"); + + assert_eq!(result.errors().len(), 1); + assert_eq!(result.errors()[0].index, 1); + assert_eq!(result.groups().len(), 2); + let raw_group = result + .groups() + .iter() + .find(|group| group.info().payload_kind == CompressedPayloadKind::Jpeg2000Codestream) + .expect("raw Part 15 group"); + assert_eq!(raw_group.source_indices(), [0, 2]); + assert_eq!(raw_group.info().route, BatchCodecRoute::Htj2k); + assert_eq!( + raw_group.info().transform, + BatchWaveletTransform::Reversible53 + ); + assert_eq!(raw_group.info().precision, 8); + assert!(!raw_group.info().signed); + let CpuBatchSamples::U8(raw_samples) = raw_group.samples() else { + panic!("OpenHT raw group must retain u8 samples") + }; + assert_eq!(raw_samples, &[expected, expected].concat()); + + let jph_group = result + .groups() + .iter() + .find(|group| group.info().payload_kind == CompressedPayloadKind::JphFile) + .expect("JPH group"); + assert_eq!(jph_group.source_indices(), [3]); + let CpuBatchSamples::U8(jph_samples) = jph_group.samples() else { + panic!("OpenHT JPH group must retain u8 samples") + }; + assert_eq!(jph_samples, expected); +} + +#[test] +fn independent_odd_openhtj2k_fixture_supports_roi_and_reduction() { + const WIDTH: u32 = 17; + const HEIGHT: u32 = 37; + let encoded = Arc::<[u8]>::from(openhtj2k_refinement_odd_fixture()); + let expected = openhtj2k_refinement_odd_pixels(); + let roi = Rect { + x: 3, + y: 5, + w: 9, + h: 21, + }; + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBatchDecoder::new(options); + let result = decoder + .decode(vec![ + EncodedImage::full(Arc::clone(&encoded)), + EncodedImage::new(Arc::clone(&encoded), DecodeRequest::Region { roi }), + EncodedImage::new( + Arc::clone(&encoded), + DecodeRequest::RegionReduced { + roi, + scale: Downscale::Half, + }, + ), + ]) + .expect("decode independent odd OpenHT request matrix"); + + assert!( + result.errors().is_empty(), + "request errors: {:?}", + result.errors() + ); + assert_eq!(result.groups().len(), 3); + assert!(result + .groups() + .iter() + .all(|group| group.info().route == BatchCodecRoute::Htj2k)); + + let full = result + .groups() + .iter() + .find(|group| group.source_indices() == [0]) + .expect("full independent group"); + assert_eq!( + full.decoded_rects(), + [Rect { + x: 0, + y: 0, + w: WIDTH, + h: HEIGHT, + }] + ); + let CpuBatchSamples::U8(full_samples) = full.samples() else { + panic!("independent odd full group must retain u8 samples") + }; + assert_eq!(full_samples, expected); + + let region = result + .groups() + .iter() + .find(|group| group.source_indices() == [1]) + .expect("independent ROI group"); + assert_eq!(region.decoded_rects(), [roi]); + let CpuBatchSamples::U8(region_samples) = region.samples() else { + panic!("independent odd ROI group must retain u8 samples") + }; + let expected_region = (roi.y..roi.y + roi.h) + .flat_map(|y| { + let start = (y * WIDTH + roi.x) as usize; + expected[start..start + roi.w as usize].iter().copied() + }) + .collect::>(); + assert_eq!(region_samples, &expected_region); + + let scaled_roi = roi.scaled_covering(Downscale::Half); + let mut scalar = J2kDecoder::new(&encoded).expect("scalar odd OpenHT decoder"); + let mut oracle = vec![0_u8; scaled_roi.w as usize * scaled_roi.h as usize]; + let outcome = scalar + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut oracle, + scaled_roi.w as usize, + PixelFormat::Gray8, + roi, + Downscale::Half, + ) + .expect("scalar odd OpenHT region-reduced oracle"); + assert_eq!(outcome.decoded, scaled_roi); + let region_reduced = result + .groups() + .iter() + .find(|group| group.source_indices() == [2]) + .expect("independent region-reduced group"); + assert_eq!(region_reduced.decoded_rects(), [scaled_roi]); + let CpuBatchSamples::U8(region_reduced_samples) = region_reduced.samples() else { + panic!("independent odd region-reduced group must retain u8 samples") + }; + assert_eq!(region_reduced_samples, &oracle); +} + +fn ht_pass_counts(prepared: &j2k::PreparedImage) -> Vec { + let plan = prepared.htj2k_plan().expect("HTJ2K referenced plan"); + let plan = plan + .adapter_view() + .downcast_ref::() + .expect("j2k-native prepared-plan adapter"); + let mut passes = Vec::new(); + for tile in plan.tiles() { + if let Some(geometry) = tile.grayscale_geometry() { + extend_ht_pass_counts(geometry, &mut passes); + } else if let Some(geometry) = tile.color_geometry() { + for component in &geometry.component_plans { + extend_ht_pass_counts(component, &mut passes); + } + } else if let Some(geometry) = tile.rgba_geometry() { + for component in &geometry.component_plans { + extend_ht_pass_counts(component, &mut passes); + } + } + } + passes +} + +fn extend_ht_pass_counts(plan: &J2kDirectGrayscalePlan, passes: &mut Vec) { + passes.extend( + plan.steps + .iter() + .filter_map(|step| match step { + J2kDirectGrayscaleStep::HtSubBand(sub_band) => Some(&sub_band.jobs), + _ => None, + }) + .flatten() + .map(|job| job.number_of_coding_passes), + ); +} + +fn layered_ht_fixture(num_layers: u8) -> (Arc<[u8]>, Vec) { + const WIDTH: u32 = 8; + const HEIGHT: u32 = 8; + let pixels = (0..WIDTH * HEIGHT) + .map(|index| ((index * 73 + index / 7) & 0xff) as u8) + .collect::>(); + let encoded = encode_htj2k( + &pixels, + WIDTH, + HEIGHT, + 1, + 8, + false, + &EncodeOptions { + reversible: false, + num_decomposition_levels: 0, + guard_bits: 2, + num_layers, + use_mct: false, + ..EncodeOptions::default() + }, + ) + .expect("encode layered HT fixture"); + let mut scalar = J2kDecoder::new(&encoded).expect("scalar layered decoder"); + let mut oracle = vec![0_u8; pixels.len()]; + scalar + .decode_into(&mut oracle, WIDTH as usize, PixelFormat::Gray8) + .expect("scalar layered oracle"); + (Arc::from(encoded), oracle) +} + +#[test] +#[expect( + clippy::too_many_lines, + reason = "one end-to-end matrix keeps pass classification and independent decoder oracles together" +)] +fn external_cleanup_magref_and_generated_sigprop_jobs_decode_in_batches() { + let (cleanup, cleanup_pixels) = layered_ht_fixture(1); + let (sigprop, sigprop_pixels) = layered_ht_fixture(2); + let (magref, magref_pixels) = layered_ht_fixture(3); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![ + EncodedImage::full(cleanup), + EncodedImage::full(sigprop), + EncodedImage::full(magref), + EncodedImage::full(Arc::from(openhtj2k_refinement_fixture())), + EncodedImage::full(Arc::from(openhtj2k_refinement_odd_fixture())), + EncodedImage::full(Arc::from(openhtj2k_sigprop_fixture())), + EncodedImage::full(Arc::from(openhtj2k_sigprop_overlap_fixture())), + ], + options, + ) + .expect("prepare HT pass matrix"); + + assert!( + prepared.errors().is_empty(), + "prepare errors: {:?}", + prepared.errors() + ); + let generated_group = prepared + .groups() + .iter() + .find(|group| group.source_indices() == [0, 1, 2]) + .expect("generated pass group"); + let pass_sets = generated_group + .images() + .iter() + .map(|image| ht_pass_counts(image).into_iter().collect::>()) + .collect::>(); + assert_eq!(pass_sets[0], BTreeSet::from([1])); + assert!( + pass_sets[1].contains(&2), + "two-layer fixture: {:?}", + pass_sets[1] + ); + assert!( + pass_sets[2].contains(&3), + "three-layer fixture: {:?}", + pass_sets[2] + ); + + let independent_passes = prepared + .groups() + .iter() + .filter(|group| group.source_indices().iter().any(|index| *index >= 3)) + .flat_map(j2k::PreparedBatchGroup::images) + .flat_map(ht_pass_counts) + .collect::>(); + assert!( + independent_passes.contains(&1), + "independent OpenHT fixtures must exercise cleanup-only blocks" + ); + assert!( + independent_passes.contains(&2), + "independent OpenHT fixtures must exercise exactly-two-pass SigProp blocks" + ); + assert!( + independent_passes.contains(&3), + "independent OpenHT fixtures must exercise three-pass refinement" + ); + + let mut decoder = CpuBatchDecoder::new(options); + let result = decoder + .decode_prepared(&prepared) + .expect("decode prepared HT pass matrix"); + assert!( + result.errors().is_empty(), + "decode errors: {:?}", + result.errors() + ); + let stats = decoder.workspace_stats(); + assert!(stats.flattened_group_plans() > 0); + assert_eq!( + stats.flattened_payload_jobs(), + stats.flattened_cleanup_jobs() + + stats.flattened_sigprop_jobs() + + stats.flattened_magref_jobs() + ); + assert!(stats.flattened_cleanup_jobs() > 0); + assert!(stats.flattened_sigprop_jobs() > 0); + assert!(stats.flattened_magref_jobs() > 0); + assert_eq!(stats.flattened_classic_jobs(), 0); + let generated = result + .groups() + .iter() + .find(|group| group.source_indices() == [0, 1, 2]) + .expect("decoded generated pass group"); + let CpuBatchSamples::U8(samples) = generated.samples() else { + panic!("generated HT pass group must retain u8 samples") + }; + assert_eq!( + samples, + &[cleanup_pixels, sigprop_pixels, magref_pixels].concat() + ); + + for (source_index, expected) in [ + (3, openhtj2k_refinement_pixels()), + (4, openhtj2k_refinement_odd_pixels()), + ] { + let group = result + .groups() + .iter() + .find(|group| group.source_indices() == [source_index]) + .expect("independent OpenHT output group"); + let CpuBatchSamples::U8(samples) = group.samples() else { + panic!("independent OpenHT fixture must retain u8 samples") + }; + assert_eq!(samples, expected); + } + + let sigprop_group = result + .groups() + .iter() + .find(|group| group.source_indices() == [5]) + .expect("independent OpenHT SigProp output group"); + let CpuBatchSamples::U16(sigprop_samples) = sigprop_group.samples() else { + panic!("independent OpenHT RGB12 fixture must retain u16 samples") + }; + let expected_sigprop = openhtj2k_sigprop_pixels_le() + .chunks_exact(2) + .map(|sample| u16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + assert_eq!(sigprop_samples.len(), expected_sigprop.len()); + let mut scalar_decoder = + J2kDecoder::new(openhtj2k_sigprop_fixture()).expect("independent OpenHT scalar decoder"); + let mut scalar_bytes = vec![0_u8; 128 * 128 * 3 * 2]; + scalar_decoder + .decode_into(&mut scalar_bytes, 128 * 3 * 2, PixelFormat::Rgb16) + .expect("independent OpenHT scalar RGB16 decode"); + let scalar_samples = scalar_bytes + .chunks_exact(2) + .map(|sample| u16::from_le_bytes([sample[0], sample[1]])) + .collect::>(); + let (scalar_difference_index, scalar_difference) = sigprop_samples + .iter() + .zip(&scalar_samples) + .enumerate() + .map(|(index, (actual, expected))| (index, actual.abs_diff(*expected))) + .max_by_key(|(_, difference)| *difference) + .unwrap_or_default(); + assert!( + scalar_difference <= 1, + "prepared decode differs from scalar by {scalar_difference} LSBs at sample {scalar_difference_index}: prepared={}, scalar={}", + sigprop_samples[scalar_difference_index], scalar_samples[scalar_difference_index] + ); + let (max_difference_index, max_difference) = sigprop_samples + .iter() + .zip(&expected_sigprop) + .enumerate() + .map(|(index, (actual, expected))| (index, actual.abs_diff(*expected))) + .max_by_key(|(_, difference)| *difference) + .unwrap_or_default(); + assert!( + max_difference <= 1, + "independent irreversible RGB12 reconstruction differs from OpenJPH by {max_difference} LSBs at sample {max_difference_index}: codec={}, OpenJPH={}", + sigprop_samples[max_difference_index], + expected_sigprop[max_difference_index] + ); + + let overlap_group = result + .groups() + .iter() + .find(|group| group.source_indices() == [6]) + .expect("independent overlapping-refinement output group"); + let CpuBatchSamples::U8(overlap_samples) = overlap_group.samples() else { + panic!("independent overlapping-refinement fixture must retain u8 samples") + }; + let expected_overlap = openhtj2k_sigprop_overlap_pixels(); + assert_eq!(overlap_samples.len(), expected_overlap.len()); + let (overlap_difference_index, overlap_difference) = overlap_samples + .iter() + .zip(expected_overlap) + .enumerate() + .map(|(index, (actual, expected))| (index, actual.abs_diff(*expected))) + .max_by_key(|(_, difference)| *difference) + .unwrap_or_default(); + assert!( + overlap_difference <= 1, + "overlapping SigProp/MagRef reconstruction differs from OpenHTJ2K by {overlap_difference} LSBs at sample {overlap_difference_index}: codec={}, OpenHTJ2K={}", + overlap_samples[overlap_difference_index], + expected_overlap[overlap_difference_index] + ); +} diff --git a/crates/j2k/tests/owned_batch_fixtures/irreversible.rs b/crates/j2k/tests/owned_batch_fixtures/irreversible.rs new file mode 100644 index 00000000..198496ae --- /dev/null +++ b/crates/j2k/tests/owned_batch_fixtures/irreversible.rs @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +fn irreversible_fixture(route: CodingRoute, components: usize) -> Arc<[u8]> { + const WIDTH: u32 = 33; + const HEIGHT: u32 = 19; + let pixels = (0..WIDTH * HEIGHT * u32::try_from(components).expect("components")) + .map(|index| ((index * 31 + index / 5 + 17) & 0xff) as u8) + .collect::>(); + let options = EncodeOptions { + reversible: false, + num_decomposition_levels: 2, + use_mct: false, + ..EncodeOptions::default() + }; + Arc::from(match route { + CodingRoute::Classic => encode( + &pixels, + WIDTH, + HEIGHT, + u16::try_from(components).expect("components"), + 8, + false, + &options, + ) + .expect("encode classic 9/7 fixture"), + CodingRoute::Htj2k => encode_htj2k( + &pixels, + WIDTH, + HEIGHT, + u16::try_from(components).expect("components"), + 8, + false, + &options, + ) + .expect("encode HTJ2K 9/7 fixture"), + }) +} + +fn irreversible_gray_with_coc_override() -> Arc<[u8]> { + let mut encoded = irreversible_fixture(CodingRoute::Classic, 1).to_vec(); + let cod = encoded + .windows(2) + .position(|marker| marker == [0xff, 0x52]) + .expect("COD marker"); + let segment_length = usize::from(u16::from_be_bytes([encoded[cod + 2], encoded[cod + 3]])); + let segment_end = cod + 2 + segment_length; + let scod = encoded[cod + 4]; + let component_parameters = encoded[cod + 9..segment_end].to_vec(); + assert_eq!(component_parameters.last(), Some(&0), "fixture uses 9/7"); + + encoded[cod + 13] = 1; + let coc_length = 2 + 1 + 1 + component_parameters.len(); + let mut coc = Vec::with_capacity(2 + coc_length); + coc.extend_from_slice(&[0xff, 0x53]); + coc.extend_from_slice( + &u16::try_from(coc_length) + .expect("small COC segment") + .to_be_bytes(), + ); + coc.push(0); + coc.push(scod); + coc.extend_from_slice(&component_parameters); + encoded.splice(segment_end..segment_end, coc); + Arc::from(encoded) +} + +fn ht_gray_with_coc_block_coding_override() -> Arc<[u8]> { + let mut encoded = irreversible_fixture(CodingRoute::Htj2k, 1).to_vec(); + let cod = encoded + .windows(2) + .position(|marker| marker == [0xff, 0x52]) + .expect("COD marker"); + let segment_length = usize::from(u16::from_be_bytes([encoded[cod + 2], encoded[cod + 3]])); + let segment_end = cod + 2 + segment_length; + let scod = encoded[cod + 4]; + let component_parameters = encoded[cod + 9..segment_end].to_vec(); + assert_ne!( + component_parameters[3] & 0x40, + 0, + "fixture uses HT block coding" + ); + + encoded[cod + 12] &= !0x40; + let coc_length = 2 + 1 + 1 + component_parameters.len(); + let mut coc = Vec::with_capacity(2 + coc_length); + coc.extend_from_slice(&[0xff, 0x53]); + coc.extend_from_slice( + &u16::try_from(coc_length) + .expect("small COC segment") + .to_be_bytes(), + ); + coc.push(0); + coc.push(scod); + coc.extend_from_slice(&component_parameters); + encoded.splice(segment_end..segment_end, coc); + Arc::from(encoded) +} + +fn irreversible_gray_with_tile_cod_override() -> Arc<[u8]> { + let mut encoded = irreversible_fixture(CodingRoute::Classic, 1).to_vec(); + let cod = encoded + .windows(2) + .position(|marker| marker == [0xff, 0x52]) + .expect("COD marker"); + let cod_length = usize::from(u16::from_be_bytes([encoded[cod + 2], encoded[cod + 3]])); + let cod_end = cod + 2 + cod_length; + let tile_cod = encoded[cod..cod_end].to_vec(); + assert_eq!(tile_cod.last(), Some(&0), "fixture uses 9/7"); + encoded[cod + 13] = 1; + + let sot = encoded + .windows(2) + .position(|marker| marker == [0xff, 0x90]) + .expect("SOT marker"); + let psot = u32::from_be_bytes([ + encoded[sot + 6], + encoded[sot + 7], + encoded[sot + 8], + encoded[sot + 9], + ]); + let updated_psot = psot + .checked_add(u32::try_from(tile_cod.len()).expect("small tile COD")) + .expect("tile-part length"); + encoded[sot + 6..sot + 10].copy_from_slice(&updated_psot.to_be_bytes()); + let tile_header_start = sot + 12; + assert_eq!( + &encoded[tile_header_start..tile_header_start + 2], + &[0xff, 0x93], + "generated fixture has an empty tile header" + ); + encoded.splice(tile_header_start..tile_header_start, tile_cod); + Arc::from(encoded) +} + +#[test] +fn batch_metadata_uses_component_coding_overrides_from_the_execution_plan() { + let prepared = prepare_batch( + vec![EncodedImage::full(irreversible_gray_with_coc_override())], + BatchDecodeOptions::default(), + ) + .expect("prepare COC override fixture"); + + assert!(prepared.errors().is_empty(), "{:?}", prepared.errors()); + assert_eq!(prepared.groups().len(), 1); + let info = prepared.groups()[0].info(); + assert_eq!(info.route, BatchCodecRoute::Classic); + assert_eq!(info.transform, BatchWaveletTransform::Irreversible97); + assert_eq!( + info.transfer_syntax, + CompressedTransferSyntax::Jpeg2000Lossy + ); +} + +#[test] +fn batch_route_uses_component_block_coding_overrides_from_the_execution_plan() { + let prepared = prepare_batch( + vec![EncodedImage::full(ht_gray_with_coc_block_coding_override())], + BatchDecodeOptions::default(), + ) + .expect("prepare HT COC override fixture"); + + assert!(prepared.errors().is_empty(), "{:?}", prepared.errors()); + assert_eq!(prepared.groups().len(), 1); + let info = prepared.groups()[0].info(); + assert_eq!(info.route, BatchCodecRoute::Htj2k); + assert_eq!(info.transform, BatchWaveletTransform::Irreversible97); + assert_eq!( + info.transfer_syntax, + CompressedTransferSyntax::HtJpeg2000Lossy + ); +} + +#[test] +fn batch_metadata_uses_tile_coding_overrides_from_the_execution_plan() { + let prepared = prepare_batch( + vec![EncodedImage::full( + irreversible_gray_with_tile_cod_override(), + )], + BatchDecodeOptions::default(), + ) + .expect("prepare tile COD override fixture"); + + assert!(prepared.errors().is_empty(), "{:?}", prepared.errors()); + assert_eq!(prepared.groups().len(), 1); + let info = prepared.groups()[0].info(); + assert_eq!(info.route, BatchCodecRoute::Classic); + assert_eq!(info.transform, BatchWaveletTransform::Irreversible97); + assert_eq!( + info.transfer_syntax, + CompressedTransferSyntax::Jpeg2000Lossy + ); +} + +#[test] +fn irreversible_97_batches_agree_with_scalar_reconstruction_within_one_lsb() { + const WIDTH: usize = 33; + const HEIGHT: usize = 19; + let fixtures = [ + (irreversible_fixture(CodingRoute::Classic, 1), 1), + (irreversible_fixture(CodingRoute::Classic, 3), 3), + (irreversible_fixture(CodingRoute::Htj2k, 1), 1), + (irreversible_fixture(CodingRoute::Htj2k, 3), 3), + ]; + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = CpuBatchDecoder::new(options); + let result = decoder + .decode( + fixtures + .iter() + .map(|(bytes, _)| EncodedImage::full(Arc::clone(bytes))) + .collect(), + ) + .expect("decode irreversible batch"); + + assert!( + result.errors().is_empty(), + "9/7 errors: {:?}", + result.errors() + ); + for (source_index, (encoded, components)) in fixtures.iter().enumerate() { + let group = result + .groups() + .iter() + .find(|group| group.source_indices() == [source_index]) + .expect("9/7 output group"); + assert_eq!( + group.info().transform, + BatchWaveletTransform::Irreversible97 + ); + let CpuBatchSamples::U8(batch) = group.samples() else { + panic!("8-bit 9/7 group must retain u8 samples") + }; + let mut scalar = J2kDecoder::new(encoded).expect("scalar 9/7 decoder"); + let mut oracle = vec![0_u8; WIDTH * HEIGHT * components]; + scalar + .decode_into( + &mut oracle, + WIDTH * components, + if *components == 1 { + PixelFormat::Gray8 + } else { + PixelFormat::Rgb8 + }, + ) + .expect("scalar 9/7 oracle"); + assert_eq!(batch.len(), oracle.len()); + assert!( + batch + .iter() + .zip(&oracle) + .all(|(batch, oracle)| batch.abs_diff(*oracle) <= 1), + "source {source_index}: batch reconstruction differs by more than one LSB" + ); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 9a7ca550..7d950a7b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,17 +14,17 @@ still-image correctness. Keep row-level status synchronized with | Crate | Class | Role | | --- | --- | --- | -| `j2k` | public codec | Primary user-facing JPEG 2000 / HTJ2K API. | +| `j2k` | public codec | Primary user-facing JPEG 2000 / HTJ2K API, including owned preparation and CPU batch decode. | | `j2k-core` | core | Shared traits, errors, geometry, pixel formats, backend requests, and device-surface contracts. | | `j2k-types` | core | Shared encode-stage contracts and semver-visible value types used by the facade, native engine, and adapters. | | `j2k-codec-math` | support | No-std shared constants and pure math tables for CPU, CUDA-Oxide, and Metal parity. | | `j2k-jpeg`, `j2k-tilecodec` | codec | CPU/native codec implementations and stable codec APIs. | | `j2k-native` | engine | Native JPEG 2000 / HTJ2K engine used by J2K APIs and adapter validation. | | `j2k-profile`, `j2k-metal-support` | support | Runtime/profile helpers used by adapters and codec crates. | -| `j2k-cuda-runtime` | CUDA engine | CUDA Driver API integration, J2K-owned kernel modules, launch orchestration, and CUDA memory helpers shared by CUDA adapters. | -| `j2k-jpeg-cuda`, `j2k-cuda`, `j2k-transcode-cuda` | CUDA adapter | Codec-facing CUDA APIs, route policy, and CUDA device memory integration for supported paths. | -| `j2k-jpeg-metal`, `j2k-metal`, `j2k-transcode-metal` | Metal adapter | macOS Metal runtime integration for supported paths. | -| `j2k-ml` | experimental integration | Independent Burn tensor decode integration; unpublished during the 0.7 cycle. | +| `j2k-cuda-runtime` | CUDA engine | CUDA Driver API integration, J2K-owned kernel modules, launch orchestration, CUDA memory helpers, and guarded external-allocation validation shared by CUDA adapters. | +| `j2k-jpeg-cuda`, `j2k-cuda`, `j2k-transcode-cuda` | CUDA adapter | Codec-facing CUDA APIs, persistent batch sessions, route policy, resident output, and validated caller-owned destinations for supported paths. | +| `j2k-jpeg-metal`, `j2k-metal`, `j2k-transcode-metal` | Metal adapter | macOS Metal runtime integration, persistent batch sessions, resident output, and validated caller-owned destinations for supported paths. | +| `j2k-ml` | experimental integration | Thin Burn allocation and codec-interop adapter for owned integer batch output; unpublished during the 0.7 cycle. | | `j2k-transcode` | transcode | JPEG-to-HTJ2K coefficient-domain transcode algorithms and shared contracts. | | `j2k-cli` | CLI | Command-line inspection and JPEG-to-HTJ2K smoke transcode entry point. | | `j2k-test-support`, `j2k-transcode-test-support` | dev helper | Shared fixture, benchmark input, and transcode oracle helpers for tests, benches, and examples. | @@ -34,6 +34,11 @@ still-image correctness. Keep row-level status synchronized with ## Dependency rules - The public `j2k` crate owns the JPEG 2000 / HTJ2K API surface. +- `j2k`, `j2k-native`, `j2k-cuda`, and `j2k-metal` own codec parsing, + preparation, grouping, decoding, scratch reuse, and device execution. +- `j2k-ml` may allocate or materialize Burn tensors and establish safe + framework/codec ordering. It must not duplicate entropy decode, transforms, + grouping policy, normalization, or training behavior. - Codec crates may depend on `j2k-core` and support crates. - Adapter crates may depend inward on codec/core/support crates. - Support crates must not depend on adapters. @@ -68,19 +73,88 @@ xtask -> j2k, j2k-codec-math, j2k-compare, j2k-native, j2k-profile, j2k-test-sup ## Backend policy -CPU is the correctness baseline. Device adapters can add resident outputs and -stage acceleration, but they must preserve explicit unsupported errors for -unsupported requests. +CPU is the correctness baseline. The owned fast-batch surface returns +homogeneous Gray/RGB/RGBA groups as native `U8`, `U16`, or `I16` samples in +NCHW or NHWC order and preserves source indices. Straight and premultiplied +alpha are distinct grouping keys. Preparation retains the caller-owned +codestream bytes and reusable decode plans without duplicating the codestream. +Broader component layouts remain on the component-plane APIs. + +Device adapters can add resident outputs and validated caller-owned +destinations, but explicit requests must return unsupported errors instead of +falling back to CPU staging. A direct external destination is the final output +allocation: decoded pixels must not cross a GPU-to-CPU-to-GPU path or a second +device output merely for framework integration. CUDA adapters use `j2k-cuda-runtime`, which owns the shared CUDA Driver API runtime, CUDA Oxide module loading, and host launch orchestration for supported CUDA codec stages. Product CUDA codec kernels are generated from CUDA Oxide projects while Rust host code retains Driver API orchestration. `cuda-runtime` support is an implementation dependency, not proof of NVIDIA performance. +The Burn bridge uses a uniquely borrowed CubeCL allocation and CUDA event +dependencies to order framework allocation, codec writes, and later tensor +consumption without a normal-path context synchronization. Metal adapters use `j2k-metal-support` for device, queue, shader-library, pipeline loading, checked buffer access, and route-label helpers. It is the sole raw Objective-C resource-construction boundary: nil is checked before any foreign handle is formed, and autoreleased command resources are retained into owned Rust handles before return. Codec-specific kernels stay in codec adapter -crates. +crates. The Burn bridge pairs wgpu and codec sessions on the same underlying +Metal device and lends the retained Burn-owned `MTLBuffer` suballocation to the +codec's validated final destination. + +HTJ2K is the optimized batch priority; classic JPEG 2000 shares the public +grouping, destination, and completion contracts and remains regression-covered. +Supported fast-batch inputs prepare one of two immutable, facade-owned plan +views. `PreparedHtj2kPlan` retains per-tile HT cleanup/refinement geometry and +byte ranges; `PreparedClassicPlan` retains per-tile classic packet/code-block +geometry plus ordered fragment ranges. Both reference compressed payloads by +offset from the original `Arc<[u8]>` and are reusable across sessions without +reparsing or duplicating the codestream. Inputs outside those retained-plan +boundaries keep metadata only and use the general CPU decoder when that broader +codec path supports them. + +`CpuBatchDecoder` uses a bounded scheduler with retained worker workspaces. It +allocates one typed buffer per homogeneous group and lets workers decode into +disjoint image regions, avoiding per-image output owners and a final batch +assembly copy. `CudaBatchDecoder` and `MetalBatchDecoder` likewise retain their +device context, streams or queues, modules or pipelines, lookup tables, events, +staging owners, and scratch pools across submissions. + +The hardware-validated direct Metal batch matrix is single-tile classic JPEG +2000 and HTJ2K Gray/RGB/RGBA output in native `U8`, `U16`, or `I16`, for +`Full`, `Region`, `Reduced`, and `RegionReduced` requests in NCHW or NHWC +destinations. Additional Metal tests validate exact multi-tile HT Gray12/RGB8 +for all four requests and classic RGB8 full output. CUDA +has RTX 4070 hardware validation for the classic and HT Gray/RGB/RGBA +`U8`/`U16`/`I16` fast-batch matrix, all four requests, both layouts, and +codec-resident, external-destination, and Burn-direct output. Its release lane +also covers classic and HT multi-tile regressions, asynchronous drop and +session reuse, and repeated-batch soaks. Reversible output is bit-exact; +irreversible 9/7 output agrees with the CPU oracle within one integer LSB. +These are correctness/support statements, not benchmark evidence. + +HT entropy work is flattened across images, bucketed by cleanup-only, +SigProp, and MagRef work, and split into bounded pass-homogeneous chunks. Chunk +status retains the original source identity where the device reports a failing +job, while the final native store still writes one dense destination per +homogeneous group. Resident and external-destination routes share the codec +pipeline; an external destination receives the final samples without a decoded +host transfer or an intermediate final device allocation. + +GPU prepared decode remains fail-closed for nonzero ROI maxshift from codestream +RGN markers and for shapes outside a backend's retained-plan boundary. +Subsampled components, mixed precision or signedness, arbitrary component +counts, and precision above 16 bits remain on the CPU component-plane APIs or +return a structured fast-batch representability error. No HT-dominant +publication throughput run has yet been recorded for this new batch +architecture. A July 19, 2026 local M4 Pro diagnostic run covers the complete +HT-dominant Gray/RGB/RGBA matrix. Burn-direct prepared Metal decode was faster +than CPU decode plus upload for 52 of 64 batch-32 cases and all 64 batch-64 +cases, while many batch-1 and batch-8 cases remained slower. That historical +harness used identical encoded content for every batch greater than one; +Metal RGB/RGBA additionally used decode-once broadcast. Those batch-32/64 +counts are therefore not a content-distinct acceptance baseline. Backend +selection stays explicit; the local run is implementation evidence, not an +adoption-facing publication claim. diff --git a/docs/benchmark-evidence.md b/docs/benchmark-evidence.md index d5a2639e..3891cef5 100644 --- a/docs/benchmark-evidence.md +++ b/docs/benchmark-evidence.md @@ -330,8 +330,124 @@ and an absolute, nonexistent `.gputrace` path in `J2K_METAL_CAPTURE_PATH`. Captured timings are diagnostic only and are not used for acceptance. +## July 19 Batch-Input Qualification + +The July 19 CUDA and Metal batch diagnostics below predate the explicit +`distinct` and `repeated` input modes. Their harnesses cloned one encoded +`Arc` for every image in a batch. Batch-1 measurements are unaffected. Every +batch-greater-than-one measurement used identical encoded content, even when +the codec still submitted each image independently. + +Metal color decoding additionally recognized pointer-identical RGB/RGBA inputs +and decoded one image before broadcasting its pixels across the output batch. +Consequently, the historical Metal RGB/RGBA batch-greater-than-one rows measure +decode-once broadcast, not distinct-image batch throughput. Other historical +batch-greater-than-one rows, including CUDA and Metal grayscale, measure +identical-content reuse. The numbers remain preserved as historical diagnostic +evidence, but none of those rows is an acceptance baseline or a claim about +content-distinct batches. The corrected harness defaults to 64 deterministic, +content-distinct codestreams per workload and labels repeated-owner runs +separately. + +## CUDA Status + +### HTJ2K batch and Burn-direct diagnostic - 2026-07-19 + +This is implementation evidence, not a publication or adoption-facing speed +claim. The source tree had uncommitted batch-pipeline changes, and the harness +uses generated fixtures rather than a pinned external corpus. + +Host and command: + +- NVIDIA GeForce RTX 4070 SUPER under WSL2, driver 596.49 +- Ubuntu 24.04 +- Rust 1.96 +- `cargo bench -p j2k-ml --bench batch_decode_cuda --features cpu,cuda -- --quick` + +The matrix covered unsigned and signed Gray12/Gray16 at 512 and 1024 pixels, +RGB8/RGB16 and RGBA8/RGBA16 at 256 and 512 pixels, all four request modes, and +batch sizes 1/8/32/64. The fail-closed harness completed 1,024 accelerated +telemetry rows. Every row had one status transfer and no decoded-pixel device +readback; the decoded D2H counters exactly matched the status counters. No row +used a codec event or context host synchronization. Codec-resident probes used +no consumer host wait; Burn-direct probes used one final consumer sync solely +to bound the end-to-end benchmark interval. The CUDA release suite separately +passed the external-destination, event ordering, drop cleanup, and 1,000-batch +session-reuse tests. + +Criterion median estimates for selected full-decode rows were: + +| Workload | Batch | CPU decode + Burn upload | Prepared Burn-direct CUDA | Ratio | +| --- | ---: | ---: | ---: | ---: | +| Gray12 512 | 32 | 168.009 MP/s | 657.198 MP/s | 3.912x | +| Gray12 512 | 64 | 156.981 MP/s | 487.790 MP/s | 3.107x | +| Gray16 1024 | 32 | 142.390 MP/s | 439.150 MP/s | 3.084x | +| Gray16 1024 | 64 | 138.807 MP/s | 294.203 MP/s | 2.120x | +| RGB16 512 | 32 | 48.602 MP/s | 117.474 MP/s | 2.417x | +| RGB16 512 | 64 | 48.151 MP/s | 103.962 MP/s | 2.159x | +| RGBA16 512 | 32 | 38.114 MP/s | 73.382 MP/s | 1.925x | +| RGBA16 512 | 64 | 37.845 MP/s | 71.748 MP/s | 1.896x | + +In this repeated-content run, across all 64 workload/request combinations, +prepared Burn-direct was faster than CPU decode plus upload in 57 batch-1 +cases and all 64 batch-8, batch-32, +and batch-64 cases. Mean staged/direct time ratios were 2.584x, 2.262x, +2.837x, and 2.881x respectively. Backend selection remains explicit because +these are quick local diagnostics, not pinned-corpus publication evidence. + ## Metal Status +### HTJ2K batch and Burn-direct diagnostic - 2026-07-19 + +This is local implementation evidence, not a publication or adoption-facing +speed claim. The source tree had uncommitted batch-pipeline changes, so the run +is not tied to a reproducible git SHA. Fixtures were generated by the benchmark +harness and are not an external corpus. + +Host and command: + +- Apple M4 Pro, 48 GB memory +- macOS 26.5.2 build `25F84` +- Rust 1.96 +- `cargo bench -p j2k-ml --bench batch_decode_metal --features cpu,metal -- --quick` + +The matrix covered unsigned and signed Gray12/Gray16 at 512 and 1024 pixels, +RGB8/RGB16 and RGBA8/RGBA16 at 256 and 512 pixels, all four request modes, and +batch sizes 1/8/32/64. The harness rejected any preparation or decode with an +indexed error, group error, or output-group count other than one. It emitted +1,280 completed telemetry rows. In that historical schema, all 1,024 +accelerated rows carried route-contract assertions for zero decoded host +uploads/readbacks, zero final device copies, one final output destination, and +one group-level completion boundary; those fields were not sampled hardware +counters. The codec session diagnostics measured one group submission per +row. Focused interop and submission tests, rather than those hard-coded +telemetry fields, are the evidence for the transfer, allocation, copy, and +completion-boundary contracts. + +The following are single warmed telemetry probes; Criterion data in the same +run remains the statistically sampled result: + +| Workload, full decode | Batch | CPU decode + Burn upload | Prepared Burn-direct Metal | +| --- | ---: | ---: | ---: | +| Gray12 512 | 32 | 424.029 MP/s | 938.249 MP/s | +| Gray12 512 | 64 | 449.605 MP/s | 1,316.428 MP/s | +| Gray16 1024 | 32 | 269.275 MP/s | 1,044.676 MP/s | +| Gray16 1024 | 64 | 280.599 MP/s | 870.603 MP/s | +| RGB16 512 | 32 | 141.822 MP/s | 719.949 MP/s | +| RGB16 512 | 64 | 139.696 MP/s | 1,560.369 MP/s | +| RGBA16 512 | 32 | 115.776 MP/s | 743.564 MP/s | +| RGBA16 512 | 64 | 114.878 MP/s | 1,399.131 MP/s | + +In this repeated-content run, across all 64 workload/request combinations, +prepared Burn-direct was faster than the staged route in 52 batch-32 cases and +all 64 batch-64 cases. It was +faster in only 10 batch-1 and 21 batch-8 cases. Because the Metal color rows at +batch greater than one were decode-once broadcast and all other batch-greater- +than-one rows used identical content, these counts do not establish a +content-distinct high-throughput route. They support keeping selection explicit +until the corrected distinct-input matrix is recorded; they do not support +blanket Auto routing or a single-image speed claim. + Metal acceleration is selective. Public claims should say Metal-accelerated stages, not complete end-to-end Metal coverage for every encode, decode, or transcode route. diff --git a/docs/env-vars.md b/docs/env-vars.md index a75380b9..4a17a533 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -68,6 +68,8 @@ override it. | Variable | Effect | Default | Stability | | --- | --- | --- | --- | +| `J2K_ML_BATCH_INPUT_MODE` | Selects content-distinct generated inputs (`distinct`) or one repeated `Arc` owner (`repeated`) for the `j2k-ml` batch benchmarks. One process uses one mode. | `distinct` | Benchmark | +| `J2K_ML_BATCH_PROCESS_MODE` | Selects uninstrumented Criterion measurement (`criterion`) or the separate low-batch telemetry/profile process (`profile`) for CUDA and Metal batch benchmarks. | `criterion` | Benchmark | | `J2K_REQUIRE_OPENJPEG` | Makes OpenJPEG parity tests and benchmark comparator runs fail instead of skip when OpenJPEG tools are unavailable. | Skip unavailable comparator paths | Benchmark | | `J2K_REQUIRE_GROK` | Makes Grok parity tests and benchmark comparator runs fail instead of skip when Grok tools or libraries are unavailable. | Skip unavailable comparator paths | Benchmark | | `J2K_REQUIRE_OPENJPH` | Makes optional OpenJPH fixture comparator rows fail instead of skip when `ojph_expand` is unavailable. Intended only for HTJ2K/JPH-compatible CLI context rows. | Skip unavailable OpenJPH path unless explicitly included | Benchmark | diff --git a/docs/j2k-ml.md b/docs/j2k-ml.md index 900ab82a..d293ae61 100644 --- a/docs/j2k-ml.md +++ b/docs/j2k-ml.md @@ -1,72 +1,160 @@ -# Burn tensor decoding with `j2k-ml` +# Burn batch decoding with `j2k-ml` `j2k-ml` is an experimental, independently maintained integration for Burn 0.21. It is not an official Tracel or Burn crate and remains `publish = false` through the 0.7 release cycle. -## Contract - -The crate accepts borrowed JP2, JPH, raw J2K, and raw HTJ2K bytes together -with the repository's full, ROI, scaled, or ROI-scaled decode request. A -single decode returns a rank-3 tensor; a batch returns one rank-4 tensor while -preserving input order. - -Defaults are: - -- `ChannelsFirst`, producing `CHW` or `NCHW`. -- `ChannelSelection::Auto`, preserving grayscale as one channel and producing - RGB for color inputs. Alpha is included only when `Rgba` is explicit. -- `FloatNormalization::Unit`, dividing canonical 8-bit samples by 255 or - canonical 16-bit samples by 65535. - -`decode_u8` and `decode_u16` preserve Burn's native unsigned integer dtype. -`decode_float` produces F32 storage in v1. `Raw` only casts. `MeanStd` applies -unit scaling and then `(x - mean) / std` per channel; channel counts, finite -values, and nonzero standard deviations are validated before pixel decoding. - -The result also reports the actual decoded rectangle, codec warnings, and the -route used. Empty batches, corrupt items, allocation overflow, unsupported -dtypes, and mismatched final shapes are errors. The indexed batch error keeps -the original codec or route failure. - -## Routes - -The `cpu` feature decodes one compact interleaved integer buffer, creates one -Burn `TensorData` for the image or entire batch, and then uses Burn operations -for permutation, F32 conversion, and normalization. It works with any Burn -`Backend` that supports the required dtype. - -The `cuda` feature targets Burn's default fused CUDA backend. It retains the -same CUDA primary context used by CubeCL, allocates through Burn/CubeCL, holds -the managed allocation guard while J2K writes, and registers the completed -allocation through public Burn fusion APIs. CubeCL 0.10 creates allocations on -a nonblocking stream but exposes no stream/event interop handle, so this route -completes that stream before J2K's default-stream access. A Rust `cuda-oxide` -kernel fuses layout conversion, integer-to-F32 conversion, and normalization. -There is no decoded-pixel device-to-host readback, tensor host upload, CUDA C, -NVCC, or C wrapper. Unsupported formats, devices, and direct paths fail with -`CudaDirect`; they never fall back to CPU. - -The `metal` feature requests strict resident J2K Metal decode. Native Metal -buffer sharing is not claimed: the route performs one packed readback for the -whole batch, uploads that compact integer representation once to Burn Metal, -and uses Burn operations for layout, conversion, and normalization. It uses no -private wgpu internals and reports `MetalStaged`. - -## Burn data loading - -The fallible single and batch functions are the primary API. Dataset storage, -labels, sampling, resizing, padding, augmentation, and worker policy remain -application-owned. When Burn's infallible `Batcher` trait is required, -`PanicOnDecodeError` is an explicitly named float-batcher adapter; its panic -includes the failed batch index and codec error. - -## Deferred scope - -V1 is limited to the existing interleaved Gray/RGB/RGBA 8/16-bit contract. -Arbitrary component tensors, signed native components, precision above 16 -bits, F16/BF16 output, resizing, DLPack, dataset formats, and native Metal -buffer sharing are deferred. +## Ownership boundary + +`j2k-ml` is a thin framework adapter, not a second codec. The `j2k`, +`j2k-native`, `j2k-cuda`, and `j2k-metal` crates own parsing, JPEG 2000 and +HTJ2K decoding, preparation, homogeneous grouping, memory reuse, and device +execution. `j2k-ml` only materializes a CPU codec group once or lends unique +Burn-owned CUDA/Metal storage to the codec's external-destination API. + +Dataset assembly, DICOM parsing, labels, sampling, resizing, padding, +prefetching, augmentation, float conversion, and normalization remain outside +the codec and this adapter. Applications cast and normalize the returned +integer tensors with ordinary Burn tensor operations. + +## Batch contract + +Inputs are owned `EncodedImage` values: an `Arc<[u8]>` containing JP2, JPH, +raw J2K, or raw HTJ2K bytes plus a full, ROI, reduced-resolution, or +ROI-and-reduced decode request. A persistent decoder can prepare and decode a +batch in one call, retain the codec's `PreparedBatch` and call +`decode_prepared` repeatedly, or regroup caller-supplied `PreparedImage` +values with `prepare_prepared_images`/`decode_prepared_images`. Regrouping does +not reparse or copy codestreams. Its source indices are positions in the new +submission; `PreparedImage::source_index` remains the original preparation +index. Strict/lenient settings must match, while layout and worker policy may +change. Preparation retains the original byte owner and codec plans; it does +not duplicate the codestream. Supported direct-plan inputs +report `PreparationDepth::Htj2kOffsetPlan` or +`PreparationDepth::ClassicOffsetPlan`; other valid inputs may remain +`MetadataOnly` for the general CPU path. + +The codec groups representable images by compatible decoded dimensions, +channel count, exact native sample type, requested layout, transform, and +backend execution shape. It never pads unlike shapes implicitly. The adapter +returns one ordinary rank-4 Burn integer tensor per homogeneous group in NCHW +or NHWC layout: + +- unsigned samples with precision at most 8 bits use `U8`; +- unsigned samples with precision from 9 through 16 bits use `U16`; +- signed samples with precision at most 16 bits use `I16`. + +`BurnBatchGroup` preserves the codec metadata, original source indices in +tensor order, actual decoded rectangles, and warnings. `BurnBatchDecode` +returns successful groups alongside indexed preparation failures and +`BurnBatchGroupError` values for homogeneous groups discarded during +submission or completion. A nonfatal group failure does not suppress other +groups; every submitted pending owner is retired before the result is exposed. +The fast batch representation is intentionally limited to uniform Gray, RGB, +or RGBA output; broader component layouts remain available through the codec's +component-plane APIs. + +JP2 channel definitions are part of the grouping key. In particular, straight +and premultiplied alpha are never combined into the same RGBA group, and the +alpha interpretation remains available in `BatchGroupInfo`. + +New batch sessions use strict decoding by default. Lenient decoding is +opt-in and reports warnings. An explicit CUDA or Metal route never stages +through CPU memory or silently falls back; unsupported inputs and interop +failures are structured errors. + +## Persistent routes + +`CpuBurnDecoder` retains a `j2k::CpuBatchDecoder` and its codec workspaces. +The codec decodes directly into one contiguous native Rust allocation per +group. The adapter constructs one `TensorData` from that allocation, preserving +the exact `U8`, `U16`, or `I16` dtype. The retained worker workspaces reuse +component, Tier-1, and IDWT owners. Supported inputs retain either a per-tile +HTJ2K cleanup/refinement offset plan or a per-tile classic packet/code-block +plan, and CPU prepared decode consumes single- and multi-tile forms without +reparsing. Inputs outside those retained-plan boundaries can remain +metadata-only and continue through the broader CPU decoder. + +`CudaBurnDecoder` retains the codec CUDA session for one Burn CUDA device. It +allocates the final Burn tensor first, validates the allocation's device, +bounds, alignment, and unique mutable ownership, and submits the codec's final +store directly into that allocation. CUDA events order CubeCL's stream and the +codec stream without a normal-path CPU wait. The decoder retains in-flight +codec resources until completion. Decoded pixels have no device-to-host +transfer, intermediate dense decoded-pixel allocation, or final +device-to-device copy. + +`MetalBurnDecoder::system_default` constructs the codec session and paired Burn +`WgpuDevice` from the same underlying Metal device. The adapter retains the +Burn-owned `MTLBuffer`, validates its suballocation layout and device identity, +and passes it to `MetalImageDestination`. The codec writes the final native +integer layout directly into that buffer before the tensor is registered with +Burn. Decoded pixels are neither read back nor uploaded again. + +Both GPU adapters retain scratch owners and destination access until one +group-level completion boundary. Dropping pending work safely retires those +resources and leaves the persistent session reusable; the normal path does not +introduce a per-image or per-kernel device synchronization. + +The direct-write guarantee is specifically zero decoded-pixel host round trips, +not a claim that compressed input or internal coefficient/scratch storage needs +no transfer or allocation. Those internal codec resources remain allowed. + +## Current exact accelerator boundary + +The accelerator adapters are deliberately narrower than the portable codec +contract. For every supported row below, `all requests` means `Full`, `Region`, +`Reduced`, and `RegionReduced`, and both NCHW and NHWC Burn tensors are +supported. Unsupported inputs return a structured group error; they never fall +back through CPU pixels. + +| Output | CPU | CUDA | Metal | +| --- | --- | --- | --- | +| Gray `U8`/`U16`/`I16` | Classic and HT, all requests | Hardware-validated classic/HT, all requests | Hardware-validated single-tile classic/HT, all requests; exact multi-tile subset below | +| RGB `U8`/`U16`/`I16` | Classic and HT, all requests | Hardware-validated classic/HT, all requests | Hardware-validated single-tile classic/HT, all requests; exact multi-tile subset below | +| RGBA `U8`/`U16`/`I16` | Classic and HT, all requests | Hardware-validated classic/HT, all requests | Hardware-validated single-tile classic/HT, all requests | + +The canonical `cargo xtask release-cuda` lane passes on the RTX 4070 runner. +Its codec targets validate classic and HT Gray/RGB/RGBA `U8`/`U16`/`I16`, all +four requests, both layouts, resident and external destinations, multi-tile +regressions, asynchronous drop and session reuse, and a 2,000-operation soak. +Its Burn targets validate the same dtype/request/layout boundary through direct +tensor destinations, plus prepared regrouping, group isolation, drop-safe +reuse, and a 1,000-batch soak. Reversible output is bit-exact; irreversible 9/7 +output agrees with the CPU oracle within one integer LSB. This establishes the +support boundary, not a throughput advantage. An explicit CUDA request still +fails rather than staging through CPU if runtime validation, device identity, +or retained-plan validation does not succeed. + +Metal hardware tests are bit-exact for independent multi-tile HT Gray12 and +RGB8 external output across full, region, reduced, and region-plus-reduced +requests, and for generated multi-tile classic RGB8 full external output. This +supplements the hardware-validated single-tile matrix; it does not imply that +every multi-tile dtype/request/layout combination has been run. + +CUDA and Metal external destinations support both NCHW and NHWC. A resident +image-surface view is exposed only for NHWC because `Surface` denotes +interleaved pixels; CUDA also exposes an explicit dense resident owner for +NCHW, while Metal NCHW callers use the dense external-destination API. Both +prepared offset-plan forms retain every participating tile. Nonzero ROI +maxshift declared by a codestream RGN marker remains unsupported on both GPU +prepared routes. + +The GPU sessions retain successful homogeneous groups when a different group +fails. Device HT jobs keep their original source identity through pass +bucketing and chunk splitting, so a status record that identifies a failing job +can name its source. When a lower-level failure cannot identify one image, the +entire dense group is discarded and its group error preserves every affected +source index. Partially written tensors are never returned. + +Aggregate HT descriptor and compressed arenas are split into bounded, +pass-homogeneous chunks without changing the public output grouping. A single +job that cannot fit the configured hard limits returns a structured error; it +is not permission to stage decoded pixels through the host. + +Reversible 5/3 output is required to match the CPU integer oracle bit for bit. +Irreversible 9/7 reconstruction may differ from that oracle by at most one +integer LSB. ## Validation and benchmarks @@ -88,9 +176,56 @@ select either CPU backend. Criterion group names include `flex` or `ndarray_arm_linux`; results from those groups are not cross-backend comparisons. -Run `cargo bench -p j2k-ml --bench tensor_decode --features cpu` for the -portable staged cases. Add `metal` on macOS or `cuda` on a Linux CUDA runner to -include the accelerator routes. Benchmark IDs and throughput record compact -upload bytes, packed Metal readback-plus-upload bytes, or zero decoded-pixel -transfer bytes for CUDA direct. CUDA should be described as the fast path only -after direct results beat its staged baseline on the target hardware. +Run `cargo bench -p j2k-ml --bench batch_decode --features cpu` for the +portable owned-batch benchmark. It reports codec-resident CPU output and +Burn-materialized output separately, includes one-shot and prepared reuse, and +covers HT-dominant unsigned and signed Gray12/Gray16 plus RGB8/RGB16 and +RGBA8/RGBA16 workloads at batch sizes 1/8/32/64 with full, ROI, reduced, and +ROI-and-reduced requests. The CUDA and Metal harnesses cover the same request +and native-output matrix. Each accelerator harness also retains a +`staged_cpu_upload_pixels` row so direct device decode can be compared with the +supported persistent CPU-decode-and-upload path on the same Burn backend. +Run the hardware matrices with `cargo xtask j2k-ml-bench-cuda` and +`cargo xtask j2k-ml-bench-metal`. Criterion reports decoded pixels per second; +divide by the decoded pixels per image for images per second or by 1,000,000 +for megapixels per second. The default `J2K_ML_BATCH_INPUT_MODE=distinct` +generates 64 deterministic, content-distinct codestreams per workload outside +the measured region and retains only one materialized workload at a time. +`J2K_ML_BATCH_INPUT_MODE=repeated` is an explicitly labeled broadcast/reuse +diagnostic that clones one `Arc`; a process and its retained sessions never mix +the two modes. The input mode is part of every Criterion group ID. + +Criterion runs default to `J2K_ML_BATCH_PROCESS_MODE=criterion` and do not +collect the one-shot telemetry probes. Metal Criterion runs reject enabled +stage timing, signposts, split-command profiling, or Xcode capture. Run a +separate low-batch diagnostic process with +`J2K_ML_BATCH_PROCESS_MODE=profile`; it covers only batches 1 and 8 and emits +telemetry without running Criterion. Codec-resident iterations wait for codec +completion. CUDA Burn-direct measurements retain their final consumer sync. +Synchronous Metal Burn-direct decode has already completed and validated the +codec work before returning, so it does not add a redundant `Wgpu::sync()`; +the staged CPU-upload row retains that synchronization. + +No publication throughput run has yet been recorded for the new batch +architecture. A July 19, 2026 local M4 Pro diagnostic run exercised the full +Metal matrix and is recorded in `docs/benchmark-evidence.md`; it does not +replace a pinned external adoption run. The CUDA harness emits codec-runtime H2D/D2H, kernel, +runtime-owned allocation, live/high-water memory, pool, event, and host-wait +counters in `cuda_telemetry_v2` rows for one completed probe per resident or +Burn-direct case. Probe throughput and counter deltas describe the same +completed decode and include the input mode. Criterion remains the separately +sampled throughput result. Those counters explicitly exclude Burn/CubeCL allocation and consumer +kernels. The high-water values are session-cumulative rather than per-case peak +memory. `consumer_host_syncs` records the explicit Burn synchronization, not a +CUDA driver wait. The Metal harness emits `metal_telemetry_v2` codec submission +and retained-pool counters plus input mode. Its decoded-transfer, +final-destination, group-wait, and consumer-synchronization columns are +prefixed `asserted_`: they disclose route contracts checked by focused tests, +not sampled hardware counters. The codec submission delta is prefixed +`measured_` and comes from session diagnostics. Neither profile +harness performs an unrecorded warmup: the first one-shot decode is cold, and +the first prepared decode includes its immutable-arena upload. Both backends must record +device-specific direct-versus-staged results before either route is described +as faster than the portable or staged path. Metal remains explicit pending a +corrected content-distinct run; the historical July 19 batch-greater-than-one +rows are qualified in `docs/benchmark-evidence.md`. diff --git a/docs/stable-api-1.0.implementation-public-api.txt b/docs/stable-api-1.0.implementation-public-api.txt index b140597d..3ec09e4c 100644 --- a/docs/stable-api-1.0.implementation-public-api.txt +++ b/docs/stable-api-1.0.implementation-public-api.txt @@ -23,6 +23,7 @@ impl j2k_core::traits::TileBatchDecode for j2k::J2kCodec impl<'a> j2k_core::traits::ImageDecode<'a> for j2k::J2kDecoder<'a> impl<'a> j2k_core::traits::ImageDecodeRows<'a, u16> for j2k::J2kDecoder<'a> impl<'a> j2k_core::traits::ImageDecodeRows<'a, u8> for j2k::J2kDecoder<'a> +pub const fn j2k::BatchGroupInfo::native_pixel_format(&self) -> core::option::Option pub fn j2k::J2kCodec::decode_tile(&mut Self::Context, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> pub fn j2k::J2kCodec::decode_tile_region(&mut Self::Context, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> pub fn j2k::J2kCodec::decode_tile_region_scaled(&mut Self::Context, &mut Self::Pool, j2k_core::pixel::PixelFormat, j2k_core::batch::TileRegionScaledDecodeJob<'_, '_>) -> core::result::Result, Self::Error> @@ -45,6 +46,8 @@ pub fn j2k::J2kError::is_truncated(&self) -> bool pub fn j2k::J2kError::is_unsupported(&self) -> bool pub fn j2k::J2kScratchPool::bytes_allocated(&self) -> usize pub fn j2k::J2kScratchPool::reset(&mut self) +pub fn j2k::PreparedClassicPlan::adapter_view(&self) -> &(dyn core::any::Any + core::marker::Send + core::marker::Sync) +pub fn j2k::PreparedHtj2kPlan::adapter_view(&self) -> &(dyn core::any::Any + core::marker::Send + core::marker::Sync) pub fn j2k::decode_tile_into_in_context(&[u8], &mut j2k::J2kContext, &mut j2k::J2kScratchPool, j2k::TileDecodeOutput<'_>) -> core::result::Result, j2k::J2kError> pub fn j2k::decode_tile_region_into_in_context(&[u8], &mut j2k::J2kContext, &mut j2k::J2kScratchPool, j2k::TileDecodeOutput<'_>, j2k_core::types::Rect) -> core::result::Result, j2k::J2kError> pub fn j2k::decode_tile_region_scaled_into_in_context(&[u8], &mut j2k::J2kContext, &mut j2k::J2kScratchPool, j2k::TileDecodeOutput<'_>, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, j2k::J2kError> @@ -119,11 +122,17 @@ pub use j2k::PrequantizedHtj2k97Subband ## `j2k-core` ```text +#[non_exhaustive] pub enum j2k_core::HtGpuJobChunkPlanError impl j2k_core::BatchAllocationBudget impl j2k_core::BatchAllocationRequest impl j2k_core::HostAllocationBudget impl j2k_core::HostAllocationError impl j2k_core::HostAllocationLimitError +impl j2k_core::HtGpuJobChunk +impl j2k_core::HtGpuJobChunkEntry +impl j2k_core::HtGpuJobChunkLimits +impl j2k_core::HtGpuJobChunkPlan +impl j2k_core::HtGpuJobChunkRequest impl j2k_core::accelerator::GpuAbi for f32 impl j2k_core::accelerator::GpuAbi for f64 impl j2k_core::accelerator::GpuAbi for i16 @@ -153,6 +162,21 @@ pub const fn j2k_core::HostAllocationError::for_elements(usize) -> Self pub const fn j2k_core::HostAllocationError::requested_bytes(self) -> usize pub const fn j2k_core::HostAllocationLimitError::cap_bytes(self) -> usize pub const fn j2k_core::HostAllocationLimitError::requested_bytes(self) -> usize +pub const fn j2k_core::HtGpuJobChunk::bucket(self) -> j2k_core::HtGpuJobPassBucket +pub const fn j2k_core::HtGpuJobChunk::descriptor_bytes(self) -> usize +pub const fn j2k_core::HtGpuJobChunk::job_count(self) -> usize +pub const fn j2k_core::HtGpuJobChunk::payload_bytes(self) -> usize +pub const fn j2k_core::HtGpuJobChunkEntry::original_job_index(self) -> usize +pub const fn j2k_core::HtGpuJobChunkEntry::source_index(self) -> usize +pub const fn j2k_core::HtGpuJobChunkLimits::max_descriptor_bytes(self) -> usize +pub const fn j2k_core::HtGpuJobChunkLimits::max_jobs(self) -> core::num::nonzero::NonZeroUsize +pub const fn j2k_core::HtGpuJobChunkLimits::max_payload_bytes(self) -> usize +pub const fn j2k_core::HtGpuJobChunkLimits::new(core::num::nonzero::NonZeroUsize, usize, usize) -> Self +pub const fn j2k_core::HtGpuJobChunkRequest::coding_passes(self) -> u8 +pub const fn j2k_core::HtGpuJobChunkRequest::descriptor_bytes(self) -> usize +pub const fn j2k_core::HtGpuJobChunkRequest::new(usize, u8, usize, usize) -> Self +pub const fn j2k_core::HtGpuJobChunkRequest::payload_bytes(self) -> usize +pub const fn j2k_core::HtGpuJobChunkRequest::source_index(self) -> usize pub const fn j2k_core::host_capacity_bytes(usize) -> usize pub const fn j2k_core::validate_cuda_surface_backend_request(j2k_core::BackendRequest) -> core::result::Result<(), j2k_core::BackendRequest> pub const i16::NAME: &'static str @@ -163,6 +187,8 @@ pub const u16::NAME: &'static str pub const u32::NAME: &'static str pub const u64::NAME: &'static str pub const u8::NAME: &'static str +pub enum j2k_core::HtGpuJobChunkLimit +pub enum j2k_core::HtGpuJobPassBucket pub fn T::decode_into(&mut self, &mut [u8], usize, j2k_core::PixelFormat) -> core::result::Result, Self::Error> pub fn T::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::PixelFormat) -> core::result::Result, Self::Error> pub fn T::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::PixelFormat, j2k_core::Rect) -> core::result::Result, Self::Error> @@ -206,6 +232,9 @@ pub fn j2k_core::HostAllocationBudget::account_bytes(&mut self, usize) -> core:: pub fn j2k_core::HostAllocationBudget::account_capacity(&mut self, usize) -> core::result::Result pub fn j2k_core::HostAllocationBudget::account_vec(&mut self, &alloc::vec::Vec) -> core::result::Result pub fn j2k_core::HostAllocationBudget::check_capacity(&self, usize) -> core::result::Result +pub fn j2k_core::HtGpuJobChunkPlan::chunk_entries(&self, usize) -> core::option::Option<&[j2k_core::HtGpuJobChunkEntry]> +pub fn j2k_core::HtGpuJobChunkPlan::chunks(&self) -> &[j2k_core::HtGpuJobChunk] +pub fn j2k_core::HtGpuJobChunkPlan::entries(&self) -> &[j2k_core::HtGpuJobChunkEntry] pub fn j2k_core::ReadySubmission::from_result(core::result::Result) -> Self pub fn j2k_core::ReadySubmission::wait(self) -> core::result::Result pub fn j2k_core::adapter_error_is_buffer_error(&impl j2k_core::AdapterErrorParts) -> bool @@ -217,6 +246,7 @@ pub fn j2k_core::checked_batch_count_sum(impl core::iter::traits::collect::IntoI pub fn j2k_core::checked_surface_len((u32, u32), usize, usize, &'static str) -> core::result::Result<(usize, usize), j2k_core::BufferError> pub fn j2k_core::copy_tight_pixels_to_strided_output(&[u8], (u32, u32), j2k_core::PixelFormat, &mut [u8], usize) -> core::result::Result<(), j2k_core::BufferError> pub fn j2k_core::ensure_allocation_within_cap(usize, usize, &'static str) -> core::result::Result +pub fn j2k_core::plan_ht_gpu_job_chunks(&[j2k_core::HtGpuJobChunkRequest], j2k_core::HtGpuJobChunkLimits) -> core::result::Result pub fn j2k_core::strided_output_len((u32, u32), usize, j2k_core::PixelFormat) -> core::result::Result pub fn j2k_core::strided_output_len_capped((u32, u32), usize, j2k_core::PixelFormat, usize, &'static str) -> core::result::Result pub fn j2k_core::submit_ready_device(&mut S, impl core::ops::function::FnOnce(&mut S) -> core::result::Result) -> j2k_core::ReadySubmission where S: j2k_core::DeviceSubmitSession + ?core::marker::Sized @@ -242,6 +272,22 @@ pub fn u64::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] pub fn u8::as_bytes(&Self) -> &[u8] pub fn u8::slice_as_bytes(&[Self]) -> &[u8] pub fn u8::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +pub j2k_core::HtGpuJobChunkLimit::DescriptorBytes +pub j2k_core::HtGpuJobChunkLimit::PayloadBytes +pub j2k_core::HtGpuJobChunkPlanError::BatchInfrastructure(j2k_core::BatchInfrastructureError) +pub j2k_core::HtGpuJobChunkPlanError::InvalidCodingPassCount +pub j2k_core::HtGpuJobChunkPlanError::InvalidCodingPassCount::coding_passes: u8 +pub j2k_core::HtGpuJobChunkPlanError::InvalidCodingPassCount::original_job_index: usize +pub j2k_core::HtGpuJobChunkPlanError::InvalidCodingPassCount::source_index: usize +pub j2k_core::HtGpuJobChunkPlanError::SingleJobTooLarge +pub j2k_core::HtGpuJobChunkPlanError::SingleJobTooLarge::cap: usize +pub j2k_core::HtGpuJobChunkPlanError::SingleJobTooLarge::limit: j2k_core::HtGpuJobChunkLimit +pub j2k_core::HtGpuJobChunkPlanError::SingleJobTooLarge::original_job_index: usize +pub j2k_core::HtGpuJobChunkPlanError::SingleJobTooLarge::requested: usize +pub j2k_core::HtGpuJobChunkPlanError::SingleJobTooLarge::source_index: usize +pub j2k_core::HtGpuJobPassBucket::CleanupOnly +pub j2k_core::HtGpuJobPassBucket::MagRef +pub j2k_core::HtGpuJobPassBucket::SigProp pub macro j2k_core::__j2k_fnv1a64_bytes! pub macro j2k_core::__j2k_fnv1a64_init! pub macro j2k_core::__j2k_fnv1a64_update! @@ -250,6 +296,11 @@ pub struct j2k_core::BatchAllocationRequest pub struct j2k_core::HostAllocationBudget pub struct j2k_core::HostAllocationError pub struct j2k_core::HostAllocationLimitError +pub struct j2k_core::HtGpuJobChunk +pub struct j2k_core::HtGpuJobChunkEntry +pub struct j2k_core::HtGpuJobChunkLimits +pub struct j2k_core::HtGpuJobChunkPlan +pub struct j2k_core::HtGpuJobChunkRequest pub struct j2k_core::ReadySubmission(_) pub trait j2k_core::CpuBackedImageDecode<'a>: j2k_core::ImageCodec + core::marker::Sized + 'a pub trait j2k_core::DeviceSubmitSession @@ -952,9 +1003,11 @@ pub unsafe fn j2k_jpeg_metal::ResidentPrivateJpegTile::into_buffer(self) -> meta ```text #[non_exhaustive] pub struct j2k_metal::MetalLosslessEncodeBatchStats #[non_exhaustive] pub struct j2k_metal::MetalLosslessEncodeStageStats +impl core::ops::drop::Drop for j2k_metal::MetalSubmission impl j2k_core::accelerator::AcceleratorSession for j2k_metal::MetalBackendSession impl j2k_core::error::AdapterErrorParts for j2k_metal::Error impl j2k_core::error::CodecError for j2k_metal::Error +impl j2k_core::traits::DeviceSubmission for j2k_metal::MetalSubmission impl j2k_core::traits::DeviceSubmission for j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch impl j2k_core::traits::DeviceSubmission for j2k_metal::SubmittedJ2kLosslessMetalEncodeBatch impl j2k_core::traits::DeviceSurface for j2k_metal::Surface @@ -963,6 +1016,7 @@ impl j2k_core::traits::ImageCodec for j2k_metal::J2kDecoder<'_> impl j2k_core::traits::TileBatchDecodeDevice for j2k_metal::Codec impl j2k_core::traits::TileBatchDecodeManyDevice for j2k_metal::Codec impl j2k_core::traits::TileBatchDecodeSubmit for j2k_metal::Codec +impl j2k_metal::Error impl j2k_metal::J2kDecoder<'_> impl j2k_metal::MetalLosslessEncodeBatchStats impl j2k_metal::MetalLosslessEncodeStageStats @@ -981,10 +1035,13 @@ pub fn j2k_metal::Error::is_buffer_error(&self) -> bool pub fn j2k_metal::Error::is_not_implemented(&self) -> bool pub fn j2k_metal::Error::is_truncated(&self) -> bool pub fn j2k_metal::Error::is_unsupported(&self) -> bool +pub fn j2k_metal::Error::session_is_unusable(&self) -> bool pub fn j2k_metal::Error::source_codec_error(&self) -> core::option::Option<&dyn j2k_core::error::CodecError> pub fn j2k_metal::J2kDecoder<'_>::decode_repeated_color_direct_to_device(&mut self, j2k_core::pixel::PixelFormat, usize) -> core::result::Result, j2k_metal::Error> +pub fn j2k_metal::J2kDecoder<'_>::decode_repeated_color_direct_to_device_with_session(&mut self, j2k_core::pixel::PixelFormat, usize, &j2k_metal::MetalBackendSession) -> core::result::Result, j2k_metal::Error> pub fn j2k_metal::J2kDecoder<'_>::decode_repeated_grayscale_auto_to_device(&mut self, j2k_core::pixel::PixelFormat, usize) -> core::result::Result, j2k_metal::Error> pub fn j2k_metal::J2kDecoder<'_>::decode_repeated_grayscale_direct_to_device(&mut self, j2k_core::pixel::PixelFormat, usize) -> core::result::Result, j2k_metal::Error> +pub fn j2k_metal::J2kDecoder<'_>::decode_repeated_grayscale_direct_to_device_with_session(&mut self, j2k_core::pixel::PixelFormat, usize, &j2k_metal::MetalBackendSession) -> core::result::Result, j2k_metal::Error> pub fn j2k_metal::J2kDecoder<'a>::cpu_decoder_mut(&mut self) -> &mut Self::Cpu pub fn j2k_metal::J2kDecoder<'a>::decode_request_to_device_with_report(&mut self, j2k_metal::MetalDecodeRequest) -> core::result::Result pub fn j2k_metal::J2kDecoder<'a>::from_cpu_view(Self::View) -> core::result::Result @@ -996,6 +1053,7 @@ pub fn j2k_metal::J2kDecoder<'a>::submit_region_to_device(&mut self, &mut Self:: pub fn j2k_metal::J2kDecoder<'a>::submit_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result pub fn j2k_metal::J2kDecoder<'a>::submit_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result pub fn j2k_metal::MetalBackendSession::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_metal::MetalBackendSession::uses_command_queue(&self, &metal::commandqueue::CommandQueueRef) -> core::result::Result pub fn j2k_metal::MetalEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport pub fn j2k_metal::MetalEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> j2k_types::stage_error::J2kEncodeStageResult>>> pub fn j2k_metal::MetalEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> j2k_types::stage_error::J2kEncodeStageResult> @@ -1014,6 +1072,8 @@ pub fn j2k_metal::MetalLosslessEncodeBatchStats::new() -> Self pub fn j2k_metal::MetalLosslessEncodeStageStats::add_assign(&mut self, Self) pub fn j2k_metal::MetalLosslessEncodeStageStats::has_timings(&self) -> bool pub fn j2k_metal::MetalLosslessEncodeStageStats::new() -> Self +pub fn j2k_metal::MetalSubmission::drop(&mut self) +pub fn j2k_metal::MetalSubmission::wait(self) -> core::result::Result pub fn j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch::wait(self) -> core::result::Result pub fn j2k_metal::SubmittedJ2kLosslessMetalEncodeBatch::wait(self) -> core::result::Result pub fn j2k_metal::Surface::backend_kind(&self) -> j2k_core::backend::BackendKind @@ -1218,12 +1278,13 @@ pub struct j2k_metal::DecodeSurfaceWithReport pub struct j2k_metal::MetalLosslessBufferEncodeBatchOutcome pub struct j2k_metal::MetalLosslessBufferEncodeOutcome pub struct j2k_metal::MetalLosslessEncodeOutcome +pub struct j2k_metal::MetalSubmission pub type j2k_metal::Codec::Context = j2k::context::J2kContext pub type j2k_metal::Codec::DeviceSurface = j2k_metal::Surface pub type j2k_metal::Codec::Error = j2k_metal::Error pub type j2k_metal::Codec::Pool = j2k::scratch::J2kScratchPool pub type j2k_metal::Codec::Session = j2k_metal::MetalSession -pub type j2k_metal::Codec::SubmittedSurface = j2k_metal::batch::MetalSubmission +pub type j2k_metal::Codec::SubmittedSurface = j2k_metal::MetalSubmission pub type j2k_metal::Codec::Warning = j2k::decode::J2kDecodeWarning pub type j2k_metal::J2kDecoder<'_>::Error = j2k_metal::Error pub type j2k_metal::J2kDecoder<'_>::Pool = j2k::scratch::J2kScratchPool @@ -1233,6 +1294,8 @@ pub type j2k_metal::J2kDecoder<'a>::DeviceSurface = j2k_metal::Surface pub type j2k_metal::J2kDecoder<'a>::Session = j2k_metal::MetalSession pub type j2k_metal::J2kDecoder<'a>::SubmittedSurface = j2k_core::traits::ReadySubmission pub type j2k_metal::J2kDecoder<'a>::View = j2k::view::J2kView<'a> +pub type j2k_metal::MetalSubmission::Error = j2k_metal::Error +pub type j2k_metal::MetalSubmission::Output = j2k_metal::Surface pub type j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch::Error = j2k_metal::Error pub type j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch::Output = j2k_metal::MetalLosslessBufferEncodeBatchOutcome pub type j2k_metal::SubmittedJ2kLosslessMetalEncodeBatch::Error = j2k_metal::Error @@ -1374,6 +1437,7 @@ pub type j2k_jpeg_cuda::Decoder<'a>::View = j2k_jpeg::decoder::view::JpegView<'a #[non_exhaustive] pub struct j2k_cuda::CudaHtj2kDecodeProfileDetail #[non_exhaustive] pub struct j2k_cuda::CudaHtj2kEncodeProfileReport #[non_exhaustive] pub struct j2k_cuda::CudaHtj2kProfileReport +impl core::convert::From for j2k_cuda::CudaDecodePoolSnapshot impl core::default::Default for j2k_cuda::CudaHtj2kEncodeProfileReport impl j2k_core::accelerator::AcceleratorSession for j2k_cuda::CudaSession impl j2k_core::error::AdapterErrorParts for j2k_cuda::Error @@ -1387,10 +1451,12 @@ impl j2k_core::traits::ImageCodec for j2k_cuda::J2kDecoder<'_> impl j2k_core::traits::TileBatchDecodeDevice for j2k_cuda::Codec impl j2k_core::traits::TileBatchDecodeManyDevice for j2k_cuda::Codec impl j2k_core::traits::TileBatchDecodeSubmit for j2k_cuda::Codec +impl j2k_cuda::CudaBatchError impl j2k_cuda::CudaHtj2kDecodeProfileDetail impl j2k_cuda::CudaHtj2kEncodeProfileReport impl j2k_cuda::CudaHtj2kProfileReport impl j2k_cuda::CudaSurfaceStats +impl j2k_cuda::Error impl j2k_types::J2kEncodeStageAccelerator for j2k_cuda::CudaEncodeStageAccelerator impl<'a> j2k_core::traits::CpuBackedImageDecode<'a> for j2k_cuda::J2kDecoder<'a> impl<'a> j2k_core::traits::ImageDecodeDevice<'a> for j2k_cuda::J2kDecoder<'a> @@ -1400,6 +1466,9 @@ pub fn j2k_cuda::Codec::submit_tile_region_scaled_to_device(&mut Self::Context, pub fn j2k_cuda::Codec::submit_tile_region_to_device(&mut Self::Context, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result pub fn j2k_cuda::Codec::submit_tile_scaled_to_device(&mut Self::Context, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result pub fn j2k_cuda::Codec::submit_tile_to_device(&mut Self::Context, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_cuda::CudaBatchError::completion_is_uncertain(&self) -> bool +pub fn j2k_cuda::CudaBatchError::session_is_unusable(&self) -> bool +pub fn j2k_cuda::CudaDecodePoolSnapshot::from(j2k_cuda_runtime::memory::pool::cache_policy::CudaBufferPoolDiagnostics) -> Self pub fn j2k_cuda::CudaEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> j2k_types::stage_error::J2kEncodeStageResult>>> pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> j2k_types::stage_error::J2kEncodeStageResult> @@ -1419,6 +1488,7 @@ pub fn j2k_cuda::CudaHtj2kEncodeProfileReport::default() -> Self pub fn j2k_cuda::CudaHtj2kEncodeProfileReport::new() -> Self pub fn j2k_cuda::CudaHtj2kProfileReport::new() -> Self pub fn j2k_cuda::CudaSession::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_cuda::CudaSession::context_for_device_interop(&mut self, usize) -> core::result::Result pub fn j2k_cuda::CudaSession::execution_stats(&self) -> j2k_core::accelerator::ExecutionStats pub fn j2k_cuda::CudaSession::record_submit(&mut self) pub fn j2k_cuda::CudaSession::with_context(j2k_cuda_runtime::context::CudaContext) -> Self @@ -1427,12 +1497,15 @@ pub fn j2k_cuda::CudaSurfaceStats::copy_kernel_dispatches(self) -> usize pub fn j2k_cuda::CudaSurfaceStats::decode_kernel_dispatches(self) -> usize pub fn j2k_cuda::CudaSurfaceStats::kernel_dispatches(self) -> usize pub fn j2k_cuda::Error::adapter_error_kind(&self) -> j2k_core::error::AdapterErrorKind +pub fn j2k_cuda::Error::completion_is_uncertain(&self) -> bool pub fn j2k_cuda::Error::is_buffer_error(&self) -> bool pub fn j2k_cuda::Error::is_not_implemented(&self) -> bool pub fn j2k_cuda::Error::is_truncated(&self) -> bool pub fn j2k_cuda::Error::is_unsupported(&self) -> bool +pub fn j2k_cuda::Error::session_is_unusable(&self) -> bool pub fn j2k_cuda::Error::source_codec_error(&self) -> core::option::Option<&dyn j2k_core::error::CodecError> pub fn j2k_cuda::J2kDecoder<'a>::cpu_decoder_mut(&mut self) -> &mut Self::Cpu +pub fn j2k_cuda::J2kDecoder<'a>::decode_batch_into_external_device_with_session(&[&[u8]], j2k_core::pixel::PixelFormat, &mut j2k_cuda_runtime::memory::CudaExternalDeviceBufferViewMut<'_>, &mut j2k_cuda::CudaSession) -> core::result::Result<(alloc::vec::Vec, j2k_cuda::CudaHtj2kProfileReport), j2k_cuda::Error> pub fn j2k_cuda::J2kDecoder<'a>::decode_batch_to_device_with_session_and_profile(&[&[u8]], j2k_core::pixel::PixelFormat, &mut j2k_cuda::CudaSession) -> core::result::Result<(alloc::vec::Vec, j2k_cuda::CudaHtj2kProfileReport), j2k_cuda::Error> pub fn j2k_cuda::J2kDecoder<'a>::decode_request_to_device_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k::adapter::device_plan::DeviceDecodeRequest, &mut j2k_cuda::CudaSession) -> core::result::Result pub fn j2k_cuda::J2kDecoder<'a>::decode_to_device_with_session_and_profile(&mut self, j2k_core::pixel::PixelFormat, &mut j2k_cuda::CudaSession) -> core::result::Result<(j2k_cuda::Surface, j2k_cuda::CudaHtj2kProfileReport), j2k_cuda::Error> @@ -2107,6 +2180,7 @@ pub struct j2k_transcode_metal::weights::SparseWeightTap impl core::convert::From for j2k_native::J2kRect impl core::error::Error for j2k_native::ResidentHtj2kEncodeError impl core::error::Error for j2k_native::packet_math::HtSegmentLengthError +impl core::fmt::Debug for j2k_native::J2kCodeBlockDecodeWorkspace impl core::fmt::Display for j2k_native::ResidentHtj2kEncodeError impl core::fmt::Display for j2k_native::packet_math::HtSegmentLengthError impl j2k_native::HtCodeBlockDecodeProfile @@ -2115,13 +2189,39 @@ impl j2k_native::HtSigPropBenchmarkState impl j2k_native::Image<'_> impl j2k_native::J2kCodeBlockDecodeProfile impl j2k_native::J2kDirectColorPlan +impl j2k_native::J2kDirectCpuEntropyWorkspace impl j2k_native::J2kDirectCpuScratch +impl j2k_native::J2kDirectDecodedComponents<'_> impl j2k_native::J2kDirectGrayscalePlan +impl j2k_native::J2kDirectRgbaPlan impl j2k_native::J2kRect +impl j2k_native::J2kReferencedClassicPlan +impl j2k_native::J2kReferencedHtj2kPlan +impl j2k_native::J2kReferencedPayloadRecordSpan +impl j2k_native::J2kReferencedTilePlan impl j2k_native::J2kRequiredBandRegion impl j2k_native::Reversible53CoefficientImage impl j2k_native::packet_math::HtSegmentLengthError +impl<'a> j2k_native::J2kDirectDecodedPlane<'a> pub const fn j2k_native::J2kDirectCpuScratch::new() -> Self +pub const fn j2k_native::J2kDirectDecodedComponents<'_>::component_count(&self) -> usize +pub const fn j2k_native::J2kDirectDecodedComponents<'_>::dimensions(&self) -> (u32, u32) +pub const fn j2k_native::J2kDirectDecodedPlane<'a>::bit_depth(self) -> u8 +pub const fn j2k_native::J2kDirectDecodedPlane<'a>::dimensions(self) -> (u32, u32) +pub const fn j2k_native::J2kDirectDecodedPlane<'a>::samples(self) -> &'a [f32] +pub const fn j2k_native::J2kReferencedClassicPlan::full_dimensions(&self) -> (u32, u32) +pub const fn j2k_native::J2kReferencedClassicPlan::output_rect(&self) -> j2k_native::J2kRect +pub const fn j2k_native::J2kReferencedHtj2kPlan::full_dimensions(&self) -> (u32, u32) +pub const fn j2k_native::J2kReferencedHtj2kPlan::output_rect(&self) -> j2k_native::J2kRect +pub const fn j2k_native::J2kReferencedPayloadRecordSpan::end_record(self) -> core::option::Option +pub const fn j2k_native::J2kReferencedTilePlan::color_geometry(&self) -> core::option::Option<&j2k_native::J2kDirectColorPlan> +pub const fn j2k_native::J2kReferencedTilePlan::decoded_rect(&self) -> j2k_native::J2kRect +pub const fn j2k_native::J2kReferencedTilePlan::destination_rect(&self) -> j2k_native::J2kRect +pub const fn j2k_native::J2kReferencedTilePlan::grayscale_geometry(&self) -> core::option::Option<&j2k_native::J2kDirectGrayscalePlan> +pub const fn j2k_native::J2kReferencedTilePlan::payload_records(&self) -> j2k_native::J2kReferencedPayloadRecordSpan +pub const fn j2k_native::J2kReferencedTilePlan::rgba_geometry(&self) -> core::option::Option<&j2k_native::J2kDirectRgbaPlan> +pub const fn j2k_native::J2kReferencedTilePlan::tile_index(&self) -> usize +pub const fn j2k_native::J2kReferencedTilePlan::wavelet_transform(&self) -> j2k_native::J2kWaveletTransform pub const fn j2k_native::J2kRequiredBandRegion::full(u32, u32) -> Self pub const fn j2k_native::J2kRequiredBandRegion::to_rect(self) -> j2k_native::J2kRect pub const fn j2k_native::idwt_required_output_margin(j2k_native::J2kWaveletTransform) -> u32 @@ -2130,6 +2230,9 @@ pub const j2k_native::DEFAULT_MAX_CODEC_BYTES: usize pub const j2k_native::DEFAULT_MAX_DECODE_BYTES: usize pub enum j2k_native::HtCodeBlockDecodePhaseLimit pub enum j2k_native::J2kDirectGrayscaleStep +pub enum j2k_native::J2kReferencedClassicPlan +pub enum j2k_native::J2kReferencedHtj2kPlan +pub enum j2k_native::J2kReferencedTileGeometry pub enum j2k_native::J2kWaveletTransform pub enum j2k_native::Jp2ChannelAssociation pub enum j2k_native::Jp2ChannelType @@ -2157,6 +2260,8 @@ pub fn j2k_native::Image<'a>::build_direct_color_plan_region_with_context(&self, pub fn j2k_native::Image<'a>::build_direct_color_plan_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::Result pub fn j2k_native::Image<'a>::build_direct_grayscale_plan_region_with_context(&self, &mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32)) -> j2k_native::Result pub fn j2k_native::Image<'a>::build_direct_grayscale_plan_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::Result +pub fn j2k_native::Image<'a>::build_referenced_classic_plan_region_with_context(&self, &mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32)) -> j2k_native::Result +pub fn j2k_native::Image<'a>::build_referenced_htj2k_plan_region_with_context(&self, &mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32)) -> j2k_native::Result pub fn j2k_native::Image<'a>::decode_components_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::Result> pub fn j2k_native::Image<'a>::decode_native_components_with_retained_capacity(&self, usize) -> j2k_native::Result pub fn j2k_native::Image<'a>::decode_native_with_retained_capacity(&self, usize) -> j2k_native::Result @@ -2167,13 +2272,34 @@ pub fn j2k_native::Image<'a>::new_with_retained_baseline(&'a [u8], &j2k_native:: pub fn j2k_native::Image<'a>::retained_allocation_bytes(&self) -> j2k_native::Result pub fn j2k_native::Image<'a>::supports_direct_device_plane_reuse(&self) -> bool pub fn j2k_native::J2kCodeBlockDecodeProfile::new() -> Self +pub fn j2k_native::J2kCodeBlockDecodeWorkspace::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_native::J2kDirectColorPlan::retained_allocation_bytes(&self) -> j2k_native::Result +pub fn j2k_native::J2kDirectCpuEntropyWorkspace::retained_classic_bytes(&self) -> usize +pub fn j2k_native::J2kDirectCpuEntropyWorkspace::retained_ht_bytes(&self) -> usize pub fn j2k_native::J2kDirectCpuScratch::clear(&mut self) pub fn j2k_native::J2kDirectCpuScratch::prepare_execution_allocation_bytes(&mut self, &j2k_native::J2kDirectColorPlan) -> j2k_native::Result +pub fn j2k_native::J2kDirectCpuScratch::retained_band_owner_count(&self) -> usize +pub fn j2k_native::J2kDirectCpuScratch::retained_classic_workspace_bytes(&self) -> usize +pub fn j2k_native::J2kDirectCpuScratch::retained_ht_workspace_bytes(&self) -> usize +pub fn j2k_native::J2kDirectDecodedComponents<'_>::plane(&self, usize) -> core::option::Option> pub fn j2k_native::J2kDirectGrayscalePlan::retained_allocation_bytes(&self) -> j2k_native::Result +pub fn j2k_native::J2kDirectRgbaPlan::retained_allocation_bytes(&self) -> j2k_native::Result pub fn j2k_native::J2kRect::from(j2k_native::J2kRequiredBandRegion) -> Self pub fn j2k_native::J2kRect::height(self) -> u32 pub fn j2k_native::J2kRect::width(self) -> u32 +pub fn j2k_native::J2kReferencedClassicPlan::color_geometry(&self) -> core::option::Option<&j2k_native::J2kDirectColorPlan> +pub fn j2k_native::J2kReferencedClassicPlan::grayscale_geometry(&self) -> core::option::Option<&j2k_native::J2kDirectGrayscalePlan> +pub fn j2k_native::J2kReferencedClassicPlan::payloads(&self) -> &[j2k_types::decode_payload::J2kClassicCodeBlockPayload] +pub fn j2k_native::J2kReferencedClassicPlan::ranges(&self) -> &[j2k_types::decode_payload::J2kCodestreamRange] +pub fn j2k_native::J2kReferencedClassicPlan::retained_allocation_bytes(&self) -> j2k_native::Result +pub fn j2k_native::J2kReferencedClassicPlan::rgba_geometry(&self) -> core::option::Option<&j2k_native::J2kDirectRgbaPlan> +pub fn j2k_native::J2kReferencedClassicPlan::tiles(&self) -> &[j2k_native::J2kReferencedTilePlan] +pub fn j2k_native::J2kReferencedHtj2kPlan::color_geometry(&self) -> core::option::Option<&j2k_native::J2kDirectColorPlan> +pub fn j2k_native::J2kReferencedHtj2kPlan::grayscale_geometry(&self) -> core::option::Option<&j2k_native::J2kDirectGrayscalePlan> +pub fn j2k_native::J2kReferencedHtj2kPlan::payloads(&self) -> &[j2k_types::decode_payload::HtCodeBlockPayloadRanges] +pub fn j2k_native::J2kReferencedHtj2kPlan::retained_allocation_bytes(&self) -> j2k_native::Result +pub fn j2k_native::J2kReferencedHtj2kPlan::rgba_geometry(&self) -> core::option::Option<&j2k_native::J2kDirectRgbaPlan> +pub fn j2k_native::J2kReferencedHtj2kPlan::tiles(&self) -> &[j2k_native::J2kReferencedTilePlan] pub fn j2k_native::J2kRequiredBandRegion::expanded_within_band(self, u32, u32, u32) -> Self pub fn j2k_native::J2kRequiredBandRegion::expanded_within_rect(self, u32, j2k_native::J2kRect) -> Self pub fn j2k_native::J2kRequiredBandRegion::height(self) -> u32 @@ -2230,7 +2356,17 @@ pub fn j2k_native::encode_with_accelerator(&[u8], u32, u32, u16, u8, bool, &j2k_ pub fn j2k_native::encode_with_accelerator_and_roi_regions(&[u8], u32, u32, u16, u8, bool, &j2k_native::EncodeOptions, &[j2k_native::EncodeRoiRegion], &mut impl j2k_types::J2kEncodeStageAccelerator) -> j2k_native::EncodeResult> pub fn j2k_native::execute_direct_color_plan_rgb8_into(&j2k_native::J2kDirectColorPlan, j2k_native::J2kRect, &mut j2k_native::J2kDirectCpuScratch, &mut [u8], usize) -> j2k_native::Result<()> pub fn j2k_native::execute_direct_color_plan_rgba8_into(&j2k_native::J2kDirectColorPlan, j2k_native::J2kRect, &mut j2k_native::J2kDirectCpuScratch, &mut [u8], usize) -> j2k_native::Result<()> +pub fn j2k_native::execute_referenced_classic_entropy_job(&j2k_native::J2kReferencedClassicPlan, j2k_native::J2kDirectCodeBlockIndex, &[u8], j2k_types::decode_payload::J2kCodestreamRange, &mut j2k_native::J2kDirectCpuScratch, &mut j2k_native::J2kDirectCpuEntropyWorkspace) -> j2k_native::Result<()> +pub fn j2k_native::execute_referenced_classic_plan<'scratch>(&j2k_native::J2kReferencedClassicPlan, &[u8], bool, &'scratch mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result> +pub fn j2k_native::execute_referenced_classic_plan_from_payloads<'scratch>(&j2k_native::J2kReferencedClassicPlan, &[u8], &[j2k_types::decode_payload::J2kClassicCodeBlockPayload], &[j2k_types::decode_payload::J2kCodestreamRange], bool, &'scratch mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result> +pub fn j2k_native::execute_referenced_htj2k_entropy_job(&j2k_native::J2kReferencedHtj2kPlan, j2k_native::J2kDirectCodeBlockIndex, &[u8], j2k_types::decode_payload::HtCodeBlockPayloadRanges, &mut j2k_native::J2kDirectCpuScratch, &mut j2k_native::J2kDirectCpuEntropyWorkspace) -> j2k_native::Result<()> +pub fn j2k_native::execute_referenced_htj2k_plan<'scratch>(&j2k_native::J2kReferencedHtj2kPlan, &[u8], bool, &'scratch mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result> +pub fn j2k_native::execute_referenced_htj2k_plan_from_payloads<'scratch>(&j2k_native::J2kReferencedHtj2kPlan, &[u8], &[j2k_types::decode_payload::HtCodeBlockPayloadRanges], bool, &'scratch mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result> pub fn j2k_native::extract_jp2_codestream_payload(&[u8]) -> j2k_native::Result<(j2k_native::Jp2FileKind, usize, &[u8])> +pub fn j2k_native::finish_referenced_classic_staged<'scratch>(&j2k_native::J2kReferencedClassicPlan, bool, &'scratch mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result> +pub fn j2k_native::finish_referenced_classic_tile_staged(&j2k_native::J2kReferencedClassicPlan, usize, bool, &mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result<()> +pub fn j2k_native::finish_referenced_htj2k_staged<'scratch>(&j2k_native::J2kReferencedHtj2kPlan, bool, &'scratch mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result> +pub fn j2k_native::finish_referenced_htj2k_tile_staged(&j2k_native::J2kReferencedHtj2kPlan, usize, bool, &mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result<()> pub fn j2k_native::forward_dwt53_reference(&[f32], u32, u32, u8) -> j2k_native::EncodeResult pub fn j2k_native::forward_dwt97_reference(&[f32], u32, u32, u8) -> j2k_native::EncodeResult pub fn j2k_native::forward_ict_reference(alloc::vec::Vec>) -> alloc::vec::Vec> @@ -2256,6 +2392,12 @@ pub fn j2k_native::packet_math::bits_for_length(u32, u8) -> u32 pub fn j2k_native::packet_math::ht_segment_lengths(u8, usize, u32, u32) -> core::result::Result<(u32, u32), j2k_native::packet_math::HtSegmentLengthError> pub fn j2k_native::packet_math::value_fits_in_bits(u32, u32) -> bool pub fn j2k_native::prepare_ht_sigprop_benchmark_state(j2k_native::HtCodeBlockDecodeJob<'_>) -> j2k_native::Result +pub fn j2k_native::prepare_referenced_classic_entropy_workspace(&j2k_native::J2kReferencedClassicPlan, &mut j2k_native::J2kDirectCpuEntropyWorkspace) -> j2k_native::Result<()> +pub fn j2k_native::prepare_referenced_classic_staged(&j2k_native::J2kReferencedClassicPlan, &mut j2k_native::J2kDirectCpuScratch, &mut j2k_native::J2kDirectCpuEntropyWorkspace) -> j2k_native::Result<()> +pub fn j2k_native::prepare_referenced_classic_tile_staged(&j2k_native::J2kReferencedClassicPlan, usize, &mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result<()> +pub fn j2k_native::prepare_referenced_htj2k_entropy_workspace(&j2k_native::J2kReferencedHtj2kPlan, &mut j2k_native::J2kDirectCpuEntropyWorkspace) -> j2k_native::Result<()> +pub fn j2k_native::prepare_referenced_htj2k_staged(&j2k_native::J2kReferencedHtj2kPlan, &mut j2k_native::J2kDirectCpuScratch, &mut j2k_native::J2kDirectCpuEntropyWorkspace) -> j2k_native::Result<()> +pub fn j2k_native::prepare_referenced_htj2k_tile_staged(&j2k_native::J2kReferencedHtj2kPlan, usize, &mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result<()> pub fn j2k_native::quantize_reversible_reference(&[f32], u16, u16, u8, bool) -> alloc::vec::Vec pub fn j2k_native::quantize_subband_reference(&[f32], u16, u16, u8, bool) -> alloc::vec::Vec pub fn j2k_native::try_deinterleave_reference(&[u8], usize, u16, u8, bool) -> j2k_native::Result>> @@ -2351,6 +2493,10 @@ pub j2k_native::J2kCodeBlockDecodeProfile::cleanup_us: u128 pub j2k_native::J2kCodeBlockDecodeProfile::magref_us: u128 pub j2k_native::J2kCodeBlockDecodeProfile::output_convert_us: u128 pub j2k_native::J2kCodeBlockDecodeProfile::sigprop_us: u128 +pub j2k_native::J2kDirectCodeBlockIndex::code_block: usize +pub j2k_native::J2kDirectCodeBlockIndex::component: usize +pub j2k_native::J2kDirectCodeBlockIndex::step: usize +pub j2k_native::J2kDirectCodeBlockIndex::tile: usize pub j2k_native::J2kDirectColorPlan::bit_depths: [u8; 3] pub j2k_native::J2kDirectColorPlan::component_plans: alloc::vec::Vec pub j2k_native::J2kDirectColorPlan::dimensions: (u32, u32) @@ -2374,6 +2520,11 @@ pub j2k_native::J2kDirectIdwtStep::ll_band_id: j2k_native::J2kDirectBandId pub j2k_native::J2kDirectIdwtStep::output_band_id: j2k_native::J2kDirectBandId pub j2k_native::J2kDirectIdwtStep::rect: j2k_native::J2kRect pub j2k_native::J2kDirectIdwtStep::transform: j2k_native::J2kWaveletTransform +pub j2k_native::J2kDirectRgbaPlan::bit_depths: [u8; 4] +pub j2k_native::J2kDirectRgbaPlan::component_plans: alloc::vec::Vec +pub j2k_native::J2kDirectRgbaPlan::dimensions: (u32, u32) +pub j2k_native::J2kDirectRgbaPlan::mct: bool +pub j2k_native::J2kDirectRgbaPlan::transform: j2k_native::J2kWaveletTransform pub j2k_native::J2kDirectStoreStep::addend: f32 pub j2k_native::J2kDirectStoreStep::copy_height: u32 pub j2k_native::J2kDirectStoreStep::copy_width: u32 @@ -2422,6 +2573,44 @@ pub j2k_native::J2kRect::x0: u32 pub j2k_native::J2kRect::x1: u32 pub j2k_native::J2kRect::y0: u32 pub j2k_native::J2kRect::y1: u32 +pub j2k_native::J2kReferencedClassicPlan::Color +pub j2k_native::J2kReferencedClassicPlan::Color::full_dimensions: (u32, u32) +pub j2k_native::J2kReferencedClassicPlan::Color::output_rect: j2k_native::J2kRect +pub j2k_native::J2kReferencedClassicPlan::Color::payloads: alloc::vec::Vec +pub j2k_native::J2kReferencedClassicPlan::Color::ranges: alloc::vec::Vec +pub j2k_native::J2kReferencedClassicPlan::Color::tiles: alloc::vec::Vec +pub j2k_native::J2kReferencedClassicPlan::Grayscale +pub j2k_native::J2kReferencedClassicPlan::Grayscale::full_dimensions: (u32, u32) +pub j2k_native::J2kReferencedClassicPlan::Grayscale::output_rect: j2k_native::J2kRect +pub j2k_native::J2kReferencedClassicPlan::Grayscale::payloads: alloc::vec::Vec +pub j2k_native::J2kReferencedClassicPlan::Grayscale::ranges: alloc::vec::Vec +pub j2k_native::J2kReferencedClassicPlan::Grayscale::tiles: alloc::vec::Vec +pub j2k_native::J2kReferencedClassicPlan::Rgba +pub j2k_native::J2kReferencedClassicPlan::Rgba::full_dimensions: (u32, u32) +pub j2k_native::J2kReferencedClassicPlan::Rgba::output_rect: j2k_native::J2kRect +pub j2k_native::J2kReferencedClassicPlan::Rgba::payloads: alloc::vec::Vec +pub j2k_native::J2kReferencedClassicPlan::Rgba::ranges: alloc::vec::Vec +pub j2k_native::J2kReferencedClassicPlan::Rgba::tiles: alloc::vec::Vec +pub j2k_native::J2kReferencedHtj2kPlan::Color +pub j2k_native::J2kReferencedHtj2kPlan::Color::full_dimensions: (u32, u32) +pub j2k_native::J2kReferencedHtj2kPlan::Color::output_rect: j2k_native::J2kRect +pub j2k_native::J2kReferencedHtj2kPlan::Color::payloads: alloc::vec::Vec +pub j2k_native::J2kReferencedHtj2kPlan::Color::tiles: alloc::vec::Vec +pub j2k_native::J2kReferencedHtj2kPlan::Grayscale +pub j2k_native::J2kReferencedHtj2kPlan::Grayscale::full_dimensions: (u32, u32) +pub j2k_native::J2kReferencedHtj2kPlan::Grayscale::output_rect: j2k_native::J2kRect +pub j2k_native::J2kReferencedHtj2kPlan::Grayscale::payloads: alloc::vec::Vec +pub j2k_native::J2kReferencedHtj2kPlan::Grayscale::tiles: alloc::vec::Vec +pub j2k_native::J2kReferencedHtj2kPlan::Rgba +pub j2k_native::J2kReferencedHtj2kPlan::Rgba::full_dimensions: (u32, u32) +pub j2k_native::J2kReferencedHtj2kPlan::Rgba::output_rect: j2k_native::J2kRect +pub j2k_native::J2kReferencedHtj2kPlan::Rgba::payloads: alloc::vec::Vec +pub j2k_native::J2kReferencedHtj2kPlan::Rgba::tiles: alloc::vec::Vec +pub j2k_native::J2kReferencedPayloadRecordSpan::first_record: usize +pub j2k_native::J2kReferencedPayloadRecordSpan::record_count: usize +pub j2k_native::J2kReferencedTileGeometry::Color(j2k_native::J2kDirectColorPlan) +pub j2k_native::J2kReferencedTileGeometry::Grayscale(j2k_native::J2kDirectGrayscalePlan) +pub j2k_native::J2kReferencedTileGeometry::Rgba(j2k_native::J2kDirectRgbaPlan) pub j2k_native::J2kRequiredBandRegion::x0: u32 pub j2k_native::J2kRequiredBandRegion::x1: u32 pub j2k_native::J2kRequiredBandRegion::y0: u32 @@ -2549,10 +2738,15 @@ pub struct j2k_native::HtSubBandDecodeJob<'a> pub struct j2k_native::J2kCodeBlockBatchJob<'a> pub struct j2k_native::J2kCodeBlockDecodeJob<'a> pub struct j2k_native::J2kCodeBlockDecodeWorkspace +pub struct j2k_native::J2kDirectCodeBlockIndex pub struct j2k_native::J2kDirectColorPlan +pub struct j2k_native::J2kDirectCpuEntropyWorkspace pub struct j2k_native::J2kDirectCpuScratch +pub struct j2k_native::J2kDirectDecodedComponents<'a> +pub struct j2k_native::J2kDirectDecodedPlane<'a> pub struct j2k_native::J2kDirectGrayscalePlan pub struct j2k_native::J2kDirectIdwtStep +pub struct j2k_native::J2kDirectRgbaPlan pub struct j2k_native::J2kDirectStoreStep pub struct j2k_native::J2kIdwtBand<'a> pub struct j2k_native::J2kIdwtRequiredInputWindows @@ -2560,6 +2754,8 @@ pub struct j2k_native::J2kInverseMctJob<'a> pub struct j2k_native::J2kOwnedCodeBlockBatchJob pub struct j2k_native::J2kOwnedSubBandPlan pub struct j2k_native::J2kRect +pub struct j2k_native::J2kReferencedPayloadRecordSpan +pub struct j2k_native::J2kReferencedTilePlan pub struct j2k_native::J2kRequiredBandRegion pub struct j2k_native::J2kSingleDecompositionIdwtJob<'a> pub struct j2k_native::J2kStoreComponentJob<'a> @@ -2580,10 +2776,13 @@ pub type j2k_native::NativeComponentPlaneParts = (alloc::vec::Vec, (u32, u32 pub use j2k_native::CpuOnlyJ2kEncodeStageAccelerator pub use j2k_native::EncodedHtJ2kCodeBlock pub use j2k_native::EncodedJ2kCodeBlock +pub use j2k_native::HtCodeBlockPayloadRanges pub use j2k_native::IrreversibleQuantizationStep pub use j2k_native::IrreversibleQuantizationSubbandScales +pub use j2k_native::J2kClassicCodeBlockPayload pub use j2k_native::J2kCodeBlockSegment pub use j2k_native::J2kCodeBlockStyle +pub use j2k_native::J2kCodestreamRange pub use j2k_native::J2kDeinterleaveToF32Job pub use j2k_native::J2kEncodeDispatchReport pub use j2k_native::J2kEncodeStageAccelerator @@ -2715,6 +2914,8 @@ pub struct j2k_types::J2kResidentHtj2kTileEncodeJob<'a> #[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob #[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreRgb8Job #[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreRgb8MctJob +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob #[repr(C)] pub struct j2k_cuda_runtime::CudaJpegBaselineEncodeHuffmanTable #[repr(C)] pub struct j2k_cuda_runtime::CudaJpegBaselineEncodeParams #[repr(C)] pub struct j2k_cuda_runtime::CudaJpegEntropyCheckpoint @@ -2728,8 +2929,11 @@ impl core::default::Default for j2k_cuda_runtime::CudaPinnedUploadStagingPoolLim impl core::ops::drop::Drop for j2k_cuda_runtime::CudaExternalHostOwner impl core::ops::drop::Drop for j2k_cuda_runtime::CudaExternalHostReservation<'_> impl core::ops::drop::Drop for j2k_cuda_runtime::CudaPinnedUploadStagingCheckout<'_, '_> +impl core::ops::drop::Drop for j2k_cuda_runtime::CudaQueuedClassicDecode impl core::ops::drop::Drop for j2k_cuda_runtime::CudaQueuedExecution impl core::ops::drop::Drop for j2k_cuda_runtime::CudaQueuedHtj2kCleanup +impl core::ops::drop::Drop for j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup +impl core::ops::drop::Drop for j2k_cuda_runtime::CudaQueuedJ2kStoreBatch impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaClassicStatus impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaHtj2kEncodeStatus impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaHtj2kPacketizationBlock @@ -2747,6 +2951,8 @@ impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreRgb16Job impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreRgb8Job impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreRgb8MctJob +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJpegBaselineEncodeHuffmanTable impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJpegBaselineEncodeParams impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJpegEntropyCheckpoint @@ -2781,8 +2987,11 @@ impl j2k_cuda_runtime::CudaPinnedUploadOperationGuard<'_> impl j2k_cuda_runtime::CudaPinnedUploadStagingCheckout<'_, '_> impl j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput impl j2k_cuda_runtime::CudaPooledKernelOutput +impl j2k_cuda_runtime::CudaQueuedClassicDecode impl j2k_cuda_runtime::CudaQueuedExecution impl j2k_cuda_runtime::CudaQueuedHtj2kCleanup +impl j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup +impl j2k_cuda_runtime::CudaQueuedJ2kStoreBatch impl j2k_cuda_runtime::CudaResidentDwt53Output impl j2k_cuda_runtime::CudaResidentDwt97Output impl<'a> j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'a> @@ -2790,6 +2999,11 @@ impl j2k_cuda_runtime::CudaDeviceBufferView<'_, T> impl j2k_cuda_runtime::CudaDeviceBufferViewMut<'_, T> pub const fn j2k_cuda_runtime::CudaExternalHostOwner::bytes(&self) -> usize pub const fn j2k_cuda_runtime::CudaExternalHostReservation<'_>::external_live_bytes(&self) -> usize +pub const fn j2k_cuda_runtime::CudaQueuedClassicDecode::status_count(&self) -> usize +pub const fn j2k_cuda_runtime::CudaQueuedHtj2kCleanup::status_count(&self) -> usize +pub const fn j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup::status_count(&self) -> usize +pub const fn j2k_cuda_runtime::CudaQueuedJ2kStoreBatch::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub const fn j2k_cuda_runtime::htj2k_cleanup_multi_descriptor_bytes() -> usize pub const j2k_cuda_runtime::CudaClassicStatus::NAME: &'static str pub const j2k_cuda_runtime::CudaHtj2kEncodeStatus::NAME: &'static str pub const j2k_cuda_runtime::CudaHtj2kPacketizationBlock::NAME: &'static str @@ -2807,6 +3021,8 @@ pub const j2k_cuda_runtime::CudaJ2kStoreRgb16Job::NAME: &'static str pub const j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob::NAME: &'static str pub const j2k_cuda_runtime::CudaJ2kStoreRgb8Job::NAME: &'static str pub const j2k_cuda_runtime::CudaJ2kStoreRgb8MctJob::NAME: &'static str +pub const j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::NAME: &'static str +pub const j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::NAME: &'static str pub const j2k_cuda_runtime::CudaJpegBaselineEncodeHuffmanTable::NAME: &'static str pub const j2k_cuda_runtime::CudaJpegBaselineEncodeParams::NAME: &'static str pub const j2k_cuda_runtime::CudaJpegEntropyCheckpoint::NAME: &'static str @@ -2844,6 +3060,7 @@ pub fn j2k_cuda_runtime::CudaContext::device_ordinal(&self) -> usize pub fn j2k_cuda_runtime::CudaContext::diagnose_jpeg_420_entropy_self_sync(&self, &j2k_cuda_runtime::CudaJpegChunkedEntropyPlan<'_>) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::diagnose_jpeg_420_entropy_self_sync_with_external_live(&self, &j2k_cuda_runtime::CudaJpegChunkedEntropyPlan<'_>, usize) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::diagnose_jpeg_420_entropy_self_sync_with_pinned_upload_operation(&self, &j2k_cuda_runtime::CudaJpegChunkedEntropyPlan<'_>, usize, &j2k_cuda_runtime::CudaPinnedUploadOperationGuard<'_>) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::diagnostics(&self) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblock_regions_resident(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob], j2k_cuda_runtime::CudaHtj2kEncodeTables<'_>) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblock_regions_resident_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob], &j2k_cuda_runtime::CudaHtj2kEncodeResources, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblock_regions_resident_with_resources_and_pool_and_live_host_bytes(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob], &j2k_cuda_runtime::CudaHtj2kEncodeResources, &j2k_cuda_runtime::CudaBufferPool, usize) -> core::result::Result @@ -2885,15 +3102,33 @@ pub fn j2k_cuda_runtime::CudaContext::j2k_ml_convert_into_external(&self, u64, u pub fn j2k_cuda_runtime::CudaContext::j2k_quantize_subband(&self, &[f32], j2k_cuda_runtime::CudaJ2kQuantizeJob) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::j2k_quantize_subband_region_resident(&self, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::j2k_quantize_subband_resident(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, j2k_cuda_runtime::CudaJ2kQuantizeJob) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_gray16_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreGray16Target<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_gray16_batch_contiguous_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreGray16Target<'_>]) -> core::result::Result<(j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> pub fn j2k_cuda_runtime::CudaContext::j2k_store_gray16_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreGray16Job) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_gray8_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreGray8Target<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_gray8_batch_contiguous_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreGray8Target<'_>]) -> core::result::Result<(j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> pub fn j2k_cuda_runtime::CudaContext::j2k_store_gray8_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreGray8Job) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_grayi16_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreGrayI16Target<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_grayi16_batch_contiguous_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreGrayI16Target<'_>]) -> core::result::Result<(j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb16_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreRgb16Job) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb16_mct_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb16_native_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb16_native_batch_contiguous_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'_>]) -> core::result::Result<(j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreRgb8Job) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_mct_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget<'_>]) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_mct_batch_contiguous_device_with_live_host_bytes(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget<'_>], usize) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_mct_batch_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget<'_>]) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_mct_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreRgb8MctJob) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_native_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_native_batch_contiguous_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'_>]) -> core::result::Result<(j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgba16_native_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgba16_native_batch_contiguous_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'_>]) -> core::result::Result<(j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgba8_native_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgba8_native_batch_contiguous_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'_>]) -> core::result::Result<(j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgbai16_native_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgbai16_native_batch_contiguous_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'_>]) -> core::result::Result<(j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgbi16_native_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgbi16_native_batch_contiguous_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'_>]) -> core::result::Result<(j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_dwt97(&self, &[f32], usize, usize, usize, usize) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_dwt97_and_live_host_bytes(&self, &[f32], usize, usize, usize, usize, usize) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_dwt97_batch_with_pool(&self, j2k_cuda_runtime::CudaDwt97BatchWithPoolRequest<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaDwt97BatchStageTimings), j2k_cuda_runtime::CudaError> @@ -2910,6 +3145,7 @@ pub fn j2k_cuda_runtime::CudaContext::pinned_upload_staging_pool_diagnostics(&se pub fn j2k_cuda_runtime::CudaContext::register_external_host_owner(&self, usize) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::time_default_stream_named_us(&self, &str, impl core::ops::function::FnMut() -> core::result::Result) -> core::result::Result<(T, u128), j2k_cuda_runtime::CudaError> pub fn j2k_cuda_runtime::CudaContext::time_default_stream_named_us_if(&self, bool, &str, impl core::ops::function::FnMut() -> core::result::Result) -> core::result::Result<(T, u128), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::upload_classic_decode_table_resources(&self) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::upload_htj2k_decode_resources_with_tables(&self, &[u8], &j2k_cuda_runtime::CudaHtj2kDecodeTableResources) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::upload_htj2k_decode_resources_with_tables_and_pool(&self, &[u8], &j2k_cuda_runtime::CudaHtj2kDecodeTableResources, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result pub fn j2k_cuda_runtime::CudaContext::upload_htj2k_decode_table_resources(&self, j2k_cuda_runtime::CudaHtj2kDecodeTables<'_>) -> core::result::Result @@ -2934,6 +3170,9 @@ pub fn j2k_cuda_runtime::CudaDwt97Output::execution(&self) -> j2k_cuda_runtime:: pub fn j2k_cuda_runtime::CudaDwt97Output::levels(&self) -> &[j2k_cuda_runtime::CudaDwt53LevelShape] pub fn j2k_cuda_runtime::CudaDwt97Output::ll_dimensions(&self) -> (u32, u32) pub fn j2k_cuda_runtime::CudaDwt97Output::transformed(&self) -> &[f32] +pub fn j2k_cuda_runtime::CudaError::completion_is_uncertain(&self) -> bool +pub fn j2k_cuda_runtime::CudaError::kernel_job_index(&self) -> core::option::Option +pub fn j2k_cuda_runtime::CudaError::session_is_unusable(&self) -> bool pub fn j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'a>::byte_len(&self) -> usize pub fn j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'a>::context(&self) -> &j2k_cuda_runtime::CudaContext pub fn j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'a>::device_ptr(&self) -> u64 @@ -3044,6 +3283,9 @@ pub fn j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput::statuses(&self) -> &[j2k_c pub fn j2k_cuda_runtime::CudaPooledKernelOutput::buffer(&self) -> core::option::Option<&j2k_cuda_runtime::CudaDeviceBuffer> pub fn j2k_cuda_runtime::CudaPooledKernelOutput::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats pub fn j2k_cuda_runtime::CudaPooledKernelOutput::into_parts(self) -> (j2k_cuda_runtime::CudaPooledDeviceBuffer, j2k_cuda_runtime::CudaExecutionStats) +pub fn j2k_cuda_runtime::CudaQueuedClassicDecode::drop(&mut self) +pub fn j2k_cuda_runtime::CudaQueuedClassicDecode::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaQueuedClassicDecode::finish(self) -> core::result::Result<(j2k_cuda_runtime::CudaExecutionStats, j2k_cuda_runtime::CudaClassicDecodeStageTimings), j2k_cuda_runtime::CudaError> pub fn j2k_cuda_runtime::CudaQueuedExecution::drop(&mut self) pub fn j2k_cuda_runtime::CudaQueuedExecution::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats pub fn j2k_cuda_runtime::CudaQueuedExecution::finish(self) -> core::result::Result @@ -3052,6 +3294,13 @@ pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanup::drop(&mut self) pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanup::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanup::finish(self) -> core::result::Result pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanup::resource_count(&self) -> usize +pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup::drop(&mut self) +pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup::finish(self) -> core::result::Result +pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup::new(&j2k_cuda_runtime::CudaContext, &j2k_cuda_runtime::CudaBufferPool, usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup::retain(&mut self, j2k_cuda_runtime::CudaQueuedHtj2kCleanup) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaQueuedJ2kStoreBatch::drop(&mut self) +pub fn j2k_cuda_runtime::CudaQueuedJ2kStoreBatch::finish(self) -> core::result::Result +pub fn j2k_cuda_runtime::CudaQueuedJ2kStoreBatch::is_complete(&self) -> core::result::Result pub fn j2k_cuda_runtime::CudaResidentDwt53Output::buffer(&self) -> &j2k_cuda_runtime::CudaDeviceBuffer pub fn j2k_cuda_runtime::CudaResidentDwt53Output::download_transformed(&self) -> core::result::Result, j2k_cuda_runtime::CudaError> pub fn j2k_cuda_runtime::CudaResidentDwt53Output::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats @@ -3117,6 +3366,24 @@ pub j2k_cuda_runtime::CudaClassicSegment::start_coding_pass: u32 pub j2k_cuda_runtime::CudaClassicSegment::use_arithmetic: bool pub j2k_cuda_runtime::CudaClassicStatus::code: u32 pub j2k_cuda_runtime::CudaClassicStatus::detail: u32 +pub j2k_cuda_runtime::CudaContextDiagnostics::cached_events: usize +pub j2k_cuda_runtime::CudaContextDiagnostics::context_host_synchronizations: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::device_allocation_bytes: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::device_allocation_operations: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::device_to_host_bytes: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::device_to_host_operations: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::event_driver_allocations: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::event_host_synchronizations: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::event_reuses: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::host_to_device_bytes: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::host_to_device_operations: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::kernel_launches: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::live_device_allocations: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::live_device_bytes: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::peak_live_device_allocations: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::peak_live_device_bytes: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::status_device_to_host_bytes: u64 +pub j2k_cuda_runtime::CudaContextDiagnostics::status_device_to_host_operations: u64 pub j2k_cuda_runtime::CudaDeviceBufferRange::len: usize pub j2k_cuda_runtime::CudaDeviceBufferRange::offset: usize pub j2k_cuda_runtime::CudaDwt53LevelShape::height: u32 @@ -3322,6 +3589,9 @@ pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::output_x: u32 pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::output_y: u32 pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::source_x: u32 pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::source_y: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Target::input: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreGray16Target::job: j2k_cuda_runtime::CudaJ2kStoreGray16Job +pub j2k_cuda_runtime::CudaJ2kStoreGray16Target::output_index: usize pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::addend: f32 pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::bit_depth: u32 pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::copy_height: u32 @@ -3333,6 +3603,12 @@ pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::output_x: u32 pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::output_y: u32 pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::source_x: u32 pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::source_y: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Target::input: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreGray8Target::job: j2k_cuda_runtime::CudaJ2kStoreGray8Job +pub j2k_cuda_runtime::CudaJ2kStoreGray8Target::output_index: usize +pub j2k_cuda_runtime::CudaJ2kStoreGrayI16Target::input: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreGrayI16Target::job: j2k_cuda_runtime::CudaJ2kStoreGray16Job +pub j2k_cuda_runtime::CudaJ2kStoreGrayI16Target::output_index: usize pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::addend0: f32 pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::addend1: f32 pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::addend2: f32 @@ -3385,6 +3661,70 @@ pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget::job: j2k_cuda_runtime::CudaJ2kS pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget::plane0: &'a j2k_cuda_runtime::CudaDeviceBuffer pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget::plane1: &'a j2k_cuda_runtime::CudaDeviceBuffer pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget::plane2: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::addend0: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::addend1: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::addend2: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::bit_depth0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::bit_depth1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::bit_depth2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::copy_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::copy_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::input_width0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::input_width1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::input_width2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::layout: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::output_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::output_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::output_x: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::output_y: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::reserved: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::source_x0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::source_x1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::source_x2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::source_y0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::source_y1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::source_y2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob::transform: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget::job: j2k_cuda_runtime::CudaJ2kStoreRgbNativeJob +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget::output_index: usize +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget::plane0: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget::plane1: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget::plane2: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::addend0: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::addend1: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::addend2: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::addend3: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::bit_depth0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::bit_depth1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::bit_depth2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::bit_depth3: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::copy_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::copy_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::input_width0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::input_width1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::input_width2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::input_width3: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::layout: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::output_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::output_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::output_x: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::output_y: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::reserved: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::source_x0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::source_x1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::source_x2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::source_x3: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::source_y0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::source_y1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::source_y2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::source_y3: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob::transform: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget::job: j2k_cuda_runtime::CudaJ2kStoreRgbaNativeJob +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget::output_index: usize +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget::plane0: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget::plane1: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget::plane2: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget::plane3: &'a j2k_cuda_runtime::CudaDeviceBuffer pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::bit_depth: u8 pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::buffer: &'a j2k_cuda_runtime::CudaDeviceBuffer pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::byte_offset: usize @@ -3537,8 +3877,10 @@ pub struct j2k_cuda_runtime::CudaBufferPoolLimits pub struct j2k_cuda_runtime::CudaBufferPoolTakeTrace pub struct j2k_cuda_runtime::CudaClassicCodeBlockJob pub struct j2k_cuda_runtime::CudaClassicDecodeStageTimings +pub struct j2k_cuda_runtime::CudaClassicDecodeTableResources pub struct j2k_cuda_runtime::CudaClassicDecodeTarget<'a> pub struct j2k_cuda_runtime::CudaClassicSegment +pub struct j2k_cuda_runtime::CudaContextDiagnostics pub struct j2k_cuda_runtime::CudaDeviceBufferRange pub struct j2k_cuda_runtime::CudaDeviceBufferView<'a, T> pub struct j2k_cuda_runtime::CudaDeviceBufferViewMut<'a, T> @@ -3585,7 +3927,12 @@ pub struct j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob pub struct j2k_cuda_runtime::CudaJ2kQuantizedSubband pub struct j2k_cuda_runtime::CudaJ2kResidentComponents pub struct j2k_cuda_runtime::CudaJ2kResidentQuantizedSubband +pub struct j2k_cuda_runtime::CudaJ2kStoreGray16Target<'a> +pub struct j2k_cuda_runtime::CudaJ2kStoreGray8Target<'a> +pub struct j2k_cuda_runtime::CudaJ2kStoreGrayI16Target<'a> pub struct j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget<'a> +pub struct j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'a> +pub struct j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'a> pub struct j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels<'a> pub struct j2k_cuda_runtime::CudaJpegBaselineEntropyEncodeBatchJob<'a> pub struct j2k_cuda_runtime::CudaJpegBaselineEntropyEncodeJob<'a> @@ -3602,20 +3949,43 @@ pub struct j2k_cuda_runtime::CudaPinnedUploadStagingPoolDiagnostics pub struct j2k_cuda_runtime::CudaPinnedUploadStagingPoolLimits pub struct j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput pub struct j2k_cuda_runtime::CudaPooledKernelOutput +pub struct j2k_cuda_runtime::CudaQueuedClassicDecode pub struct j2k_cuda_runtime::CudaQueuedExecution pub struct j2k_cuda_runtime::CudaQueuedHtj2kCleanup +pub struct j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup +pub struct j2k_cuda_runtime::CudaQueuedJ2kStoreBatch pub struct j2k_cuda_runtime::CudaResidentDwt53Output pub struct j2k_cuda_runtime::CudaResidentDwt97Output pub struct j2k_cuda_runtime::CudaTranscodeDwt97Bands pub struct j2k_cuda_runtime::CudaTranscodeReversible53Bands +pub unsafe fn j2k_cuda_runtime::CudaContext::decode_classic_codeblocks_multi_enqueue_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &j2k_cuda_runtime::CudaClassicDecodeTableResources, &[j2k_cuda_runtime::CudaClassicDecodeTarget<'_>], &j2k_cuda_runtime::CudaBufferPool, usize) -> core::result::Result +pub unsafe fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_dequantize_multi_enqueue_into_status_group(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCleanupTarget<'_>], &j2k_cuda_runtime::CudaBufferPool, usize, &j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup, usize) -> core::result::Result +pub unsafe fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_dequantize_multi_enqueue_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCleanupTarget<'_>], &j2k_cuda_runtime::CudaBufferPool, usize) -> core::result::Result +pub unsafe fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_multi_enqueue_into_status_group(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCleanupTarget<'_>], &j2k_cuda_runtime::CudaBufferPool, usize, &j2k_cuda_runtime::CudaQueuedHtj2kCleanupGroup, usize) -> core::result::Result pub unsafe fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCleanupTarget<'_>], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result pub unsafe fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool_and_live_host_bytes(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCleanupTarget<'_>], &j2k_cuda_runtime::CudaBufferPool, usize) -> core::result::Result +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_dequantize_queued_htj2k_cleanup_enqueue(&self, &j2k_cuda_runtime::CudaQueuedHtj2kCleanup) -> core::result::Result pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_batch_device_enqueue_with_pool(&self, &[j2k_cuda_runtime::CudaJ2kIdwtTarget<'_>], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_batch_sequence_enqueue_with_pool(&self, &[&[j2k_cuda_runtime::CudaJ2kIdwtTarget<'_>]], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_batch_sequence_enqueue_with_pool_and_live_host_bytes(&self, &[&[j2k_cuda_runtime::CudaJ2kIdwtTarget<'_>]], &j2k_cuda_runtime::CudaBufferPool, usize) -> core::result::Result +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_gray16_batch_into_external_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreGray16Target<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaExecutionStats), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_gray16_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreGray16Target<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_gray8_batch_into_external_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreGray8Target<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaExecutionStats), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_gray8_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreGray8Target<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_grayi16_batch_into_external_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreGrayI16Target<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaExecutionStats), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_grayi16_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreGrayI16Target<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_rgb16_native_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_mct_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_native_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_rgba16_native_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_rgba8_native_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_rgbai16_native_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbaNativeTarget<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaContext::j2k_store_rgbi16_native_batch_into_external_device_enqueue(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgbNativeTarget<'_>], &mut j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaQueuedJ2kStoreBatch), j2k_cuda_runtime::CudaError> pub unsafe fn j2k_cuda_runtime::CudaContext::submit_default_stream_named(&self, &str, impl core::ops::function::FnMut() -> core::result::Result) -> core::result::Result +pub unsafe fn j2k_cuda_runtime::CudaContext::with_primary_stream_ordering(&self, u64, &mut Owner, impl core::ops::function::FnOnce() -> core::result::Result) -> core::result::Result, j2k_cuda_runtime::CudaError> pub unsafe fn j2k_cuda_runtime::CudaExternalDeviceBufferViewMut<'a>::from_raw_parts(&j2k_cuda_runtime::CudaContext, u64, usize, usize, &'a mut Owner) -> core::result::Result pub unsafe fn j2k_cuda_runtime::CudaQueuedExecution::release_pool_reuse_after_completion(&mut self) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +pub unsafe fn j2k_cuda_runtime::CudaQueuedJ2kStoreBatch::release_after_stream_completion(self) -> j2k_cuda_runtime::CudaExecutionStats ``` ## `j2k-profile` diff --git a/docs/stable-api-1.0.public-api.txt b/docs/stable-api-1.0.public-api.txt index bcfa7980..ffc6f91f 100644 --- a/docs/stable-api-1.0.public-api.txt +++ b/docs/stable-api-1.0.public-api.txt @@ -14,23 +14,47 @@ It is the item-level companion to `docs/stable-api-1.0.md`: every public module, ```text #[non_exhaustive] pub enum j2k::BackendErrorKind +#[non_exhaustive] pub enum j2k::BatchAlpha +#[non_exhaustive] pub enum j2k::BatchCodecRoute +#[non_exhaustive] pub enum j2k::BatchErrorStage +#[non_exhaustive] pub enum j2k::BatchItemError +#[non_exhaustive] pub enum j2k::BatchLayout +#[non_exhaustive] pub enum j2k::BatchWaveletTransform +#[non_exhaustive] pub enum j2k::CpuBatchSamples +#[non_exhaustive] pub enum j2k::DecodeRequest #[non_exhaustive] pub enum j2k::J2kDecodeWarning #[non_exhaustive] pub enum j2k::J2kDecodedColorSpace #[non_exhaustive] pub enum j2k::J2kError #[non_exhaustive] pub enum j2k::J2kFileColorSpec<'a> +#[non_exhaustive] pub enum j2k::NonRepresentableReason +#[non_exhaustive] pub enum j2k::PreparationDepth #[non_exhaustive] pub struct j2k::J2kLosslessEncodeOptions #[non_exhaustive] pub struct j2k::J2kLossyEncodeOptions #[non_exhaustive] pub struct j2k::J2kToHtj2kOptions +impl core::default::Default for j2k::BatchDecodeOptions +impl core::default::Default for j2k::DecodeSettings impl core::default::Default for j2k::J2kLosslessEncodeOptions impl core::default::Default for j2k::J2kLossyEncodeOptions impl core::default::Default for j2k::J2kRowDecodeOptions impl core::default::Default for j2k::J2kToHtj2kOptions impl core::error::Error for j2k::NativeBackendError +impl core::fmt::Debug for j2k::CpuBatchDecoder impl core::fmt::Display for j2k::BackendErrorKind +impl core::fmt::Display for j2k::BatchErrorStage impl core::fmt::Display for j2k::J2kDecodeWarning impl core::fmt::Display for j2k::NativeBackendError +impl core::fmt::Display for j2k::NonRepresentableReason impl j2k::BackendError +impl j2k::BatchDecoder for j2k::CpuBatchDecoder +impl j2k::BatchGroupInfo +impl j2k::CpuBatchDecodeResult +impl j2k::CpuBatchDecoder +impl j2k::CpuBatchGroup +impl j2k::CpuBatchSamples +impl j2k::CpuBatchWorkspaceStats +impl j2k::DecodeSettings impl j2k::DeviceDecodePlan +impl j2k::EncodedImage impl j2k::J2kContext impl j2k::J2kDecodedNativeComponents impl j2k::J2kDecoder<'_> @@ -45,6 +69,11 @@ impl j2k::J2kRowDecodeOptions impl j2k::J2kScratchPool impl j2k::J2kSupportInfo impl j2k::J2kToHtj2kOptions +impl j2k::PreparedBatch +impl j2k::PreparedBatchGroup +impl j2k::PreparedClassicPlan +impl j2k::PreparedHtj2kPlan +impl j2k::PreparedImage impl<'a> j2k::J2kCodestreamPayload<'a> impl<'a> j2k::J2kComponentPlane<'a> impl<'a> j2k::J2kDecodedComponents<'a> @@ -58,6 +87,39 @@ impl<'a> j2k::J2kLosslessTypedComponentSamples<'a> impl<'a> j2k::J2kLossySamples<'a> impl<'a> j2k::J2kView<'a> pub const fn j2k::BackendError::kind(&self) -> j2k::BackendErrorKind +pub const fn j2k::CpuBatchDecoder::options(&self) -> j2k::BatchDecodeOptions +pub const fn j2k::CpuBatchGroup::samples(&self) -> &j2k::CpuBatchSamples +pub const fn j2k::CpuBatchSamples::sample_type(&self) -> j2k_core::sample::SampleType +pub const fn j2k::CpuBatchWorkspaceStats::component_owner_reuses(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::compressed_arena_reuses(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::cross_image_entropy_windows(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::decode_calls(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::entropy_job_dispatches(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_classic_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_cleanup_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_group_plans(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_magref_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_payload_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_sigprop_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::idwt_owner_reuses(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::output_compaction_copied_samples(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::output_group_allocations(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::preparation_calls(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::preparation_worker_reuses(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::prepared_plan_decode_calls(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::retained_component_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_compressed_arena_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_idwt_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_prepared_plan_classic_workspace_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_prepared_plan_ht_workspace_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_scratch_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_tier1_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::scratch_capacity_retries(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::tier1_owner_reuses(self) -> u64 +pub const fn j2k::DecodeSettings::is_strict(self) -> bool +pub const fn j2k::DecodeSettings::lenient() -> Self +pub const fn j2k::DecodeSettings::lenient_tolerance_enabled(self) -> bool +pub const fn j2k::DecodeSettings::strict() -> Self pub const fn j2k::J2kCodestreamPayload<'a>::codestream(self) -> &'a [u8] pub const fn j2k::J2kCodestreamPayload<'a>::codestream_offset(self) -> usize pub const fn j2k::J2kCodestreamPayload<'a>::payload_kind(self) -> j2k_core::passthrough::CompressedPayloadKind @@ -89,6 +151,8 @@ pub const fn j2k::J2kRowDecodeOptions::new_with_max_stripe_bytes(u32, usize) -> pub const fn j2k::J2kRowDecodeOptions::with_max_stripe_bytes(self, usize) -> Self pub const fn j2k::J2kScratchPool::new() -> Self pub const fn j2k::J2kToHtj2kOptions::new(j2k_core::passthrough::CompressedPayloadKind, j2k::J2kProgressionOrder, j2k::J2kEncodeValidation) -> Self +pub const fn j2k::PreparedBatch::options(&self) -> j2k::BatchDecodeOptions +pub const fn j2k::PreparedBatchGroup::options(&self) -> j2k::BatchDecodeOptions pub enum j2k::CpuDecodeParallelism pub enum j2k::DeviceDecodeRequest pub enum j2k::EncodeBackendPreference @@ -110,6 +174,41 @@ pub fn j2k::BackendError::not_implemented(impl core::convert::Into) -> Self pub fn j2k::BackendError::unsupported(impl core::convert::Into) -> Self pub fn j2k::BackendErrorKind::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k::BatchDecodeOptions::default() -> Self +pub fn j2k::BatchDecoder::decode_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::BatchDecoder::decode_prepared(&mut self, &j2k::PreparedBatch) -> core::result::Result +pub fn j2k::BatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::BatchDecoder::options(&self) -> j2k::BatchDecodeOptions +pub fn j2k::BatchDecoder::prepare_batch(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::BatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::BatchErrorStage::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k::BatchGroupInfo::samples_per_image(&self) -> core::option::Option +pub fn j2k::CpuBatchDecodeResult::errors(&self) -> &[j2k::IndexedBatchError] +pub fn j2k::CpuBatchDecodeResult::groups(&self) -> &[j2k::CpuBatchGroup] +pub fn j2k::CpuBatchDecodeResult::into_parts(self) -> (alloc::vec::Vec, alloc::vec::Vec) +pub fn j2k::CpuBatchDecoder::decode(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_prepared(&mut self, &j2k::PreparedBatch) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_prepared(&mut self, &j2k::PreparedBatch) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k::CpuBatchDecoder::new(j2k::BatchDecodeOptions) -> Self +pub fn j2k::CpuBatchDecoder::options(&self) -> j2k::BatchDecodeOptions +pub fn j2k::CpuBatchDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::prepare_batch(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::retained_worker_count(&self) -> usize +pub fn j2k::CpuBatchDecoder::workspace_stats(&self) -> j2k::CpuBatchWorkspaceStats +pub fn j2k::CpuBatchGroup::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k::CpuBatchGroup::info(&self) -> &j2k::BatchGroupInfo +pub fn j2k::CpuBatchGroup::into_parts(self) -> (j2k::BatchGroupInfo, alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec>, j2k::CpuBatchSamples) +pub fn j2k::CpuBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k::CpuBatchGroup::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k::CpuBatchSamples::is_empty(&self) -> bool +pub fn j2k::CpuBatchSamples::len(&self) -> usize +pub fn j2k::DecodeSettings::default() -> Self pub fn j2k::DeviceDecodePlan::for_image((u32, u32), j2k::DeviceDecodeRequest) -> core::result::Result pub fn j2k::DeviceDecodePlan::is_full_frame(self) -> bool pub fn j2k::DeviceDecodePlan::output_dims(self) -> (u32, u32) @@ -118,6 +217,8 @@ pub fn j2k::DeviceDecodePlan::scale(self) -> j2k_core::scale::Downscale pub fn j2k::DeviceDecodePlan::source_dims(self) -> (u32, u32) pub fn j2k::DeviceDecodePlan::source_rect(self) -> j2k_core::types::Rect pub fn j2k::DeviceDecodePlan::target_resolution(self) -> core::option::Option<(u32, u32)> +pub fn j2k::EncodedImage::full(alloc::sync::Arc<[u8]>) -> Self +pub fn j2k::EncodedImage::new(alloc::sync::Arc<[u8]>, j2k::DecodeRequest) -> Self pub fn j2k::J2kComponentPlane<'a>::bit_depth(&self) -> u8 pub fn j2k::J2kComponentPlane<'a>::dimensions(&self) -> (u32, u32) pub fn j2k::J2kComponentPlane<'a>::samples(&self) -> &'a [f32] @@ -199,6 +300,40 @@ pub fn j2k::J2kView<'a>::passthrough_candidate(&self) -> core::option::Option::support_info(&self) -> core::option::Option<&j2k::J2kSupportInfo> pub fn j2k::NativeBackendError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k::NativeBackendError::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)> +pub fn j2k::NonRepresentableReason::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k::PreparedBatch::errors(&self) -> &[j2k::IndexedBatchError] +pub fn j2k::PreparedBatch::groups(&self) -> &[j2k::PreparedBatchGroup] +pub fn j2k::PreparedBatch::into_parts(self) -> (alloc::sync::Arc<[j2k::PreparedBatchGroup]>, alloc::sync::Arc<[j2k::IndexedBatchError]>, j2k::BatchDecodeOptions) +pub fn j2k::PreparedBatchGroup::images(&self) -> &[j2k::PreparedImage] +pub fn j2k::PreparedBatchGroup::info(&self) -> &j2k::BatchGroupInfo +pub fn j2k::PreparedBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k::PreparedClassicPlan::is_color(&self) -> bool +pub fn j2k::PreparedClassicPlan::is_empty(&self) -> bool +pub fn j2k::PreparedClassicPlan::is_grayscale(&self) -> bool +pub fn j2k::PreparedClassicPlan::is_rgba(&self) -> bool +pub fn j2k::PreparedClassicPlan::payload(&self, usize) -> core::option::Option +pub fn j2k::PreparedClassicPlan::payload_count(&self) -> usize +pub fn j2k::PreparedClassicPlan::payloads(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator + '_ +pub fn j2k::PreparedClassicPlan::range(&self, usize) -> core::option::Option +pub fn j2k::PreparedClassicPlan::range_count(&self) -> usize +pub fn j2k::PreparedClassicPlan::ranges(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator + '_ +pub fn j2k::PreparedHtj2kPlan::is_color(&self) -> bool +pub fn j2k::PreparedHtj2kPlan::is_empty(&self) -> bool +pub fn j2k::PreparedHtj2kPlan::is_grayscale(&self) -> bool +pub fn j2k::PreparedHtj2kPlan::is_rgba(&self) -> bool +pub fn j2k::PreparedHtj2kPlan::payload(&self, usize) -> core::option::Option +pub fn j2k::PreparedHtj2kPlan::payload_count(&self) -> usize +pub fn j2k::PreparedHtj2kPlan::payloads(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator + '_ +pub fn j2k::PreparedImage::bytes(&self) -> &alloc::sync::Arc<[u8]> +pub fn j2k::PreparedImage::classic_plan(&self) -> core::option::Option<&j2k::PreparedClassicPlan> +pub fn j2k::PreparedImage::codestream_range(&self) -> j2k_types::decode_payload::J2kCodestreamRange +pub fn j2k::PreparedImage::decode_settings(&self) -> j2k::DecodeSettings +pub fn j2k::PreparedImage::htj2k_plan(&self) -> core::option::Option<&j2k::PreparedHtj2kPlan> +pub fn j2k::PreparedImage::plan(&self) -> j2k::DeviceDecodePlan +pub fn j2k::PreparedImage::preparation_depth(&self) -> j2k::PreparationDepth +pub fn j2k::PreparedImage::request(&self) -> j2k::DecodeRequest +pub fn j2k::PreparedImage::source_index(&self) -> usize +pub fn j2k::PreparedImage::support(&self) -> &j2k::J2kSupportInfo pub fn j2k::decode_tiles_into(&mut [j2k::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result>, j2k::TileBatchError> pub fn j2k::decode_tiles_region_into(&mut [j2k::TileRegionDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result>, j2k::TileBatchError> pub fn j2k::decode_tiles_region_scaled_into(&mut [j2k::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result>, j2k::TileBatchError> @@ -215,6 +350,8 @@ pub fn j2k::extract_j2k_codestream_payload(&[u8]) -> core::result::Result) -> u8 pub fn j2k::j2k_lossless_decomposition_levels_for_options(j2k::J2kLosslessSamples<'_>, j2k::J2kLosslessEncodeOptions) -> u8 pub fn j2k::j2k_lossless_decomposition_levels_for_progression(j2k::J2kLosslessSamples<'_>, j2k::J2kProgressionOrder) -> u8 +pub fn j2k::prepare_batch(alloc::vec::Vec, j2k::BatchDecodeOptions) -> core::result::Result +pub fn j2k::prepare_batch_from_images(alloc::vec::Vec, j2k::BatchDecodeOptions) -> core::result::Result pub fn j2k::recode_j2k_to_htj2k_lossless(&[u8], j2k::J2kToHtj2kOptions) -> core::result::Result pub fn j2k::wrap_j2k_codestream(&[u8], j2k::J2kFileWrapOptions<'_>) -> core::result::Result, j2k::J2kError> pub j2k::BackendErrorKind::Buffer @@ -224,8 +361,53 @@ pub j2k::BackendErrorKind::Other pub j2k::BackendErrorKind::Truncated pub j2k::BackendErrorKind::Unsupported pub j2k::BackendErrorKind::Validation +pub j2k::BatchAlpha::None +pub j2k::BatchAlpha::Premultiplied +pub j2k::BatchAlpha::Straight +pub j2k::BatchCodecRoute::Classic +pub j2k::BatchCodecRoute::Htj2k +pub j2k::BatchDecodeOptions::layout: j2k::BatchLayout +pub j2k::BatchDecodeOptions::settings: j2k::DecodeSettings +pub j2k::BatchDecodeOptions::workers: core::option::Option +pub j2k::BatchErrorStage::Decode +pub j2k::BatchErrorStage::Prepare +pub j2k::BatchGroupInfo::alpha: j2k::BatchAlpha +pub j2k::BatchGroupInfo::color: j2k_core::pixel::PixelLayout +pub j2k::BatchGroupInfo::colorspace: j2k_core::types::Colorspace +pub j2k::BatchGroupInfo::dimensions: (u32, u32) +pub j2k::BatchGroupInfo::layout: j2k::BatchLayout +pub j2k::BatchGroupInfo::payload_kind: j2k_core::passthrough::CompressedPayloadKind +pub j2k::BatchGroupInfo::precision: u8 +pub j2k::BatchGroupInfo::route: j2k::BatchCodecRoute +pub j2k::BatchGroupInfo::sample_type: j2k_core::sample::SampleType +pub j2k::BatchGroupInfo::signed: bool +pub j2k::BatchGroupInfo::transfer_syntax: j2k_core::passthrough::CompressedTransferSyntax +pub j2k::BatchGroupInfo::transform: j2k::BatchWaveletTransform +pub j2k::BatchItemError::Codec +pub j2k::BatchItemError::Codec::source: alloc::sync::Arc +pub j2k::BatchItemError::Codec::stage: j2k::BatchErrorStage +pub j2k::BatchItemError::NonRepresentableBatchOutput +pub j2k::BatchItemError::NonRepresentableBatchOutput::reason: j2k::NonRepresentableReason +pub j2k::BatchItemError::PreparedDecodeSettingsMismatch +pub j2k::BatchItemError::PreparedDecodeSettingsMismatch::prepared: j2k::DecodeSettings +pub j2k::BatchItemError::PreparedDecodeSettingsMismatch::requested: j2k::DecodeSettings +pub j2k::BatchLayout::Nchw +pub j2k::BatchLayout::Nhwc +pub j2k::BatchWaveletTransform::Irreversible97 +pub j2k::BatchWaveletTransform::Reversible53 +pub j2k::CpuBatchSamples::I16(alloc::vec::Vec) +pub j2k::CpuBatchSamples::U16(alloc::vec::Vec) +pub j2k::CpuBatchSamples::U8(alloc::vec::Vec) pub j2k::CpuDecodeParallelism::Auto pub j2k::CpuDecodeParallelism::Serial +pub j2k::DecodeRequest::Full +pub j2k::DecodeRequest::Reduced +pub j2k::DecodeRequest::Reduced::scale: j2k_core::scale::Downscale +pub j2k::DecodeRequest::Region +pub j2k::DecodeRequest::Region::roi: j2k_core::types::Rect +pub j2k::DecodeRequest::RegionReduced +pub j2k::DecodeRequest::RegionReduced::roi: j2k_core::types::Rect +pub j2k::DecodeRequest::RegionReduced::scale: j2k_core::scale::Downscale pub j2k::DeviceDecodeRequest::Full pub j2k::DeviceDecodeRequest::Region pub j2k::DeviceDecodeRequest::Region::roi: j2k_core::types::Rect @@ -237,6 +419,8 @@ pub j2k::DeviceDecodeRequest::Scaled::scale: j2k_core::scale::Downscale pub j2k::EncodeBackendPreference::Auto pub j2k::EncodeBackendPreference::CpuOnly pub j2k::EncodeBackendPreference::RequireDevice +pub j2k::EncodedImage::bytes: alloc::sync::Arc<[u8]> +pub j2k::EncodedImage::request: j2k::DecodeRequest pub j2k::EncodedJ2k::backend: j2k_core::backend::BackendKind pub j2k::EncodedJ2k::bit_depth: u8 pub j2k::EncodedJ2k::codestream: alloc::vec::Vec @@ -254,6 +438,8 @@ pub j2k::EncodedLossyJ2k::height: u32 pub j2k::EncodedLossyJ2k::report: j2k::J2kLossyEncodeReport pub j2k::EncodedLossyJ2k::signed: bool pub j2k::EncodedLossyJ2k::width: u32 +pub j2k::IndexedBatchError::index: usize +pub j2k::IndexedBatchError::source: j2k::BatchItemError pub j2k::J2kBlockCodingMode::Classic pub j2k::J2kBlockCodingMode::HighThroughput pub j2k::J2kChannelAssociation::Color @@ -482,6 +668,16 @@ pub j2k::J2kToHtj2kReport::mode: j2k::J2kToHtj2kMode pub j2k::J2kToHtj2kReport::output_payload_kind: j2k_core::passthrough::CompressedPayloadKind pub j2k::J2kToHtj2kReport::output_transfer_syntax: j2k_core::passthrough::CompressedTransferSyntax pub j2k::J2kToHtj2kReport::width: u32 +pub j2k::NonRepresentableReason::ComponentSubsampling +pub j2k::NonRepresentableReason::MixedPrecision +pub j2k::NonRepresentableReason::MixedSignedness +pub j2k::NonRepresentableReason::MixedWaveletTransform +pub j2k::NonRepresentableReason::PrecisionAboveSixteen +pub j2k::NonRepresentableReason::UnsupportedColor +pub j2k::NonRepresentableReason::UnsupportedComponentCount +pub j2k::PreparationDepth::ClassicOffsetPlan +pub j2k::PreparationDepth::Htj2kOffsetPlan +pub j2k::PreparationDepth::MetadataOnly pub j2k::ReencodedHtj2k::bytes: alloc::vec::Vec pub j2k::ReencodedHtj2k::report: j2k::J2kToHtj2kReport pub j2k::ReversibleTransform::None53 @@ -491,9 +687,18 @@ pub j2k::TileDecodeOutput::out: &'o mut [u8] pub j2k::TileDecodeOutput::stride: usize pub mod j2k pub struct j2k::BackendError +pub struct j2k::BatchDecodeOptions +pub struct j2k::BatchGroupInfo +pub struct j2k::CpuBatchDecodeResult +pub struct j2k::CpuBatchDecoder +pub struct j2k::CpuBatchGroup +pub struct j2k::CpuBatchWorkspaceStats +pub struct j2k::DecodeSettings pub struct j2k::DeviceDecodePlan +pub struct j2k::EncodedImage pub struct j2k::EncodedJ2k pub struct j2k::EncodedLossyJ2k +pub struct j2k::IndexedBatchError pub struct j2k::J2kChannelDefinition pub struct j2k::J2kCodec pub struct j2k::J2kCodestreamPayload<'a> @@ -525,8 +730,18 @@ pub struct j2k::J2kSupportInfo pub struct j2k::J2kToHtj2kReport pub struct j2k::J2kView<'a> pub struct j2k::NativeBackendError +pub struct j2k::PreparedBatch +pub struct j2k::PreparedBatchGroup +pub struct j2k::PreparedClassicPlan +pub struct j2k::PreparedHtj2kPlan +pub struct j2k::PreparedImage pub struct j2k::ReencodedHtj2k pub struct j2k::TileDecodeOutput<'o> +pub trait j2k::BatchDecoder +pub type j2k::BatchDecoder::Error: core::convert::From +pub type j2k::BatchDecoder::Output +pub type j2k::CpuBatchDecoder::Error = j2k_core::batch::BatchInfrastructureError +pub type j2k::CpuBatchDecoder::Output = j2k::CpuBatchDecodeResult pub type j2k::TileBatchError = j2k_core::batch::BatchDecodeError pub type j2k::TileDecodeJob<'i, 'o> = j2k_core::batch::TileDecodeJob<'i, 'o> pub type j2k::TileRegionDecodeJob<'i, 'o> = j2k_core::batch::TileRegionDecodeJob<'i, 'o> @@ -534,16 +749,22 @@ pub type j2k::TileRegionScaledDecodeJob<'i, 'o> = j2k_core::batch::TileRegionSca pub type j2k::TileScaledDecodeJob<'i, 'o> = j2k_core::batch::TileScaledDecodeJob<'i, 'o> pub use j2k::BackendKind pub use j2k::BackendRequest +pub use j2k::BatchColor +pub use j2k::BatchInfrastructureError pub use j2k::BufferError +pub use j2k::ClassicCodeBlockPayload pub use j2k::CodecError pub use j2k::CompressedPayloadKind pub use j2k::CompressedTransferSyntax pub use j2k::DecodeOutcome pub use j2k::DecodeRowsError pub use j2k::Downscale +pub use j2k::Htj2kPayloadRanges pub use j2k::ImageCodec pub use j2k::ImageDecode pub use j2k::ImageDecodeRows +pub use j2k::J2kCodestreamRange +pub use j2k::NativeSampleType pub use j2k::PassthroughCandidate pub use j2k::PassthroughDecision pub use j2k::PassthroughRejectReason @@ -581,7 +802,9 @@ impl j2k_core::CpuFeatures impl j2k_core::Downscale impl j2k_core::PassthroughRequirements impl j2k_core::PixelFormat +impl j2k_core::PixelLayout impl j2k_core::Rect +impl j2k_core::Sample for i16 impl j2k_core::Sample for u16 impl j2k_core::Sample for u8 impl j2k_core::TileBatchOptions @@ -626,6 +849,7 @@ pub const fn j2k_core::PixelFormat::bytes_per_sample(self) -> usize pub const fn j2k_core::PixelFormat::channels(self) -> usize pub const fn j2k_core::PixelFormat::layout(self) -> j2k_core::PixelLayout pub const fn j2k_core::PixelFormat::sample(self) -> j2k_core::SampleType +pub const fn j2k_core::PixelLayout::channels(self) -> usize pub const fn j2k_core::Rect::full((u32, u32)) -> Self pub const fn j2k_core::TileBatchOptions::new(core::option::Option) -> Self pub const fn j2k_core::accelerator::DeviceMemoryRange::new(j2k_core::BackendKind, u64, usize, usize) -> Self @@ -635,6 +859,8 @@ pub const fn j2k_core::accelerator::SurfaceMetadata::byte_len(self) -> usize pub const fn j2k_core::accelerator::SurfaceMetadata::new(j2k_core::BackendKind, j2k_core::accelerator::SurfaceResidency, (u32, u32), j2k_core::PixelFormat, usize) -> Self pub const fn j2k_core::accelerator::SurfaceMetadata::with_byte_offset(self, usize) -> Self pub const fn j2k_core::accelerator::SurfaceResidency::for_backend(j2k_core::BackendKind) -> Self +pub const i16::BITS: u8 +pub const i16::TYPE: j2k_core::SampleType pub const j2k_core::BackendRequest::ACCELERATED: Self pub const j2k_core::BackendRequest::CPU_ONLY: Self pub const j2k_core::BackendRequest::STRICT_CUDA: Self @@ -758,6 +984,8 @@ pub j2k_core::BatchInfrastructureError::ResultIndexOutOfBounds::job_count: usize pub j2k_core::BatchInfrastructureError::ResultKindMismatch pub j2k_core::BatchInfrastructureError::ResultKindMismatch::index: usize pub j2k_core::BatchInfrastructureError::SchedulerPoisoned +pub j2k_core::BatchInfrastructureError::UnsupportedContract +pub j2k_core::BatchInfrastructureError::UnsupportedContract::what: &'static str pub j2k_core::BatchInfrastructureError::WorkerPanicked pub j2k_core::BatchInfrastructureError::WorkerPanicked::worker: usize pub j2k_core::BatchInfrastructureError::WorkerSlotMissing @@ -887,10 +1115,13 @@ pub j2k_core::PassthroughRequirements::tile_layout: core::option::Option j2k_metal::J2kDecoder<'a> impl<'a> j2k_metal::MetalLosslessEncodeTile<'a> +pub const fn j2k_metal::MetalBatchDecoder::options(&self) -> j2k::owned_batch::contracts::BatchDecodeOptions pub const fn j2k_metal::MetalDecodeRequest::full(j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> Self pub const fn j2k_metal::MetalDecodeRequest::region(j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> Self pub const fn j2k_metal::MetalDecodeRequest::region_scaled(j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> Self pub const fn j2k_metal::MetalDecodeRequest::scaled(j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> Self +pub const fn j2k_metal::MetalResidentBatch::byte_len(&self) -> usize +pub const fn j2k_metal::MetalResidentBatch::byte_offset(&self) -> usize +pub const fn j2k_metal::MetalResidentBatch::image_count(&self) -> usize +pub const fn j2k_metal::MetalResidentBatch::image_stride_bytes(&self) -> usize pub enum j2k_metal::Error pub enum j2k_metal::MetalDecodeOp pub enum j2k_metal::MetalEncodeInputStaging @@ -1983,6 +2236,47 @@ pub fn j2k_metal::MetalBackendSession::device(&self) -> &metal::device::DeviceRe pub fn j2k_metal::MetalBackendSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_metal::MetalBackendSession::new(metal::device::Device) -> Self pub fn j2k_metal::MetalBackendSession::system_default() -> core::result::Result +pub fn j2k_metal::MetalBackendSession::with_command_queue(metal::device::Device, metal::commandqueue::CommandQueue) -> core::result::Result +pub fn j2k_metal::MetalBatchDecodeResult::errors(&self) -> &[j2k::owned_batch::errors::IndexedBatchError] +pub fn j2k_metal::MetalBatchDecodeResult::group_errors(&self) -> &[j2k_metal::MetalBatchGroupError] +pub fn j2k_metal::MetalBatchDecodeResult::groups(&self) -> &[j2k_metal::MetalBatchGroup] +pub fn j2k_metal::MetalBatchDecodeResult::into_parts(self) -> (alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec) +pub fn j2k_metal::MetalBatchDecoder::backend_session(&self) -> &j2k_metal::MetalBackendSession +pub fn j2k_metal::MetalBatchDecoder::decode_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::decode_prepared_group(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::decode_prepared_group_into(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, &j2k_metal_support::resident::destination::MetalImageDestination) -> core::result::Result<(), j2k_metal::Error> +pub fn j2k_metal::MetalBatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal::MetalBatchDecoder::options(&self) -> j2k::owned_batch::contracts::BatchDecodeOptions +pub fn j2k_metal::MetalBatchDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submissions(&self) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submit_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submit_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submit_prepared_group_into(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, j2k_metal_support::resident::destination::MetalImageDestination) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submit_prepared_group_into_for_consumer_queue(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, j2k_metal_support::resident::destination::MetalImageDestination, &metal::commandqueue::CommandQueueRef) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::system_default() -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::system_default_with_options(j2k::owned_batch::contracts::BatchDecodeOptions) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::validate_destination(&self, &j2k_metal_support::resident::destination::MetalImageDestination, (u32, u32), j2k_core::pixel::PixelFormat) -> core::result::Result<(), j2k_metal::Error> +pub fn j2k_metal::MetalBatchDecoder::with_backend_session(j2k_metal::MetalBackendSession) -> Self +pub fn j2k_metal::MetalBatchDecoder::with_backend_session_and_options(j2k_metal::MetalBackendSession, j2k::owned_batch::contracts::BatchDecodeOptions) -> Self +pub fn j2k_metal::MetalBatchGroup::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k_metal::MetalBatchGroup::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal::MetalBatchGroup::info(&self) -> &j2k::owned_batch::contracts::BatchGroupInfo +pub fn j2k_metal::MetalBatchGroup::into_parts(self) -> j2k_metal::MetalBatchGroupParts +pub fn j2k_metal::MetalBatchGroup::into_resident_batch(self) -> core::option::Option +pub fn j2k_metal::MetalBatchGroup::resident_batch(&self) -> core::option::Option<&j2k_metal::MetalResidentBatch> +pub fn j2k_metal::MetalBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k_metal::MetalBatchGroup::surfaces(&self) -> &[j2k_metal::Surface] +pub fn j2k_metal::MetalBatchGroup::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k_metal::MetalBatchGroupCompletion::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k_metal::MetalBatchGroupCompletion::into_parts(self) -> (alloc::vec::Vec, alloc::vec::Vec>) +pub fn j2k_metal::MetalBatchGroupCompletion::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k_metal::MetalBatchGroupError::into_parts(self) -> (alloc::vec::Vec, j2k_metal::Error) +pub fn j2k_metal::MetalBatchGroupError::source(&self) -> &j2k_metal::Error +pub fn j2k_metal::MetalBatchGroupError::source_indices(&self) -> &[usize] pub fn j2k_metal::MetalEncodeStageAccelerator::default() -> Self pub fn j2k_metal::MetalEncodeStageAccelerator::for_auto_host_output() -> Self pub fn j2k_metal::MetalEncodeStageAccelerator::for_ht_code_block_encode() -> Self @@ -2006,6 +2300,8 @@ pub fn j2k_metal::MetalLosslessEncodeTile<'a>::from_resident(&'a j2k_metal_suppo pub fn j2k_metal::MetalLosslessEncodeTile<'a>::output_dimensions(self) -> (u32, u32) pub fn j2k_metal::MetalLosslessEncodeTile<'a>::pitch_bytes(self) -> usize pub fn j2k_metal::MetalLosslessEncodeTile<'a>::pixel_format(self) -> j2k_core::pixel::PixelFormat +pub fn j2k_metal::MetalResidentBatch::device_registry_id(&self) -> u64 +pub fn j2k_metal::MetalResidentBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_metal::MetalSession::backend_session(&self) -> core::option::Option<&j2k_metal::MetalBackendSession> pub fn j2k_metal::MetalSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_metal::MetalSession::submissions(&self) -> core::result::Result @@ -2020,6 +2316,15 @@ pub fn j2k_metal::MetalTileBatch::submissions(&self) -> core::result::Result Self pub fn j2k_metal::NativeBackendError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_metal::NativeBackendError::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)> +pub fn j2k_metal::SubmittedMetalGroupDecodeInto::enqueue_consumer_wait(&mut self, &metal::commandqueue::CommandQueueRef) -> core::result::Result<(), j2k_metal::Error> +pub fn j2k_metal::SubmittedMetalGroupDecodeInto::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal::SubmittedMetalGroupDecodeInto::wait(self) -> core::result::Result +pub fn j2k_metal::SubmittedMetalGroupDecodeInto::wait(self) -> core::result::Result +pub fn j2k_metal::SubmittedMetalPreparedBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal::SubmittedMetalPreparedBatch::is_empty(&self) -> bool +pub fn j2k_metal::SubmittedMetalPreparedBatch::len(&self) -> usize +pub fn j2k_metal::SubmittedMetalPreparedBatch::wait(self) -> core::result::Result +pub fn j2k_metal::SubmittedMetalPreparedBatch::wait(self) -> core::result::Result pub fn j2k_metal::Surface::as_bytes(&self) -> core::result::Result, j2k_metal::Error> pub fn j2k_metal::Surface::download_into(&self, &mut [u8], usize) -> core::result::Result<(), j2k_metal::Error> pub fn j2k_metal::Surface::into_resident_metal_image(self) -> core::option::Option @@ -2107,6 +2412,11 @@ pub mod j2k_metal pub struct j2k_metal::Codec pub struct j2k_metal::J2kDecoder<'a> pub struct j2k_metal::MetalBackendSession +pub struct j2k_metal::MetalBatchDecodeResult +pub struct j2k_metal::MetalBatchDecoder +pub struct j2k_metal::MetalBatchGroup +pub struct j2k_metal::MetalBatchGroupCompletion +pub struct j2k_metal::MetalBatchGroupError pub struct j2k_metal::MetalDecodeRequest pub struct j2k_metal::MetalEncodeStageAccelerator pub struct j2k_metal::MetalEncodedJ2k @@ -2114,18 +2424,31 @@ pub struct j2k_metal::MetalLosslessEncodeBatchRequest<'a, 'b> pub struct j2k_metal::MetalLosslessEncodeConfig pub struct j2k_metal::MetalLosslessEncodeResidency pub struct j2k_metal::MetalLosslessEncodeTile<'a> +pub struct j2k_metal::MetalResidentBatch pub struct j2k_metal::MetalSession pub struct j2k_metal::MetalTileBatch pub struct j2k_metal::NativeBackendError pub struct j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch pub struct j2k_metal::SubmittedJ2kLosslessMetalEncodeBatch +pub struct j2k_metal::SubmittedMetalGroupDecodeInto +pub struct j2k_metal::SubmittedMetalPreparedBatch pub struct j2k_metal::Surface +pub type j2k_metal::MetalBatchDecoder::Error = j2k_metal::Error +pub type j2k_metal::MetalBatchDecoder::Output = j2k_metal::MetalBatchDecodeResult +pub type j2k_metal::MetalBatchGroupParts = (j2k::owned_batch::contracts::BatchGroupInfo, alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec>, alloc::vec::Vec) +pub type j2k_metal::SubmittedMetalGroupDecodeInto::Error = j2k_metal::Error +pub type j2k_metal::SubmittedMetalGroupDecodeInto::Output = j2k_metal::MetalBatchGroupCompletion +pub type j2k_metal::SubmittedMetalPreparedBatch::Error = j2k_metal::Error +pub type j2k_metal::SubmittedMetalPreparedBatch::Output = j2k_metal::MetalBatchDecodeResult pub unsafe fn j2k_metal::MetalEncodedJ2k::from_raw_parts(metal::buffer::Buffer, core::ops::range::Range, usize, (u32, u32), u8, u8, bool) -> core::result::Result pub unsafe fn j2k_metal::MetalEncodedJ2k::into_codestream_buffer(self) -> metal::buffer::Buffer pub unsafe fn j2k_metal::MetalLosslessEncodeTile<'a>::from_buffer(&'a metal::buffer::Buffer, usize, (u32, u32), usize, (u32, u32), j2k_core::pixel::PixelFormat) -> Self +pub unsafe fn j2k_metal::MetalResidentBatch::metal_buffer(&self) -> &metal::buffer::Buffer pub unsafe fn j2k_metal::Surface::metal_buffer(&self) -> core::option::Option<(&metal::buffer::Buffer, usize)> pub use j2k_metal::J2kContext pub use j2k_metal::J2kScratchPool +pub use j2k_metal::MetalImageDestination +pub use j2k_metal::MetalImageLayout pub use j2k_metal::SurfaceResidency ``` @@ -2208,21 +2531,73 @@ pub use j2k_jpeg_cuda::ScratchPool ## `j2k-cuda` ```text +#[non_exhaustive] pub enum j2k_cuda::CudaBatchError #[non_exhaustive] pub enum j2k_cuda::Error impl core::error::Error for j2k_cuda::NativeBackendError impl core::fmt::Debug for j2k_cuda::CudaSession +impl core::fmt::Debug for j2k_cuda::SubmittedCudaExternalBatch +impl core::fmt::Debug for j2k_cuda::SubmittedCudaResidentBatch impl core::fmt::Display for j2k_cuda::NativeBackendError +impl j2k::owned_batch::prepared::BatchDecoder for j2k_cuda::CudaBatchDecoder +impl j2k_cuda::CudaBatchDecodeResult +impl j2k_cuda::CudaBatchDecoder +impl j2k_cuda::CudaBatchGroup +impl j2k_cuda::CudaBatchGroupError +impl j2k_cuda::CudaDecodePoolDiagnostics +impl j2k_cuda::CudaDecodePoolSnapshot impl j2k_cuda::CudaEncodeStageAccelerator impl j2k_cuda::CudaEncodeStageTimings impl j2k_cuda::CudaEncodedJ2k +impl j2k_cuda::CudaExternalBatchGroup +impl j2k_cuda::CudaResidentBatchBuffer impl j2k_cuda::CudaResidentCodestreamBuffer impl j2k_cuda::CudaSession impl j2k_cuda::CudaSurface<'_> +impl j2k_cuda::SubmittedCudaExternalBatch +impl j2k_cuda::SubmittedCudaResidentBatch impl j2k_cuda::Surface impl<'a> j2k_cuda::J2kDecoder<'a> +pub const fn j2k_cuda::CudaBatchDecoder::options(&self) -> j2k::owned_batch::contracts::BatchDecodeOptions +pub const fn j2k_cuda::CudaBatchDecoder::session(&self) -> &j2k_cuda::CudaSession +pub const fn j2k_cuda::CudaBatchDecoder::with_session_and_options(j2k_cuda::CudaSession, j2k::owned_batch::contracts::BatchDecodeOptions) -> Self +pub const fn j2k_cuda::CudaBatchGroup::dense_output(&self) -> &j2k_cuda::CudaResidentBatchBuffer +pub const fn j2k_cuda::CudaBatchGroup::info(&self) -> &j2k::owned_batch::contracts::BatchGroupInfo +pub const fn j2k_cuda::CudaDecodePoolSnapshot::peak_retained_bytes_upper_bound(self) -> usize +pub const fn j2k_cuda::CudaDecodePoolSnapshot::retained_bytes(self) -> usize pub const fn j2k_cuda::CudaEncodeStageAccelerator::collected_stage_timings(&self) -> j2k_cuda::CudaEncodeStageTimings pub const fn j2k_cuda::CudaEncodeStageTimings::saturating_add(self, Self) -> Self pub const fn j2k_cuda::CudaEncodeStageTimings::total_us(self) -> u128 +pub const fn j2k_cuda::CudaExternalBatchGroup::info(&self) -> &j2k::owned_batch::contracts::BatchGroupInfo +pub enum j2k_cuda::CudaExternalBatchTryFinish +pub fn j2k_cuda::CudaBatchDecodeResult::errors(&self) -> &[j2k::owned_batch::errors::IndexedBatchError] +pub fn j2k_cuda::CudaBatchDecodeResult::group_errors(&self) -> &[j2k_cuda::CudaBatchGroupError] +pub fn j2k_cuda::CudaBatchDecodeResult::groups(&self) -> &[j2k_cuda::CudaBatchGroup] +pub fn j2k_cuda::CudaBatchDecodeResult::into_parts(self) -> (alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec) +pub fn j2k_cuda::CudaBatchDecoder::decode_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::decode_pool_diagnostics(&self) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::diagnostics(&self) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::new() -> Self +pub fn j2k_cuda::CudaBatchDecoder::options(&self) -> j2k::owned_batch::contracts::BatchDecodeOptions +pub fn j2k_cuda::CudaBatchDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::session_mut(&mut self) -> &mut j2k_cuda::CudaSession +pub fn j2k_cuda::CudaBatchDecoder::submit_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::submit_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::with_options(j2k::owned_batch::contracts::BatchDecodeOptions) -> Self +pub fn j2k_cuda::CudaBatchDecoder::with_session(j2k_cuda::CudaSession) -> Self +pub fn j2k_cuda::CudaBatchGroup::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k_cuda::CudaBatchGroup::into_parts(self) -> (j2k::owned_batch::contracts::BatchGroupInfo, alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec>, alloc::vec::Vec, j2k_cuda::CudaResidentBatchBuffer) +pub fn j2k_cuda::CudaBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k_cuda::CudaBatchGroup::surfaces(&self) -> &[j2k_cuda::Surface] +pub fn j2k_cuda::CudaBatchGroup::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k_cuda::CudaBatchGroupError::into_parts(self) -> (alloc::vec::Vec, j2k_cuda::Error) +pub fn j2k_cuda::CudaBatchGroupError::source(&self) -> &j2k_cuda::Error +pub fn j2k_cuda::CudaBatchGroupError::source_indices(&self) -> &[usize] +pub fn j2k_cuda::CudaDecodePoolDiagnostics::peak_retained_bytes_upper_bound(self) -> usize +pub fn j2k_cuda::CudaDecodePoolDiagnostics::retained_bytes(self) -> usize pub fn j2k_cuda::CudaEncodeStageAccelerator::for_auto_host_output() -> Self pub fn j2k_cuda::CudaEncodeStageAccelerator::prefer_cpu_forward_rct(self, bool) -> Self pub fn j2k_cuda::CudaEncodeStageAccelerator::prefer_cpu_ht_subband(self, bool) -> Self @@ -2232,10 +2607,18 @@ pub fn j2k_cuda::CudaEncodeStageAccelerator::reset_collected_stage_timings(&mut pub fn j2k_cuda::CudaEncodedJ2k::codestream(&self) -> &j2k_cuda::CudaResidentCodestreamBuffer pub fn j2k_cuda::CudaEncodedJ2k::into_parts(self) -> (j2k_cuda::CudaEncodedJ2kMetadata, j2k_cuda::CudaResidentCodestreamBuffer) pub fn j2k_cuda::CudaEncodedJ2k::metadata(&self) -> &j2k_cuda::CudaEncodedJ2kMetadata +pub fn j2k_cuda::CudaExternalBatchGroup::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k_cuda::CudaExternalBatchGroup::ranges(&self) -> &[j2k_cuda_runtime::memory::CudaDeviceBufferRange] +pub fn j2k_cuda::CudaExternalBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k_cuda::CudaExternalBatchGroup::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k_cuda::CudaResidentBatchBuffer::buffer(&self) -> &j2k_cuda_runtime::memory::CudaDeviceBuffer +pub fn j2k_cuda::CudaResidentBatchBuffer::ranges(&self) -> &[j2k_cuda_runtime::memory::CudaDeviceBufferRange] pub fn j2k_cuda::CudaResidentCodestreamBuffer::buffer(&self) -> &j2k_cuda_runtime::memory::CudaDeviceBuffer pub fn j2k_cuda::CudaResidentCodestreamBuffer::byte_len(&self) -> usize pub fn j2k_cuda::CudaResidentCodestreamBuffer::download(&self) -> core::result::Result, j2k_cuda::Error> pub fn j2k_cuda::CudaResidentCodestreamBuffer::into_buffer(self) -> j2k_cuda_runtime::memory::CudaDeviceBuffer +pub fn j2k_cuda::CudaSession::decode_pool_diagnostics(&self) -> core::result::Result +pub fn j2k_cuda::CudaSession::diagnostics(&self) -> core::result::Result pub fn j2k_cuda::CudaSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_cuda::CudaSession::is_runtime_initialized(&self) -> bool pub fn j2k_cuda::CudaSession::submissions(&self) -> u64 @@ -2250,6 +2633,15 @@ pub fn j2k_cuda::J2kDecoder<'a>::decode_to_host_surface(&mut self, j2k_core::pix pub fn j2k_cuda::J2kDecoder<'a>::new(&'a [u8]) -> core::result::Result pub fn j2k_cuda::NativeBackendError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_cuda::NativeBackendError::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)> +pub fn j2k_cuda::SubmittedCudaExternalBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_cuda::SubmittedCudaExternalBatch::group(&self) -> &j2k_cuda::CudaExternalBatchGroup +pub fn j2k_cuda::SubmittedCudaExternalBatch::is_complete(&self) -> core::result::Result +pub fn j2k_cuda::SubmittedCudaExternalBatch::try_finish(self) -> core::result::Result +pub fn j2k_cuda::SubmittedCudaExternalBatch::wait(self) -> core::result::Result +pub fn j2k_cuda::SubmittedCudaResidentBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_cuda::SubmittedCudaResidentBatch::is_complete(&self) -> core::result::Result +pub fn j2k_cuda::SubmittedCudaResidentBatch::pending_group_count(&self) -> usize +pub fn j2k_cuda::SubmittedCudaResidentBatch::wait(self) -> core::result::Result pub fn j2k_cuda::Surface::as_host_bytes(&self) -> core::option::Option<&[u8]> pub fn j2k_cuda::Surface::cuda_surface(&self) -> core::option::Option> pub fn j2k_cuda::Surface::download_batch_tight(&[Self]) -> core::result::Result, j2k_cuda::Error> @@ -2264,6 +2656,19 @@ pub fn j2k_cuda::encode_lossless_from_cuda_buffers(&[j2k_cuda::CudaLosslessEncod pub fn j2k_cuda::encode_lossless_from_cuda_buffers_to_cuda_buffers(&[j2k_cuda::CudaLosslessEncodeTile<'_>], &j2k::encode::contracts::J2kLosslessEncodeOptions, &mut j2k_cuda::CudaSession) -> core::result::Result, j2k_cuda::Error> pub fn j2k_cuda::submit_lossless_from_cuda_buffer(j2k_cuda::CudaLosslessEncodeTile<'_>, &j2k::encode::contracts::J2kLosslessEncodeOptions, &mut j2k_cuda::CudaSession) -> core::result::Result pub fn j2k_cuda::submit_lossless_from_cuda_buffers(&[j2k_cuda::CudaLosslessEncodeTile<'_>], &j2k::encode::contracts::J2kLosslessEncodeOptions, &mut j2k_cuda::CudaSession) -> core::result::Result +pub j2k_cuda::CudaBatchError::GroupExecution +pub j2k_cuda::CudaBatchError::GroupExecution::source: alloc::boxed::Box +pub j2k_cuda::CudaBatchError::GroupExecution::source_indices: alloc::vec::Vec +pub j2k_cuda::CudaBatchError::Infrastructure(j2k_core::batch::BatchInfrastructureError) +pub j2k_cuda::CudaDecodePoolDiagnostics::batch_decode: core::option::Option +pub j2k_cuda::CudaDecodePoolDiagnostics::decode: core::option::Option +pub j2k_cuda::CudaDecodePoolSnapshot::cached_buffers: usize +pub j2k_cuda::CudaDecodePoolSnapshot::cached_bytes: usize +pub j2k_cuda::CudaDecodePoolSnapshot::deferred_buffers: usize +pub j2k_cuda::CudaDecodePoolSnapshot::deferred_bytes: usize +pub j2k_cuda::CudaDecodePoolSnapshot::peak_cached_bytes: usize +pub j2k_cuda::CudaDecodePoolSnapshot::peak_deferred_bytes: usize +pub j2k_cuda::CudaDecodePoolSnapshot::reuse_holds: usize pub j2k_cuda::CudaEncodeStageTimings::deinterleave_us: u128 pub j2k_cuda::CudaEncodeStageTimings::dwt_us: u128 pub j2k_cuda::CudaEncodeStageTimings::ht_encode_us: u128 @@ -2279,6 +2684,8 @@ pub j2k_cuda::CudaEncodedJ2kMetadata::dispatch_report: j2k_types::J2kEncodeDispa pub j2k_cuda::CudaEncodedJ2kMetadata::height: u32 pub j2k_cuda::CudaEncodedJ2kMetadata::signed: bool pub j2k_cuda::CudaEncodedJ2kMetadata::width: u32 +pub j2k_cuda::CudaExternalBatchTryFinish::Complete(j2k_cuda::CudaExternalBatchGroup) +pub j2k_cuda::CudaExternalBatchTryFinish::Pending(j2k_cuda::SubmittedCudaExternalBatch) pub j2k_cuda::CudaLosslessEncodeResidency::codestream_assembly_used: bool pub j2k_cuda::CudaLosslessEncodeResidency::coefficient_prep_used: bool pub j2k_cuda::CudaLosslessEncodeResidency::packetization_used: bool @@ -2290,12 +2697,18 @@ pub j2k_cuda::CudaLosslessEncodeTile::output_height: u32 pub j2k_cuda::CudaLosslessEncodeTile::output_width: u32 pub j2k_cuda::CudaLosslessEncodeTile::pitch_bytes: usize pub j2k_cuda::CudaLosslessEncodeTile::width: u32 +pub j2k_cuda::CudaSessionDiagnostics::pools: j2k_cuda::CudaDecodePoolDiagnostics +pub j2k_cuda::CudaSessionDiagnostics::runtime: core::option::Option pub j2k_cuda::Error::Buffer(j2k_core::error::BufferError) pub j2k_cuda::Error::CudaCleanupFailed pub j2k_cuda::Error::CudaCleanupFailed::cleanup: alloc::boxed::Box pub j2k_cuda::Error::CudaCleanupFailed::primary: alloc::boxed::Box pub j2k_cuda::Error::CudaRuntime pub j2k_cuda::Error::CudaRuntime::source: j2k_cuda_runtime::error::CudaError +pub j2k_cuda::Error::CudaTier1JobFailed +pub j2k_cuda::Error::CudaTier1JobFailed::original_job_index: usize +pub j2k_cuda::Error::CudaTier1JobFailed::source: j2k_cuda_runtime::error::CudaError +pub j2k_cuda::Error::CudaTier1JobFailed::source_index: usize pub j2k_cuda::Error::CudaUnavailable pub j2k_cuda::Error::Decode(j2k::error::J2kError) pub j2k_cuda::Error::HostAllocationFailed @@ -2305,6 +2718,7 @@ pub j2k_cuda::Error::HostAllocationTooLarge pub j2k_cuda::Error::HostAllocationTooLarge::cap: usize pub j2k_cuda::Error::HostAllocationTooLarge::requested: usize pub j2k_cuda::Error::HostAllocationTooLarge::what: &'static str +pub j2k_cuda::Error::HtJobChunkPlan(j2k_core::batch::gpu_job_chunk::HtGpuJobChunkPlanError) pub j2k_cuda::Error::NativeDecode pub j2k_cuda::Error::NativeDecode::context: &'static str pub j2k_cuda::Error::NativeDecode::source: j2k_cuda::NativeBackendError @@ -2314,20 +2728,35 @@ pub j2k_cuda::Error::UnsupportedCudaRequest pub j2k_cuda::Error::UnsupportedCudaRequest::reason: &'static str pub mod j2k_cuda pub struct j2k_cuda::Codec +pub struct j2k_cuda::CudaBatchDecodeResult +pub struct j2k_cuda::CudaBatchDecoder +pub struct j2k_cuda::CudaBatchGroup +pub struct j2k_cuda::CudaBatchGroupError +pub struct j2k_cuda::CudaDecodePoolDiagnostics +pub struct j2k_cuda::CudaDecodePoolSnapshot pub struct j2k_cuda::CudaEncodeStageAccelerator pub struct j2k_cuda::CudaEncodeStageTimings pub struct j2k_cuda::CudaEncodedJ2k pub struct j2k_cuda::CudaEncodedJ2kMetadata +pub struct j2k_cuda::CudaExternalBatchGroup pub struct j2k_cuda::CudaLosslessEncodeResidency pub struct j2k_cuda::CudaLosslessEncodeTile<'a> +pub struct j2k_cuda::CudaResidentBatchBuffer pub struct j2k_cuda::CudaResidentCodestreamBuffer pub struct j2k_cuda::CudaSession +pub struct j2k_cuda::CudaSessionDiagnostics pub struct j2k_cuda::CudaSurface<'a> pub struct j2k_cuda::J2kDecoder<'a> pub struct j2k_cuda::NativeBackendError +pub struct j2k_cuda::SubmittedCudaExternalBatch +pub struct j2k_cuda::SubmittedCudaResidentBatch pub struct j2k_cuda::SubmittedJ2kLosslessCudaEncode pub struct j2k_cuda::SubmittedJ2kLosslessCudaEncodeBatch pub struct j2k_cuda::Surface +pub type j2k_cuda::CudaBatchDecoder::Error = j2k_cuda::CudaBatchError +pub type j2k_cuda::CudaBatchDecoder::Output = j2k_cuda::CudaBatchDecodeResult +pub unsafe fn j2k_cuda::CudaBatchDecoder::decode_batch_into(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, &mut j2k_cuda_runtime::memory::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result +pub unsafe fn j2k_cuda::CudaBatchDecoder::submit_batch_into(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, &mut j2k_cuda_runtime::memory::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result pub use j2k_cuda::J2kContext pub use j2k_cuda::J2kScratchPool pub use j2k_cuda::SurfaceResidency @@ -2825,12 +3254,14 @@ pub struct j2k_transcode_cuda::CudaRuntimeFailure #[non_exhaustive] pub enum j2k_metal_support::MetalSupportError #[non_exhaustive] pub struct j2k_metal_support::MetalRouteProfileLabels impl core::error::Error for j2k_metal_support::MetalSupportError +impl core::fmt::Debug for j2k_metal_support::MetalImageDestination impl core::fmt::Debug for j2k_metal_support::ResidentMetalImage impl core::fmt::Debug for j2k_metal_support::SubmittedMetalImages impl core::fmt::Display for j2k_metal_support::MetalCommandEncoderKind impl core::fmt::Display for j2k_metal_support::MetalSupportError impl core::ops::drop::Drop for j2k_metal_support::SubmittedMetalImages impl j2k_core::traits::DeviceSubmission for j2k_metal_support::SubmittedMetalImages +impl j2k_metal_support::MetalImageDestination impl j2k_metal_support::MetalImageLayout impl j2k_metal_support::MetalPipelineLoader impl j2k_metal_support::MetalRouteProfileLabels @@ -2839,9 +3270,15 @@ impl j2k_metal_support::ResidentMetalImage impl j2k_metal_support::SubmittedMetalImages impl core::clone::Clone for j2k_metal_support::MetalRuntimeSession impl j2k_metal_support::MetalRuntimeSession +pub const fn j2k_metal_support::MetalImageDestination::buffer_len(&self) -> usize +pub const fn j2k_metal_support::MetalImageDestination::device_registry_id(&self) -> u64 +pub const fn j2k_metal_support::MetalImageDestination::layout(&self) -> j2k_metal_support::MetalImageLayout pub const fn j2k_metal_support::MetalImageLayout::byte_len(self) -> usize pub const fn j2k_metal_support::MetalImageLayout::byte_offset(self) -> usize pub const fn j2k_metal_support::MetalImageLayout::dimensions(self) -> (u32, u32) +pub const fn j2k_metal_support::MetalImageLayout::image_count(self) -> usize +pub const fn j2k_metal_support::MetalImageLayout::image_offset_bytes(self, usize) -> core::option::Option +pub const fn j2k_metal_support::MetalImageLayout::image_stride_bytes(self) -> usize pub const fn j2k_metal_support::MetalImageLayout::pitch_bytes(self) -> usize pub const fn j2k_metal_support::MetalImageLayout::pixel_format(self) -> j2k_core::pixel::PixelFormat pub const fn j2k_metal_support::MetalRouteProfileLabels::new(&'static str, &'static str) -> Self @@ -2861,7 +3298,12 @@ pub const fn j2k_metal_support::reject_explicit_metal_route(&'static str) -> j2k pub const fn j2k_metal_support::reject_unsupported_backend_route() -> j2k_metal_support::MetalRouteProfileLabels pub const fn j2k_metal_support::two_d_threads_per_group(u64, u64) -> metal::types::MTLSize pub fn j2k_metal_support::MetalCommandEncoderKind::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal_support::MetalImageDestination::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal_support::MetalImageDestination::validate_batch(&self, (u32, u32), j2k_core::pixel::PixelFormat, usize) -> core::result::Result<(), j2k_metal_support::MetalSupportError> +pub fn j2k_metal_support::MetalImageDestination::validate_device(&self, &metal::device::DeviceRef) -> core::result::Result<(), j2k_metal_support::MetalSupportError> +pub fn j2k_metal_support::MetalImageDestination::validate_image(&self, (u32, u32), j2k_core::pixel::PixelFormat) -> core::result::Result<(), j2k_metal_support::MetalSupportError> pub fn j2k_metal_support::MetalImageLayout::new(usize, (u32, u32), usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_metal_support::MetalImageLayout::new_batch(usize, (u32, u32), usize, j2k_core::pixel::PixelFormat, usize, usize) -> core::result::Result pub fn j2k_metal_support::MetalPipelineLoader::library(&self) -> &metal::library::Library pub fn j2k_metal_support::MetalPipelineLoader::new(&metal::device::DeviceRef, &str) -> core::result::Result pub fn j2k_metal_support::MetalPipelineLoader::pipeline(&self, &str) -> core::result::Result @@ -2969,6 +3411,7 @@ pub j2k_metal_support::MetalSupportError::TextureDescriptorInvalid pub j2k_metal_support::MetalSupportError::TextureDescriptorInvalid::reason: &'static str pub j2k_metal_support::MetalSupportError::TextureDescriptorUnavailable pub mod j2k_metal_support +pub struct j2k_metal_support::MetalImageDestination pub struct j2k_metal_support::MetalImageLayout pub struct j2k_metal_support::MetalPipelineLoader pub struct j2k_metal_support::MetalRuntimeSession @@ -2976,6 +3419,8 @@ pub struct j2k_metal_support::ResidentMetalImage pub struct j2k_metal_support::SubmittedMetalImages pub type j2k_metal_support::SubmittedMetalImages::Error = j2k_metal_support::MetalSupportError pub type j2k_metal_support::SubmittedMetalImages::Output = alloc::vec::Vec +pub unsafe fn j2k_metal_support::MetalImageDestination::from_exclusive_buffer(metal::buffer::Buffer, j2k_metal_support::MetalImageLayout) -> core::result::Result +pub unsafe fn j2k_metal_support::MetalImageDestination::raw_buffer(&self) -> &metal::buffer::Buffer pub unsafe fn j2k_metal_support::ResidentMetalImage::from_completed_buffer(metal::buffer::Buffer, j2k_metal_support::MetalImageLayout) -> core::result::Result pub unsafe fn j2k_metal_support::ResidentMetalImage::raw_buffer(&self) -> &metal::buffer::Buffer pub unsafe fn j2k_metal_support::SubmittedMetalImages::from_uncommitted(&metal::device::DeviceRef, metal::commandbuffer::CommandBuffer, alloc::vec::Vec<(metal::buffer::Buffer, j2k_metal_support::MetalImageLayout)>, alloc::vec::Vec) -> core::result::Result @@ -3137,6 +3582,8 @@ impl j2k_native::DecodeError impl j2k_native::DecodeSettings impl j2k_native::DecodedNativeComponents impl j2k_native::DecoderContext<'_> +impl j2k_native::DecoderWorkspace +impl j2k_native::DecoderWorkspaceStats impl j2k_native::NativeComponentPlane impl<'a> j2k_native::ComponentPlane<'a> impl<'a> j2k_native::DecodedComponents<'a> @@ -3145,6 +3592,17 @@ pub const fn j2k_native::DecodeError::classify(&self) -> j2k_native::DecodeError pub const fn j2k_native::DecodeSettings::lenient() -> Self pub const fn j2k_native::DecodeSettings::lenient_tolerance_enabled(&self) -> bool pub const fn j2k_native::DecodeSettings::strict() -> Self +pub const fn j2k_native::DecoderContext<'_>::workspace_stats(&self) -> j2k_native::DecoderWorkspaceStats +pub const fn j2k_native::DecoderWorkspace::stats(&self) -> j2k_native::DecoderWorkspaceStats +pub const fn j2k_native::DecoderWorkspaceStats::component_owner_reuses(self) -> u64 +pub const fn j2k_native::DecoderWorkspaceStats::decode_calls(self) -> u64 +pub const fn j2k_native::DecoderWorkspaceStats::idwt_owner_reuses(self) -> u64 +pub const fn j2k_native::DecoderWorkspaceStats::retained_component_bytes(self) -> usize +pub const fn j2k_native::DecoderWorkspaceStats::retained_idwt_bytes(self) -> usize +pub const fn j2k_native::DecoderWorkspaceStats::retained_scratch_bytes(self) -> usize +pub const fn j2k_native::DecoderWorkspaceStats::retained_tier1_bytes(self) -> usize +pub const fn j2k_native::DecoderWorkspaceStats::scratch_capacity_retries(self) -> u64 +pub const fn j2k_native::DecoderWorkspaceStats::tier1_owner_reuses(self) -> u64 pub enum j2k_native::ColorSpace pub enum j2k_native::CpuDecodeParallelism pub enum j2k_native::EncodeProgressionOrder @@ -3173,6 +3631,8 @@ pub fn j2k_native::DecodedNativeComponents::has_alpha(&self) -> bool pub fn j2k_native::DecodedNativeComponents::planes(&self) -> &[j2k_native::NativeComponentPlane] pub fn j2k_native::DecoderContext<'_>::cpu_decode_parallelism(&self) -> j2k_native::CpuDecodeParallelism pub fn j2k_native::DecoderContext<'_>::default() -> Self +pub fn j2k_native::DecoderContext<'_>::from_workspace(j2k_native::DecoderWorkspace) -> Self +pub fn j2k_native::DecoderContext<'_>::into_workspace(self) -> j2k_native::DecoderWorkspace pub fn j2k_native::DecoderContext<'_>::set_cpu_decode_parallelism(&mut self, j2k_native::CpuDecodeParallelism) pub fn j2k_native::DecodingError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_native::DirectPlanUnsupportedReason::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -3284,6 +3744,8 @@ pub j2k_native::DirectPlanUnsupportedReason::ComponentUnitSampled pub j2k_native::DirectPlanUnsupportedReason::GrayscaleImageWithoutAlpha pub j2k_native::DirectPlanUnsupportedReason::GrayscaleSingleComponentCodestream pub j2k_native::DirectPlanUnsupportedReason::GrayscaleSingleTileCodestream +pub j2k_native::DirectPlanUnsupportedReason::RgbaFourComponentRgbCodestream +pub j2k_native::DirectPlanUnsupportedReason::RgbaRgbImageWithAlpha pub j2k_native::EncodeComponentPlane::data: &'a [u8] pub j2k_native::EncodeComponentPlane::x_rsiz: u8 pub j2k_native::EncodeComponentPlane::y_rsiz: u8 @@ -3438,6 +3900,8 @@ pub struct j2k_native::DecodeSettings pub struct j2k_native::DecodedComponents<'a> pub struct j2k_native::DecodedNativeComponents pub struct j2k_native::DecoderContext<'a> +pub struct j2k_native::DecoderWorkspace +pub struct j2k_native::DecoderWorkspaceStats pub struct j2k_native::EncodeComponentPlane<'a> pub struct j2k_native::EncodeOptions pub struct j2k_native::EncodeRoiRegion @@ -3462,9 +3926,13 @@ impl core::cmp::PartialEq for j2k_types::J2kEncodeStageError impl core::default::Default for j2k_types::IrreversibleQuantizationSubbandScales impl core::error::Error for j2k_types::J2kEncodeStageError impl core::fmt::Display for j2k_types::J2kEncodeStageError +impl j2k_types::J2kClassicCodeBlockPayload +impl j2k_types::J2kCodestreamRange impl j2k_types::J2kEncodeDispatchReport impl j2k_types::J2kEncodeStageError impl j2k_types::J2kPacketizationProgressionOrder +pub const fn j2k_types::J2kClassicCodeBlockPayload::end_range(self) -> core::option::Option +pub const fn j2k_types::J2kCodestreamRange::end(self) -> core::option::Option pub const fn j2k_types::J2kEncodeStageError::arithmetic_overflow(&'static str) -> Self pub const fn j2k_types::J2kEncodeStageError::host_allocation_failed(&'static str, usize) -> Self pub const fn j2k_types::J2kEncodeStageError::internal_invariant(&'static str) -> Self @@ -3512,12 +3980,17 @@ pub j2k_types::EncodedJ2kCodeBlock::data: alloc::vec::Vec pub j2k_types::EncodedJ2kCodeBlock::missing_bit_planes: u8 pub j2k_types::EncodedJ2kCodeBlock::number_of_coding_passes: u8 pub j2k_types::EncodedJ2kCodeBlock::segments: alloc::vec::Vec +pub j2k_types::HtCodeBlockPayloadRanges::cleanup: j2k_types::J2kCodestreamRange +pub j2k_types::HtCodeBlockPayloadRanges::refinement: core::option::Option pub j2k_types::IrreversibleQuantizationStep::exponent: u8 pub j2k_types::IrreversibleQuantizationStep::mantissa: u16 pub j2k_types::IrreversibleQuantizationSubbandScales::high_high: f32 pub j2k_types::IrreversibleQuantizationSubbandScales::high_low: f32 pub j2k_types::IrreversibleQuantizationSubbandScales::low_high: f32 pub j2k_types::IrreversibleQuantizationSubbandScales::low_low: f32 +pub j2k_types::J2kClassicCodeBlockPayload::combined_length: usize +pub j2k_types::J2kClassicCodeBlockPayload::first_range: usize +pub j2k_types::J2kClassicCodeBlockPayload::range_count: usize pub j2k_types::J2kCodeBlockSegment::data_length: u32 pub j2k_types::J2kCodeBlockSegment::data_offset: u32 pub j2k_types::J2kCodeBlockSegment::end_coding_pass: u8 @@ -3528,6 +4001,8 @@ pub j2k_types::J2kCodeBlockStyle::segmentation_symbols: bool pub j2k_types::J2kCodeBlockStyle::selective_arithmetic_coding_bypass: bool pub j2k_types::J2kCodeBlockStyle::termination_on_each_pass: bool pub j2k_types::J2kCodeBlockStyle::vertically_causal_context: bool +pub j2k_types::J2kCodestreamRange::length: usize +pub j2k_types::J2kCodestreamRange::offset: usize pub j2k_types::J2kDeinterleaveToF32Job::bit_depth: u8 pub j2k_types::J2kDeinterleaveToF32Job::num_components: u16 pub j2k_types::J2kDeinterleaveToF32Job::num_pixels: usize @@ -3761,10 +4236,13 @@ pub mod j2k_types pub struct j2k_types::CpuOnlyJ2kEncodeStageAccelerator pub struct j2k_types::EncodedHtJ2kCodeBlock pub struct j2k_types::EncodedJ2kCodeBlock +pub struct j2k_types::HtCodeBlockPayloadRanges pub struct j2k_types::IrreversibleQuantizationStep pub struct j2k_types::IrreversibleQuantizationSubbandScales +pub struct j2k_types::J2kClassicCodeBlockPayload pub struct j2k_types::J2kCodeBlockSegment pub struct j2k_types::J2kCodeBlockStyle +pub struct j2k_types::J2kCodestreamRange pub struct j2k_types::J2kDeinterleaveToF32Job<'a> pub struct j2k_types::J2kEncodeDispatchReport pub struct j2k_types::J2kForwardDwt53Job<'a> @@ -3881,6 +4359,11 @@ pub j2k_cuda_runtime::CudaError::InternalInvariant pub j2k_cuda_runtime::CudaError::InternalInvariant::what: &'static str pub j2k_cuda_runtime::CudaError::InvalidArgument pub j2k_cuda_runtime::CudaError::InvalidArgument::message: alloc::string::String +pub j2k_cuda_runtime::CudaError::KernelJobStatus +pub j2k_cuda_runtime::CudaError::KernelJobStatus::code: u32 +pub j2k_cuda_runtime::CudaError::KernelJobStatus::detail: u32 +pub j2k_cuda_runtime::CudaError::KernelJobStatus::job_index: usize +pub j2k_cuda_runtime::CudaError::KernelJobStatus::kernel: &'static str pub j2k_cuda_runtime::CudaError::KernelStatus pub j2k_cuda_runtime::CudaError::KernelStatus::code: u32 pub j2k_cuda_runtime::CudaError::KernelStatus::detail: u32 diff --git a/docs/unsafe-audit.md b/docs/unsafe-audit.md index 29e67063..6598b04f 100644 --- a/docs/unsafe-audit.md +++ b/docs/unsafe-audit.md @@ -12,6 +12,7 @@ moved, or removed. | `crates/j2k-core/src/accelerator.rs` | Shared GPU ABI marker trait and byte-view helpers for host/device struct transfers. | Implementers are plain-data GPU ABI values with stable layout and valid byte views. | Core API tests plus backend layout/parity tests for GPU ABI structs. | | `crates/j2k-core/src/backend.rs` | CPU feature detection and backend probing. | CPU intrinsics are called only behind matching architecture/configuration checks. | Cross-architecture CI matrix and backend feature tests. | | `crates/j2k-cuda-runtime/src/bytes/abi.rs` | Unsafe `GpuAbi` marker implementations for CUDA host/device records. | Every marked type is `repr(C)`, accepts every bit pattern, and has a compile-time field-offset walk proving its complete representation contains no padding; explicit reserved-tail fields are initialized and mirrored in device records without changing ABI sizes or offsets. | Safe byte-view tests, ABI size/offset tests, CUDA ABI repository policy, strict cuda-oxide builds, and runtime parity tests. | +| `crates/j2k-cuda-runtime/src/classic_decode/launch.rs` | Classic JPEG 2000 Tier-1 CUDA launch, raw kernel-parameter binding, and unsafe queued submission. | Preflight proves context identity, payload and segment bounds, code-block geometry, output ranges, target disjointness, and launch geometry before raw parameter binding. The caller retains payload, table, and coefficient-target owners; the returned guard retains copied jobs, segments, scratch, statuses, and the pool reuse hold until completion, and launch failure establishes completion before reuse. | Classic preflight boundary and adversarial tests, the deferred-status-copy regression, classic CUDA parity, registry structure policy, and strict GPU validation. | | `crates/j2k-cuda-runtime/src/context/creation.rs` | CUDA driver initialization, device discovery, owned-context creation, and primary-context retain cleanup. | Driver entry points receive correctly typed writable outputs; the ordinal is range-checked; owned and retained-primary handles have distinct exactly-once cleanup paths, including validation failure. | CUDA context identity/retain/release tests with real driver paths plus the strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/context/inner.rs` | Cached CUDA resource teardown, context destruction, and thread-safety marker implementations. | The context handle remains live while used; mutable caches are mutex-protected; poisoned contexts skip individual resource transitions and defer reclamation to context-wide destruction. | CUDA lifecycle/cache/drop tests, concurrent runtime tests, and the strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/context/kernel_cache.rs` | CUDA PTX module/function loading and cached opaque kernel handles. | PTX and symbol names are NUL-terminated, loaded handles belong to the owning context, the cache mutex coordinates handle sharing and eventual unload, and a lookup or rollback failure quarantines the context without exposing an uncertain module handle. | Pure module-lookup and failed-rollback regressions, CUDA kernel-cache/metadata tests, and the strict GPU validation workflow. | @@ -29,40 +30,75 @@ moved, or removed. | `crates/j2k-cuda-runtime/src/cuda_oxide_jpeg_decode/simt/src/main.rs` | cuda-oxide device baseline JPEG entropy decode and RGB8 output kernels for 4:2:0, 4:2:2, and 4:4:4 resident inputs. | Device pointers refer to live entropy, Huffman-table, checkpoint, output, and status buffers. Host preflight proves the exact sampling-derived MCU grid, a strict checkpoint partition from MCU zero, left-aligned bounded bit state, canonical role-correct tables, and a pitched last RGB byte within `u32`; the device repeats those bounds with checked arithmetic and stores a nonzero status on every defensive rejection before raw pointer reads or writes. | Pure adversarial plan/table/address tests, generated-kernel metadata and PTX assembly checks, exact-source 4:2:0/4:2:2/4:4:4 owned/caller-buffer parity on the NVIDIA runner, and GPU validation with `J2K_REQUIRE_CUDA_JPEG_HARDWARE_DECODE`. | | `crates/j2k-cuda-runtime/src/cuda_oxide_jpeg_encode/simt/src/main.rs` | cuda-oxide device baseline JPEG entropy encode kernels for single and same-buffer batch resident input tiles. | Host preflight validates input and entropy allocation ranges, disjoint batch outputs, sampling/MCU geometry, every kernel-side `u32` expression, nonzero quantizers, and canonical prefix-free non-all-ones Huffman codes before driver work. Device pointers then refer to live validated buffers, and status rows surface missing symbols or capacity exhaustion. | Pure adversarial layout/table/boundary tests, nonzero-offset single and batch round trips, generated-kernel metadata tests, and strict resident CUDA encode tests on a self-hosted runner. | | `crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/simt/src/main.rs` | cuda-oxide device J2K generic inverse-DWT kernels. | Before allocation or driver work, Rust validates coherent output/band rectangles, parity-derived band geometry, every input/output allocation, context ownership, alias rules, iteration bounds, and the maximum `u32` linear index. Device pointers then refer to live launch-bounded buffers retained through completion. | Pure full-job and adversarial geometry tests, queued ownership/alias tests, IDWT parity and metadata tests when cuda-oxide PTX is generated, plus strict GPU validation workflow. | -| `crates/j2k-cuda-runtime/src/cuda_oxide_j2k_ml/simt/src/main.rs` | cuda-oxide device interleaved-sample to Burn tensor conversion kernel. | Host preflight validates source/destination context identity, byte lengths, alignment, channel count, batch offset, normalization values, and launch size; each thread bounds-checks its source index and maps to one validated CHW/HWC destination element. | External-buffer bounds tests, direct CUDA U8/U16/F32 parity, batch/layout/normalization tests, and strict CUDA hardware validation. | +| `crates/j2k-cuda-runtime/src/cuda_oxide_j2k_ml/simt/src/main.rs` | Legacy codec-runtime interleaved-sample conversion kernel retained for compatibility; the thin `j2k-ml` batch adapter does not call it. | Host preflight validates source/destination context identity, byte lengths, alignment, channel count, batch offset, conversion parameters, and launch size; each thread bounds-checks its source index and maps to one validated CHW/HWC destination element. | Legacy external-buffer conversion bounds/parity tests and strict CUDA hardware validation; the `j2k-ml` source policy separately forbids this conversion path in the current adapter. | | `crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/exports.rs` | cuda-oxide device J2K encode-stage kernel entry points for deinterleave, color transforms, DWT, quantization, HTJ2K compaction, and cleanup packetization. | Device pointers refer to live, launch-bounded sample/tile, scratch, compact-output, packet-output, metadata, status, and job buffers; forward-DWT host preflight caps decomposition levels and proves the last live sample index fits `u32`; every entry point exits or strides inside validated launch/job bounds before raw pointer access. | Forward-DWT boundary/degenerate/index tests, J2K encode-stage parity, HTJ2K compaction fixtures, packetization parity, generated-PTX metadata tests, and the strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/packetization.rs` | Raw status and packet-header writes shared by cuda-oxide HTJ2K packetization kernels. | Callers provide a live status row and output span for the current launch-bounded packet job; header/body lengths are checked against capacity before writes and only block-owned state is mutated. | HTJ2K packetization scalar-parity and overflow/status tests, generated-PTX metadata tests, and the strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/cuda_oxide_transcode/simt/src/exports.rs` | cuda-oxide device reversible 5/3 and irreversible 9/7 transcode kernel entry points, including batch, code-block quantize, and fused column-lift+quantize stages. | Device pointers refer to live, launch-bounded DCT block, spatial sample, row-band, quantized code-block, and output band buffers; every kernel exits out-of-range threads before raw pointer access, and cooperative row-lift synchronization has no divergent early return before barriers. | Transcode parity and generated-PTX metadata tests plus the strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/driver.rs` | CUDA/NVTX driver FFI symbol loading and error-name conversion. | Loaded symbols match CUDA Driver API signatures and the driver library outlives function pointers. | CUDA runtime tests with real and fake driver paths. | | `crates/j2k-cuda-runtime/src/execution.rs` | CUDA kernel parameter ABI, kernel launch, checked device copy, and context synchronization. | Kernel parameter pointers remain valid through launch; buffers belong to the current context; launch, copy, and synchronization are lifecycle-gated; only successful synchronization establishes completion. | CUDA lifecycle/launch/parity tests, wrong-context regressions, and strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/execution/memory_ops.rs` | Bounds-checked CUDA device memset operations. | Destination buffers belong to the current context, the complete byte extent is validated before driver work, and the context lifecycle gate remains held through each memset call. | CUDA memory bounds, wrong-context, lifecycle, and strict GPU validation tests. | -| `crates/j2k-cuda-runtime/src/execution/events.rs` | CUDA stream/event RAII, event timing, NVTX-scoped default-stream completion, and the explicit unsafe submission-only helper. | Event and stream handles belong to the owning context; creation and destruction use stateful gates that quarantine on failure; every safe timing path establishes completion even when timing is disabled. Callers of `submit_default_stream_named` retain all reachable resources and prevent mutation/reuse/release until a real same-context completion point. | CUDA event/stream lifecycle, timing-disabled completion, submission-source policy, queued-resource tests, and the strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/execution/events.rs` | CUDA stream/event RAII, event timing, guarded external-stream ordering, NVTX-scoped default-stream completion, and the explicit unsafe submission-only helper. | Event and stream handles belong to the owning context; creation and destruction use stateful gates that quarantine on failure; every safe timing path establishes completion even when timing is disabled. The external-stream bridge accepts only a retained-primary context, confines the raw stream token to one call, orders caller-to-codec and codec-to-caller work with two same-context events, and requires the returned owner to retain every asynchronously reachable codec resource. Callers of `submit_default_stream_named` retain all reachable resources and prevent mutation/reuse/release until a real same-context completion point. | CUDA event/stream lifecycle, two-way primary-stream bridge ordering, no-context-sync source regression, timing-disabled completion, submission-source policy, queued-resource tests, and the strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/execution/events/handles.rs` | CUDA stream/event RAII handle operations and `Send` marker implementations. | Handles are created only after successful non-null validation, remain tied to their owning context, and are destroyed under that context's lifecycle gate; failed destruction retains the context rather than exposing uncertain resource ownership. | Successful-null creation regressions, event/stream lifecycle and wrong-context tests, and the strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/execution/events/handles/stream.rs` | CUDA stream destruction and the owned stream handle's `Send` marker implementation. | The stream was created by its retained owning context, destruction runs under that context's lifecycle gate, and a failed destroy retains the context so no uncertain driver resource is reclaimed. The handle owns no Rust memory alias. | Stream lifecycle, failed-destruction quarantine, cross-thread ownership, and strict CUDA validation tests. | +| `crates/j2k-cuda-runtime/src/execution/events/interop.rs` | Unsafe two-way ordering bridge between a retained primary CUDA context and an external runtime stream. | The raw stream belongs to the same retained primary context and its external managed owner remains exclusively borrowed for the call; events order external work before codec submission and codec work before later external consumption; submitted work returns a typed owner retaining every asynchronously reachable resource. | Same-context event-ordering tests, CubeCL direct-destination integration, wrong-context rejection, and strict CUDA validation. | | `crates/j2k-cuda-runtime/src/execution/queued.rs` | Queued CUDA completion guard and unsafe release after caller-certified completion. | Pooled resources remain owned behind a reuse hold until synchronization succeeds or an unsafe caller proves actual same-context completion; uncertain completion abandons the hold and retains resources. | Queued-resource structure policy, adapter completion tests, and strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/htj2k_decode/completion.rs` | Unsafe asynchronous HTJ2K cleanup enqueue with explicit queued-resource ownership. | Payload, table, target, and pool allocations belong to the launch context; target allocations and job write regions are validated as disjoint; all resources remain live and unavailable for reuse until status readback, context synchronization, or another proven completion point; failed completion attempts poison resource lifetimes instead of freeing uncertain allocations. | Queued-cleanup ownership/source-policy tests, CUDA runtime tests, adapter host-surface tests, and the strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_dequant_enqueue.rs` | Unsafe fused cleanup/dequantization enqueue with an explicit queued-resource owner. | Payloads, lookup tables, coefficient targets, and pool allocations are context-matched and pairwise validated before enqueue; the returned guard keeps them unavailable for mutation or reuse until a same-context completion point, and failed completion quarantines uncertain resources. | Fused-path async-mode source regression, HT cleanup/dequant parity, queued-resource lifecycle tests, and strict CUDA validation. | +| `crates/j2k-cuda-runtime/src/htj2k_decode/completion/cleanup_enqueue.rs` | Unsafe asynchronous HTJ2K cleanup enqueue, including group-owned status subranges. | Preflight validates context identity, payload bounds, pairwise-disjoint coefficient targets and job writes, pool ownership, and status-group bounds before launch. The caller retains payload, tables, coefficient targets, and any status group; the returned guard retains uploaded descriptors, owned statuses, and the pool reuse hold until a same-context completion point, while launch failure synchronizes before reuse. | Queued-completion and host-allocation ownership policies, grouped-status and indexed-error tests, HT cleanup/refinement parity, and strict NVIDIA validation. | | `crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant.rs` | Ordered asynchronous HTJ2K dequantization submission on the default stream. | The owning decode operation retains coefficient and job-metadata buffers until the ordered status readback establishes completion; synchronous timing mode completes the launch before returning. | HTJ2K dequantization parity and metadata tests, queued decode ownership tests, and the strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/htj2k_decode/completion/dequant/queued.rs` | Unsafe dequantization enqueue that consumes metadata retained by a queued HT cleanup guard. | The cleanup guard, every referenced coefficient target, and the owning pool remain live, context-matched, immutable, and unavailable for reuse until later same-context completion. | Queued cleanup/dequant ordering tests, lifecycle source policy, parity tests, and strict CUDA validation. | | `crates/j2k-cuda-runtime/src/jpeg/decode_launch.rs` | Validated CUDA JPEG decode launch orchestration and pointer-table kernel parameter ABI marker implementations. | Full plan/table/checkpoint/output validation and fallible status allocation precede launch; owned output is initialized before device work; each marked `repr(C)` value contains only device-pointer scalars and matches the pointer-only kernel structure. | Adversarial decode validation and ordering policy, kernel metadata/layout tests, initialized-output parity, and JPEG CUDA adapter tests. | | `crates/j2k-cuda-runtime/src/jpeg/types.rs` | CUDA JPEG encode/decode kernel parameter ABI marker implementations and opaque prepared decode tables. | Each marked `repr(C)` parameter value contains only CUDA scalar ABI fields, is passed by value through the kernel-parameter pointer, and matches its CUDA kernel signature. External safe code cannot forge the fixed Huffman representation fields and must use canonical construction before the full plan validator applies role-specific constraints. | CUDA runtime JPEG ABI/metadata, malformed-table validation, and JPEG CUDA adapter tests. | | `crates/j2k-cuda-runtime/src/j2k_decode/idwt.rs` | Unsafe asynchronous IDWT batch and sequence enqueue with pooled metadata retention. | Every band, output, and pool belongs to the launch context; full job geometry/allocation and alias validation completes before metadata upload; aggregation is checked and fallible; callers retain target allocations and exclude mutation/reuse until the queued handle is finished, dropped, or released after an actual context completion point. | Full-job boundary tests, IDWT ownership/source-policy tests, CUDA runtime tests, adapter host-surface tests, and the strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/j2k_decode/idwt/sequence.rs` | Unsafe multi-stage inverse-DWT enqueue with one pooled metadata upload. | All target buffers and the pool belong to the context and remain live, immutable, and unavailable for reuse until queued completion; same-stage outputs are disjoint from concurrent inputs, while validated dependencies may alias only across ordered stages. | Pure sequence context/alias regressions, two-stage runtime parity tests, queued-resource policy, and the strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/j2k_decode/idwt/tests.rs` | Test-only invocation of the unsafe queued IDWT sequence boundary. | The exercised jobs enqueue no device work, every buffer and pool belongs to the launch context, all resources remain live through `finish`, and the test establishes completion before release. | The ordered-stage alias regression itself plus the unsafe-audit path inventory. | +| `crates/j2k-cuda-runtime/src/j2k_decode/store/grayscale_batch.rs` | Batched asynchronous native `U8`/`U16`/`I16` grayscale final-store submission. | Host preflight validates every source job, output range, destination alignment, disjoint item offset, active count, and maximum pixel launch bound. Uploaded job descriptors and all source/destination allocations remain owned by the returned completion guard until its default-stream event finishes; an enqueue error establishes completion before resources are released. | Pure batch-store bounds/layout checks, signed/unsigned CUDA parity, queued completion/drop tests, and strict CUDA hardware validation. | +| `crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_batch.rs` | Batched asynchronous exact-native RGB `U8`/`U16` final-store submission. | Host preflight validates all three source planes, transform/layout selectors, native precision, full destination coverage, aggregate output extent, alignment, context identity, and launch bounds. The uploaded descriptor table and every source/destination owner remain live until the one group event retires; error recovery establishes completion or quarantines uncertain ownership. | ABI/layout checks, pure bounds tests, NVIDIA-gated exact RGB parity tests, and queued external-destination lifecycle tests. | +| `crates/j2k-cuda-runtime/src/j2k_decode/store/color_native_rgba_batch.rs` | Batched asynchronous exact-native RGBA `U8`/`U16`/`I16` final-store submission. | Host preflight validates all four source planes, native storage and layout selectors, full destination coverage, aggregate extent, alignment, launch bounds, and context identity. The uploaded descriptors and every source/destination owner remain live until the group event retires; uncertain owned completion retains the allocation and external completion requires caller quarantine. | ABI/layout and bounds tests, classic/HT exact RGBA parity for resident and external destinations, drop-safe completion tests, and strict CUDA hardware validation. | +| `crates/j2k-cuda-runtime/src/j2k_decode/store/batch.rs` | Owned batched RGB8 inverse-color final-store submission. | Every plane, output allocation, and uploaded descriptor buffer belongs to the active context and remains live through the immediate completion boundary; launch geometry and active-job counts are validated before enqueue. | Batch-store bounds/layout tests, MCT parity tests, and strict CUDA hardware validation. | +| `crates/j2k-cuda-runtime/src/j2k_decode/store/batch/external.rs` | Asynchronous RGB8 inverse-color final store into caller-owned CUDA storage. | The caller exclusively retains all source planes and the validated external destination until the returned completion guard retires. The guard owns uploaded descriptors, enqueue failures establish completion before release, and uncertain completion requires destination quarantine. | External-destination identity/bounds tests, queued completion/drop regressions, and strict CUDA hardware validation. | +| `crates/j2k-cuda-runtime/src/j2k_decode/store_launch.rs` | Unsafe enqueue-only launch wrappers for batched grayscale and exact-native RGB final-store kernels. | Callers have validated every pointer-bearing descriptor and launch bound, keep the uploaded job table plus all referenced input/output allocations live, and prevent access or reuse until a later same-context completion point. | Batch-store preflight tests, kernel metadata checks, completion-owner regressions, and strict CUDA hardware validation. | +| `crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native.rs` | Unsafe enqueue-only launch wrappers for exact-native RGB `U8`/`U16` kernels. | Callers validate the descriptor buffer, job count, maximum pixel extent, destination layout, and context identity, then retain every referenced allocation until a later same-context completion point. | Native-color preflight and ABI tests, kernel metadata checks, NVIDIA-gated parity tests, and queued completion regressions. | +| `crates/j2k-cuda-runtime/src/j2k_decode/store_launch/color_native_rgba.rs` | Unsafe enqueue-only launch wrappers for exact-native RGBA `U8`/`U16`/`I16` kernels. | Callers validate the pointer-bearing descriptor buffer, active job count, maximum pixel extent, destination layout, and context identity, then retain all four source planes, the destination, and uploaded descriptors until a same-context completion event retires. | RGBA preflight and ABI tests, kernel-symbol inventory checks, exact NVIDIA parity, and queued completion regressions. | | `crates/j2k-cuda-runtime/src/memory.rs` | CUDA device/pinned memory allocation, copies, downloads, typed buffer views, and the non-owning external-allocation bridge. | Device ranges and context ownership are checked before address comparison; the external view requires a lifetime-bound exclusive managed-owner borrow and validates non-null range, alignment, overflow, and actual driver context without assuming ownership; allocation/free transitions remain with the original owner. | CUDA lifecycle/memory tests, external-view foreign-context/non-ownership tests, wrong-context regressions, and strict CUDA hardware validation. | | `crates/j2k-cuda-runtime/src/memory/pinned_staging/operations/growth.rs` | Page-locked upload-staging allocation and transactional pool growth. | Requested lengths are capped and context-wide host headroom is reserved before allocation; successful CUDA allocation must return a non-null pointer; the operation gate serializes pool mutation; authority and active-pool accounting roll back transactionally, and failed or uncertain releases retain ownership instead of reusing or losing the allocation. | Context host-authority owner-first/pinned-first, reservation rollback, pinned-staging cap, checkout, retention, diagnostics, and real upload tests plus the strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/memory/pool.rs` | CUDA device-buffer pooling, deferred reuse, pooled uploads/downloads, and uninitialized host readback. | Pool ownership is context-checked; reuse holds defer recycling until completion; uncertain completion retains the pool and allocation ownership; host vector length is set only after an exact successful device copy. | CUDA pool/reuse tests, queued-resource structure policy, wrong-context regressions, and strict GPU validation workflow. | | `crates/j2k-cuda-runtime/src/memory/pool/readback.rs` | Fallible pooled-device readback into uninitialized `Vec` spare capacity. | Capacity is reserved before the copy, the destination spare slice has exactly the requested length, and `Vec::set_len` runs only after a successful device copy initialized that complete extent. | Pooled readback parity/error tests, allocation-failure coverage, and the unsafe-audit path inventory. | | `crates/j2k-cuda-runtime/src/tests.rs` | Test-only fake CUDA driver construction and external-buffer boundary invocation. | Fake FFI signatures match production driver types; real-device external views borrow live owned allocations exclusively and are dropped before the original owner is reused. | CUDA runtime unit tests, foreign-context rejection, batch-offset bounds, and non-ownership regressions. | +| `crates/j2k-cuda-runtime/src/tests/context_external.rs` | Test-only raw CUDA external-view construction plus primary-context identity and diagnostics validation. | Every raw pointer and byte length comes from a live allocation held under an exclusive mutable borrow; same-context views are dropped before the owner is reused, deliberately foreign contexts are rejected, and the view never assumes allocation ownership. Primary-context retains and releases remain balanced through their RAII owners. | Context identity and retain/release tests, external-view ownership and foreign-context rejection, diagnostics accounting, destination-offset bounds, and strict CUDA hardware validation. | +| `crates/j2k-cuda-runtime/src/tests/grayscale_external.rs` | Test-only construction and use of external CUDA destinations for native grayscale batch stores. | Each view exclusively borrows a live allocation; undersized and foreign-context destinations fail before launch; valid suballocations remain owned and inaccessible until synchronous completion or the returned pending owner retires. | Bounds, context identity, suballocation offset, exact store, and unsafe-API contract regressions on the CUDA lane. | | `crates/j2k-cuda-runtime/src/tests/pipeline.rs` | Test-only calls to asynchronous HTJ2K cleanup and IDWT enqueue APIs. | Every borrowed test allocation and session-private pool belongs to the launch context, remains live and unmodified while queued work is pending, and is retained through explicit synchronization or queued status completion. | Focused CUDA pipeline tests exercise queued IDWT batches, IDWT sequences, and empty HTJ2K cleanup resource ownership. | | `crates/j2k-cuda-runtime/src/transcode/types.rs` | CUDA transcode kernel parameter ABI marker implementation. | The marked parameter type is `repr(C)`, contains only CUDA scalar ABI fields, and matches the fused column-lift/quantize kernel signature. | CUDA transcode ABI/metadata tests and transform parity tests. | | `crates/j2k-compare/src/grok.rs` | Grok FFI comparison harness. | External decoder pointers are checked for null and output buffers are sized before copy. | Optional Grok/OpenJPEG parity tests. | | `crates/j2k-compare/src/openjpeg.rs` | OpenJPEG FFI comparison harness. | OpenJPEG stream/image lifetimes are paired with cleanup and component buffers are bounds-checked. | Optional Grok/OpenJPEG parity tests. | -| `crates/j2k-ml/src/cuda/interop.rs` | Sole Burn bridge construction of a lifetime-bound view over CubeCL-managed CUDA memory. | CubeCL's nonblocking allocation stream completes before J2K default-stream access; the managed-resource guard remains exclusively borrowed and live through validated current-context cuda-oxide conversion and completion; the external view never frees or escapes with Burn memory, and the tensor is registered only after completion. | CUDA direct U8/U16/F32, batch, normalization, default fusion, autodiff lifting, external-buffer ownership, and strict hardware tests. | -| `crates/j2k-ml/tests/cuda.rs` | Test-only adoption of a real CubeCL stream-ordered allocation through the J2K external-buffer boundary. | CubeCL allocation completion is established first; its managed-resource guard remains exclusively borrowed and live for the view; the driver validates pool provenance, device identity, and current-context accessibility. | The CubeCL allocation interop regression itself plus the strict named CUDA tensor suite. | +| `crates/j2k-ml/src/cuda/batch.rs` | Thin adapter call into the codec's asynchronous caller-owned CUDA destination. | The fresh Burn allocation remains uniquely borrowed during submission; `fill_batch_int_tensor` establishes both stream dependencies; and the returned pending owner retains the allocation and all codec scratch until explicit wait or drop-safe retirement. | CUDA prepared-batch API tests, pending-work retirement checks, direct integer parity on the NVIDIA lane, and the source-enforced zero-copy adapter contract. | +| `crates/j2k-ml/src/cuda/interop.rs` | Sole Burn bridge construction of a lifetime-bound view over CubeCL-managed CUDA memory. | The adapter obtains the context from its persistent codec session through an ordinal-checked interop accessor and retains no second authoritative context field; the stream token never escapes the bridge; the managed-resource guard remains exclusively borrowed while retained-primary-context identity, allocation extent, and alignment are validated; CUDA events order CubeCL allocation before the codec and later Burn consumption after the codec without a CPU wait; the external view never frees or escapes with Burn memory; and the tensor is registered only after both event dependencies and the completion owner exist. | CUDA exact `U8`/`U16`/`I16` batch parity, stream ordering, external-buffer ownership, source policy forbidding host transfers and synchronization, and strict hardware tests. | +| `crates/j2k-ml/src/metal/interop.rs` | Pairing Burn wgpu and codec sessions on one Metal device and exact command queue, plus adoption of a uniquely borrowed Burn-owned `MTLBuffer` as the codec destination. | Patched HAL accessors transfer exactly one Objective-C retain that `metal::Device`, `metal::CommandQueue`, or `metal::Buffer` adopts exactly once; the retained command queue is the same queue Burn uses, so producer and later consumer work are ordered without an event allocation, signal, or wait command; the CubeCL resource guard keeps the exact suballocation live; allocations explicitly own any four-byte tracker padding; the hidden wgpu tracker hook is called only for that validated range after the exact-queue dependency is registered; checked layout construction validates dtype, row/image strides, bounds, and device identity; no tensor alias is registered until codec submission completes. | Metal direct integer batch parity including odd allocation tails, exact-queue identity and zero-event diagnostics, suballocation bounds/offset tests, source policy forbidding decoded readback/upload, and strict Metal validation. | +| `crates/j2k-ml/tests/cuda.rs` | Test-only adoption of a real CubeCL stream-ordered allocation through the J2K external-buffer boundary. | The managed-resource guard remains exclusively borrowed and live for the view; the driver validates pool provenance, device identity, and current-context accessibility; event dependencies order the codec and CubeCL streams before the allocation is consumed. | The CubeCL allocation interop regression itself plus the strict named CUDA tensor suite. | +| `crates/j2k-cuda/src/batch.rs` | Asynchronous prepared grayscale batch submission into a validated caller-owned CUDA allocation. | The caller keeps the destination live and inaccessible from unordered streams; compressed arenas, descriptors, scratch, status, and final-store resources remain owned by `SubmittedCudaExternalBatch`; successful wait or event-ordered retirement precedes reuse, while drop safely establishes completion. | External destination bounds/identity tests, drop-before-wait reuse regression, signed/ROI/reduction batch parity, and strict CUDA hardware validation. | +| `crates/j2k-cuda/tests/batch_decoder_api.rs` | Test-only construction and asynchronous reuse of a caller-owned CUDA destination. | The owned allocation remains exclusively borrowed while each external view exists; no access overlaps submission; dropping or waiting the completion owner establishes completion before the same allocation is reused or downloaded. | The drop-safe pending-work and reusable-session regression itself on the strict CUDA hardware lane. | +| `crates/j2k-cuda/tests/batch_decoder_api/classic_native.rs` | Test-only integer byte views and external CUDA destinations for exact classic JPEG 2000 batches. | Byte views are formed only over initialized integer slices and copied immediately. Each CUDA allocation is fresh and exclusively borrowed by the external view, remains live and inaccessible while work is pending, and is read back only after successful completion. | Classic reversible and irreversible Gray/RGB native-type, request, layout, resident/external, and CPU-oracle parity on the strict CUDA lane. | +| `crates/j2k-cuda/tests/batch_decoder_api/refinement_overlap.rs` | Test-only external CUDA destination for an independent HTJ2K SigProp/MagRef overlap fixture. | The fresh allocation remains live and exclusively borrowed through submission; no host access occurs until the completion owner is waited, after which the exact validated range is downloaded. | The independent OpenHTJ2K refinement-overlap resident/external oracle regression on NVIDIA hardware. | +| `crates/j2k-cuda/tests/batch_decoder_api/rgba.rs` | Test-only external CUDA destinations for exact RGBA batch decoding. | Every allocation is fresh, correctly aligned, and exclusively retained while submitted codec work is pending; completion precedes download or reuse, and the CPU integer oracle fixes the expected byte order for each native type and layout. | Classic/HT RGBA `U8`/`U16`/`I16`, Full/ROI/reduction, NCHW/NHWC, resident/external parity on the strict CUDA lane. | +| `crates/j2k-cuda/tests/batch_decoder_api/session_soak.rs` | Test-only repeated external CUDA allocation handoff during the 1,000-batch reuse soak. | The one allocation remains live across the soak but is exclusively borrowed for each submission; every pending owner is waited before the next view or final readback, and pool diagnostics are observed only at completion boundaries. | The resident/external 1,000-batch stable-memory and reusable-session regression on NVIDIA hardware. | | `crates/j2k-cuda/src/decoder.rs` | Adapter-side unsafe release of queued CUDA pool holds after synchronous downstream completion. | Session-private pools remain confined to the owning context; IDWT target owners stay live until context synchronization or a synchronously completed MCT/store launch proves all preceding default-stream work complete. | CUDA decoder structure/accounting tests, queued-resource source policy, host-surface tests, and strict GPU validation. | +| `crates/j2k-cuda/src/decoder/grayscale_batch.rs` | Adapter submission of batched native grayscale final stores into owned or caller-owned destinations. | Every coefficient owner and final-store descriptor remains live behind the pending high-level owner; externally borrowed storage stays exclusive until wait or drop-safe retirement, and uncertain completion follows the public quarantine contract. | Gray `U8`/`U16`/`I16` full/ROI/reduction parity, external-allocation bounds tests, pending-drop reuse, and strict CUDA hardware validation. | +| `crates/j2k-cuda/src/decoder/color_batch/native_batch.rs` | Exact-native RGB batch completion orchestration and unsafe release of a queued final-store hold after proven same-stream completion. | The final store is ordered before cleanup and classic status readback on the same default stream, and the unsafe release occurs only when that readback establishes completion without an uncertain error. The pending owner retains store, IDWT, cleanup, classic, decoded-component, payload, and external-destination ownership until retirement; uncertain completion abandons rather than reuses reachable resources. | Classic and HT exact RGB full/ROI/reduced NCHW/NHWC resident/external parity, indexed status failures, pending-drop lifecycle tests, queued-completion policy, and strict NVIDIA validation. | +| `crates/j2k-cuda/src/decoder/color_batch/native_batch/store.rs` | Adapter submission of the exact RGB final store into an owned or externally borrowed destination. | The shared payload, component coefficients, IDWT/cleanup guards, uploaded final-store descriptors, and destination all share one ordered default-stream lifetime. External submission returns an owner that retires the store event before releasing earlier-stage resources; uncertain completion leaks/quarantines rather than reusing reachable allocations. | NVIDIA-gated exact RGB full/ROI/reduced parity, pending-drop reuse, resident/external allocation tests, and strict CUDA hardware validation. | | `crates/j2k-cuda/src/decoder/resident/cleanup_dequant.rs` | Adapter calls to the unsafe HTJ2K-cleanup enqueue API. | Component work owns every coefficient target, decode resources and session pools stay live and context-matched, and queued handles are retained through status completion, explicit synchronization, or synchronously completed dependent work. | Resident decode unit tests, queued-resource source policy, host-surface tests, and strict GPU validation. | +| `crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued.rs` | Adapter construction and unsafe submission of cross-image classic Tier-1 targets with indexed error mapping. | Source and component counts are checked before submission; component work retains disjoint coefficient destinations, while the surrounding batch owner retains decode payload and session table resources until the queued guard completes. The runtime guard owns copied descriptors, scratch, statuses, and pool holds, and device status indexes are translated through retained source identities. | Classic source-index mapping, runtime preflight and deferred-status tests, exact classic RGB parity, pending-completion lifecycle tests, focused-module policy, and strict CUDA validation. | +| `crates/j2k-cuda/src/decoder/resident/cleanup_dequant/enqueue.rs` | Adapter-side selection and chaining of unsafe cleanup, refinement, and dequantization enqueues. | Component work retains all coefficient targets; prepared payload/table resources and session pools remain context-matched and live; the queued guard survives through the ordered final store or another proven same-context completion point. | Cleanup-only/refinement route tests, queued ownership source policy, exact batch parity, and strict CUDA validation. | +| `crates/j2k-cuda/src/decoder/resident/chunked_cleanup.rs` | Adapter construction and unsafe submission of bounded cross-image HT cleanup/refinement chunks. | Selected jobs are sorted and coalesced so each coefficient band appears in one pairwise-disjoint runtime target; payload offsets and status identities are checked while flattening. Component owners retain every destination, and the submitted chunk retains uploaded payload, tables, descriptors, pool holds, and status mapping through ordered dequantization and completion. | Same-band coalescing and indexed-failure unit tests, independent refinement-overlap parity, signed/color batch regressions, and strict NVIDIA validation. | | `crates/j2k-cuda/src/decoder/resident/idwt.rs` | Adapter calls to unsafe IDWT enqueue APIs. | Component work owns every coefficient target, decode resources and session pools stay live and context-matched, and queued handles are retained through status completion, explicit synchronization, or synchronously completed dependent work. | Resident decode unit tests, queued-resource source policy, host-surface tests, and strict GPU validation. | +| `crates/j2k-cuda/tests/batch_decoder_api/signed_rgb.rs` | Test-only external allocation handoff for exact signed RGB batch decoding. | The CUDA allocation remains live and exclusively borrowed while the destination view exists; submitted work is waited before host readback or allocation reuse; every request/layout case compares the completed destination against the CPU integer oracle. | Independent OpenJPH signed 8/12/16-bit RGB Full/ROI/reduced parity and resident/external destination regressions on NVIDIA hardware. | | `crates/j2k-cuda/src/surface.rs` | CUDA surface batch downloads into preallocated host output buffers. | Batch range math is checked and host Vec length is set only after successful device copy. | CUDA surface and batch download tests. | +| `crates/j2k-metal/src/batch_decoder.rs` | Codec-owned/external Metal batch destination adoption and immutable resident-batch raw access. | Codec-owned destinations wrap fresh same-device buffers exclusively and remain retained by pending work; external destinations have validated layout, range, alignment, and device identity. Completed buffers are adopted only after producer completion, and raw resident access is read-only while an owner remains live and no writer overlaps it. | Gray/RGB/RGBA NCHW/NHWC resident and external parity, destination bounds/device tests, continuation/drop-safe session tests, and strict Metal hardware validation. | | `crates/j2k-metal/src/compute/direct_buffers.rs` | MetalDirect host slice buffer uploads and reusable shared-buffer zeroing. | Typed host slices are copied into Metal-owned buffers with validated byte lengths, and reusable buffers are allocated to at least the requested byte count before writes. | MetalDirect decode/encode and resident batch tests. | | `crates/j2k-metal/src/compute/abi.rs` | Unsafe `GpuAbi` marker implementations for Metal compute parameter and status structs. | Every marked type is `repr(C)`, `Copy`, contains only numeric scalar/array fields, accepts every bit pattern, and is mirrored by the Metal shader declaration. | Exact size/alignment/offset tests cover host-read status layouts; Metal compilation checks shader sources and route-specific runtime parity exercises parameter and job layouts. | +| `crates/j2k-metal/src/compute/decode_dispatch/store/destination.rs` | Unsafe raw Metal destination-buffer binding for the native integer final-store kernels. | `MetalImageDestination` has validated device identity, exact allocation range, pixel format, pitch, item stride, alignment, and bounds; the synchronous or submitted decode-into owner retains exclusive destination access through command completion; the raw handle is confined to one encoder binding. | Metal destination layout/bounds/suballocation regressions, exact native integer batch parity, and strict Metal validation. | +| `crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination/store.rs` | Binding codec-owned or caller-owned Metal buffers for the exact-native RGB/RGBA group store. | Resident buffers are freshly allocated and exclusively wrapped; external buffers have already passed device, range, pitch, image-stride, layout, and pixel-format validation. The raw handle is confined to one encoder binding, and all planes, descriptors, scratch, destination ownership, and the selected completion dependency remain retained by the submitted group. | Exact RGB/RGBA NCHW/NHWC parity, shared-allocation resident views, destination offset/alignment tests, pending-drop reuse, and strict Metal hardware validation. | +| `crates/j2k-metal/src/compute/direct_grayscale_execute/destination/submission.rs` | Metal producer/consumer ordering and completion-resource retirement for external destinations. | Exact producer/consumer queues commit with no event bridge. Different queues on the same device share one session-owned `MTLEvent` timeline; value assignment, producer signal, consumer wait encoding, and both commits are serialized under the timeline mutex so values cannot commit out of order. A different device is rejected before producer commit. The legacy post-submission `enqueue_consumer_wait` path retains its `MTLSharedEvent` compatibility bridge. Every route retains destination, scratch, statuses, command buffers, and event owners until host completion and codec-status validation or drop-safe retirement. | Exact-queue zero-event, same-device cross-queue ordering and timeline reuse, concurrent cloned-session ordering, different-device rejection, legacy deferred bridge, pending-drop reuse, and strict Metal hardware validation. | +| `crates/j2k-metal/src/compute/direct_grayscale_execute/destination/group_encode.rs` | Raw caller-owned Metal buffer binding for the coalesced exact-native grayscale group store. | The destination has passed same-device, dense batch geometry, byte-range, stride, alignment, and pixel-format validation; its exclusive owner and all input/scratch resources remain retained until the submitted command buffer completes or is safely retired. The raw handle is confined to the final-store encoder binding. | Stacked Gray U8/U16/I16 parity, destination bounds and suballocation tests, indexed status mapping, pending-drop session reuse, the 1,000-batch retention soak, and strict Metal hardware validation. | | `crates/j2k-metal/src/compute/direct_surface_pack.rs` | Checked CPU-side upload of staged floating-point plane data into a Metal buffer. | The checked write validates byte bounds and alignment, and occurs during plan preparation before the buffer is bound to or submitted in a command buffer. | MetalDirect surface-pack, decode, and encode tests. | | `crates/j2k-metal/src/compute/direct_flattened.rs` | Flattened hybrid CPU-tier1 worker output pointers. | Work items are built from non-overlapping packed coefficient ranges, and each pointer is converted back to a mutable slice with the validated per-item output length before a single decode write. | MetalDirect hybrid batch and flattened CPU-tier1 tests. | | `crates/j2k-metal/src/compute/gpu_timing.rs` | Metal command-buffer GPU timestamp queries through Objective-C messages. | Timestamps are queried only for completed command buffers, duplicate buffers are ignored, and non-finite or inverted timestamp pairs are discarded. | Resident encode/decode profiling tests and GPU validation workflow. | @@ -76,6 +112,8 @@ moved, or removed. | `crates/j2k-metal/src/surface.rs` | Unsafe raw Metal surface handle access and owned single-surface downloads. | Raw-handle callers complete prior writers and exclude aliased mutation; safe download checks offset/length and copies only after decode completion, so no safe borrowed host slice escapes. | Metal surface download, raw-handle device tests, device-output tests, and decode route tests. | | `crates/j2k-metal/src/surface/readback.rs` | One-allocation readback of a completed Metal surface batch. | Every surface is Metal-resident on the supplied device, source/destination offsets and aggregate size are checked, the single blit command completes before the staging buffer is copied, and no borrowed host view escapes. | `j2k-ml` Metal batch parity, route, and packed-transfer instrumentation tests plus strict Metal validation. | | `crates/j2k-metal/examples/resident_encode_buffer.rs` | Example construction of borrowed resident Metal encode tiles. | Buffers come from the same demonstrated session, are initialized before construction, remain immutable, and outlive the awaited batch. | Example/bench build gates and strict Clippy. | +| `crates/j2k-metal/tests/batch_classic_color.rs` | Test-only external Metal destination adoption and completed readback for classic exact-native RGB batches. | Each shared buffer and selected subrange is fresh and exclusively owned by the pending codec submission; the test waits for completion before the checked raw read, and retains no writable alias or later GPU writer. | Classic signed/unsigned RGB, reversible/irreversible, request/layout, offset, and CPU-oracle parity on the Metal release lane. | +| `crates/j2k-metal/tests/batch_rgba.rs` | Test-only external destination adoption and immutable resident-buffer readback for exact RGBA batches. | Fresh external ranges remain exclusively retained until completion; codec-owned resident groups are exposed only after successful producer completion. Checked raw reads cover validated ranges and occur without any concurrent CPU/GPU writer or escaped handle. | Classic/HT RGBA native-type/request/layout external and NCHW resident parity on the Metal release lane. | | `crates/j2k-metal/tests/device.rs` | Raw Metal surface/codestream constructor and consuming-handoff regression tests. | Test allocations have completed or have no writers; raw ranges are valid; tests retain no concurrently mutating alias. | The device integration test target on the Metal release lane. | | `crates/j2k-metal/tests/encode_auto_routing_benchmark.rs` | Benchmark-only borrowed resident Metal input tiles. | Inputs are allocated by the measured session, initialized before timing, immutable during both batches, and retained through completion. | Benchmark build gate and the explicit resident-codestream performance guard. | | `crates/j2k-jpeg-metal/benches/encode_baseline.rs` | Benchmark-only borrowed Metal JPEG encode tiles. | Shared buffers are initialized before descriptor construction, described ranges are disjoint where batched, and remain immutable and live through synchronous encode completion. | Benchmark build gate and JPEG Metal encode benchmark execution. | @@ -108,6 +146,7 @@ moved, or removed. | `crates/j2k-metal-support/src/buffer_access.rs` | Raw CPU access to shared Metal-buffer contents. | Typed ranges require padding-free `GpuAbi`, checked byte arithmetic, alignment, bounds, and CPU-visible storage; callers must prove command completion and exclusive access. | Checked read/write/fill regressions, nil/range/alignment tests, adapter alias-order policies, and real Metal tests. | | `crates/j2k-metal-support/src/pipeline.rs` | Raw Objective-C construction and error extraction for compile options, shader libraries, and compute pipelines. | Selector ABIs are fixed, Objective-C errors are copied into owned Rust text, and nil is rejected before any foreign owning handle is formed. | Injected-nil compile-option tests, shader/pipeline compile tests, and strict all-target Clippy. | | `crates/j2k-metal-support/src/resident.rs` | Opaque immutable Metal image ownership, unsafe raw adoption/access, and completion-retaining image submissions. | Layout construction and adoption validate dimensions, pitch, overflow, allocation bounds, and device identity; safe clones and subviews expose no mutable handle; producers transfer exclusive outputs and every raw-bound input into a submission that retains them until successful completion or a blocking drop. | Layout/bounds/device regressions, input-retention and drop-before-wait tests, resident JPEG/J2K encode parity, unsafe API snapshots, and strict Metal validation. | +| `crates/j2k-metal-support/src/resident/destination.rs` | Checked adoption and raw binding of an exclusively writable caller-owned Metal image subrange. | The unsafe constructor requires the external framework to honor exclusive GPU writes; it validates allocation bounds, pixel layout, strides, alignment, and device identity and retains the buffer. The raw handle is exposed only to an audited encoder while the destination owner remains live through completion. | Destination layout/bounds/alignment/device tests, Burn suballocation interop tests, exact native batch parity, and strict Metal validation. | | `crates/j2k-metal-support/src/runtime.rs` | Raw Objective-C construction of command queues, command buffers, and compute/blit encoders. | Each raw result is checked for nil before a foreign handle is formed; non-owning command/encoder results are retained into owned Rust handles before the creation autorelease pool can drain; completion status is validated before safe readback or reuse. | Injected-nil queue/buffer/encoder tests, an autorelease-pool escape regression, real command-resource tests, completion regressions, and strict all-target Clippy. | | `crates/j2k-metal-support/src/tests.rs` | Test-only padding-free ABI marker, injected nil pointers, and raw checked buffer-access calls. | The synthetic ABI type intentionally proves zero-sized rejection; nil pointers are never dereferenced or wrapped; real buffers are completed or never submitted and exclusively accessed during each checked read/write. | The Metal support tests themselves plus the unsafe-audit path inventory. | | `crates/j2k-metal-support/src/tests/resident.rs` | Test-only resident-image adoption, raw read binding, completed readback, and submission construction. | Adopted inputs are synchronously initialized without writable aliases; outputs are fresh and exclusively written by their retained command; raw input bindings are included in submission keepalives; readback occurs only after successful completion. | Layout/bounds/device, input-retention, immutable output, and drop-before-wait regressions on the Metal release lane. | diff --git a/engineering/public-api-review-0.7.4.yml b/engineering/public-api-review-0.7.4.yml index 27f65b77..a820b52d 100644 --- a/engineering/public-api-review-0.7.4.yml +++ b/engineering/public-api-review-0.7.4.yml @@ -5,18 +5,18 @@ candidate_version: "0.7.4" reviews: j2k: removed_fingerprint: "fnv1a64:2bb387c8d09725f1" - added_fingerprint: "none" - hidden_count: 100 - hidden_fingerprint: "fnv1a64:41827cac381efc6f" - rationale: "Reviewed the intentional removal of the pass-through j2k::DecoderContext re-export; callers now construct and pass J2kContext directly as documented in the changelog." - hidden_rationale: "Reviewed the complete hidden j2k inventory: batch decode implementations and in-context facade functions now accept J2kContext directly, with codec behavior and the remaining context contract unchanged." + added_fingerprint: "fnv1a64:44c04bf4ec370245" + hidden_count: 103 + hidden_fingerprint: "fnv1a64:2439f331dd8532b9" + rationale: "Reviewed the existing intentional DecoderContext re-export removal and the additive owned-batch contract: Arc-backed inputs, prepared grouping, exact native sample types, effective wavelet-transform metadata, indexed failures, retained plans, CPU sessions, and workspace diagnostics. No additional baseline item is removed or changed." + hidden_rationale: "Reviewed the complete hidden j2k inventory, including the immutable adapter-view borrows used to downcast retained classic and HT plans and BatchGroupInfo::native_pixel_format, which validates native Gray/RGB/RGBA U8/U16/I16 metadata for codec and framework adapters without exposing ownership or raw handles; the existing direct J2kContext migration remains unchanged." j2k-core: removed_fingerprint: "fnv1a64:e3cc7ab60eb765cd" - added_fingerprint: "fnv1a64:166af8cd2f5d3fbd" - hidden_count: 141 - hidden_fingerprint: "fnv1a64:d4ca6ced92ff6406" - rationale: "Reviewed the complete core contraction: the generic DecoderContext forwarding owner is removed, every TileBatchDecode context parameter uses the concrete associated Context, and the ordered collector requires explicit aggregate and collection limits." - hidden_rationale: "Reviewed the complete hidden j2k-core inventory, including removal of the legacy ordered collector while its bounded with-limits replacement remains available and direct associated-context signatures preserve the decode trait behavior." + added_fingerprint: "fnv1a64:d41e22eaacbb6b8c" + hidden_count: 189 + hidden_fingerprint: "fnv1a64:b87aa26da547e491" + rationale: "Reviewed the existing associated-context contraction together with additive signed i16 Sample/SampleType, Gray/RGB/RGBA pixel formats, PixelLayout channel accounting, and typed unsupported-contract errors required for exact medical batch output. No additional baseline item is removed or changed." + hidden_rationale: "Reviewed the complete hidden j2k-core inventory, including the bounded HT GPU job chunk planner, pass buckets, source-preserving entries and typed limit errors used by both accelerators; the existing associated-context migration remains unchanged." j2k-codec-math: removed_fingerprint: "none" added_fingerprint: "none" @@ -46,11 +46,11 @@ reviews: hidden_rationale: "Reviewed the complete hidden JPEG Metal inventory: batch and submit paths accept j2k_jpeg::DecoderContext directly, while resident-image ownership, session lifetime, and typed Metal error contracts remain unchanged." j2k-metal: removed_fingerprint: "none" - added_fingerprint: "none" - hidden_count: 287 - hidden_fingerprint: "fnv1a64:8ebd9cdd986dadbf" - rationale: "Reviewed the ordinary j2k-metal inventory against 0.7.3; no documented public API items changed." - hidden_rationale: "Reviewed the complete hidden J2K Metal inventory: device batch and submit methods use the concrete associated context directly, with resident ownership, session lifecycle, fallback, and typed error semantics unchanged." + added_fingerprint: "fnv1a64:2b83a651e326c970" + hidden_count: 299 + hidden_fingerprint: "fnv1a64:a496e2469837d3a8" + rationale: "Reviewed the additive persistent Metal batch API: prepared and one-shot sessions, resident groups, validated external destinations, asynchronous submissions, completion metadata, indexed group errors, exact caller-supplied command-queue construction, and known-consumer-queue submission that avoids a bridge on the producer queue and installs same-device event ordering otherwise. No baseline item is removed or changed." + hidden_rationale: "Reviewed the complete hidden J2K Metal inventory, including the reachable MetalSubmission associated type, session-reusing direct decode helpers, DeviceSubmission owners, exact-command-queue identity, and session-unusable classification used to quarantine retained state; buffer ownership, drop-safe completion, and typed Metal failures remain explicit." j2k-jpeg-cuda: removed_fingerprint: "none" added_fingerprint: "none" @@ -60,11 +60,11 @@ reviews: hidden_rationale: "Reviewed the complete hidden JPEG CUDA inventory: batch and submit methods receive the associated concrete context directly, while session-bound execution, allocation accounting, and typed CUDA failures remain unchanged." j2k-cuda: removed_fingerprint: "none" - added_fingerprint: "none" - hidden_count: 171 - hidden_fingerprint: "fnv1a64:56bd9f23e4d2c9fb" - rationale: "Reviewed the ordinary j2k-cuda inventory against 0.7.3; no documented public API items changed." - hidden_rationale: "Reviewed the complete hidden J2K CUDA inventory: device batch and submit methods use the concrete associated context directly, while resident decode, encode stages, session lifecycle, and typed runtime failures remain unchanged." + added_fingerprint: "fnv1a64:4c809347b1ded1da" + hidden_count: 181 + hidden_fingerprint: "fnv1a64:6bb25d5241c4e86a" + rationale: "Reviewed the additive persistent CUDA batch API: prepared decode, required dense resident output ownership, external destinations, asynchronous completion, source-indexed group failures, pool/runtime diagnostics, exact destination ranges, and session reuse. No baseline item is removed or changed." + hidden_rationale: "Reviewed the complete hidden J2K CUDA inventory, including completion-uncertainty and session-usability classification, pool-diagnostic conversion, the session-owned same-device interop context accessor, and the guarded external-device batch bridge; device identity, event ordering, and typed runtime failures remain explicit." j2k-transcode: removed_fingerprint: "none" added_fingerprint: "none" @@ -81,10 +81,10 @@ reviews: hidden_rationale: "Reviewed the complete hidden CUDA transcode inventory; accelerator routing, resident handoff, allocation caps, and source-preserving runtime errors are unchanged from 0.7.3." j2k-metal-support: removed_fingerprint: "none" - added_fingerprint: "none" + added_fingerprint: "fnv1a64:759663efe6a53439" hidden_count: 11 hidden_fingerprint: "fnv1a64:671e50d4d121b419" - rationale: "Reviewed the ordinary j2k-metal-support inventory against 0.7.3; no public API items changed." + rationale: "Reviewed the additive MetalImageDestination contract: exclusive retained buffers, batch suballocation layout, device identity, bounds and pixel-format validation, and explicitly unsafe raw-buffer access. No baseline item is removed or changed." hidden_rationale: "Reviewed the complete hidden Metal support inventory; checked allocation, command, completion, buffer ownership, and audited unsafe boundaries are unchanged from the published baseline." j2k-transcode-metal: removed_fingerprint: "none" @@ -95,25 +95,25 @@ reviews: hidden_rationale: "Reviewed the complete hidden Metal transcode inventory; transform, buffer, code-block, resident handoff, fallback, and typed source-chain contracts are unchanged from 0.7.3." j2k-native: removed_fingerprint: "none" - added_fingerprint: "none" - hidden_count: 535 - hidden_fingerprint: "fnv1a64:7c931c0e7c4eb27b" - rationale: "Reviewed the ordinary j2k-native inventory against 0.7.3; no public API items changed." - hidden_rationale: "Reviewed the complete hidden native inventory; decode, encode, resident, allocation-accounting, and typed validation and invariant boundaries are unchanged from the published baseline." + added_fingerprint: "fnv1a64:4313e3ca2306917e" + hidden_count: 661 + hidden_fingerprint: "fnv1a64:14c8dc9c142b3008" + rationale: "Reviewed the additive reusable DecoderWorkspace and statistics surface plus explicit RGBA direct-plan rejection reasons. These expose retained CPU ownership and observability without removing or changing any baseline item." + hidden_rationale: "Reviewed the complete hidden native inventory, including per-tile referenced classic/HT/RGBA plans, offset payload descriptors, staged entropy and IDWT execution, decoded-plane borrows, and retained-allocation metrics used by the facade and device adapters." j2k-types: removed_fingerprint: "none" - added_fingerprint: "none" + added_fingerprint: "fnv1a64:cec0bd880fac46b9" hidden_count: 54 hidden_fingerprint: "fnv1a64:4b30413a7bfffeb5" - rationale: "Reviewed the ordinary j2k-types inventory against 0.7.3; its documented public API is unchanged." + rationale: "Reviewed the additive borrowed-codestream payload descriptors: checked range ends, classic range spans, and separate HT cleanup/refinement ranges. These preserve source offsets without duplicating compressed payload bytes, and no baseline item is removed or changed." hidden_rationale: "Reviewed the complete hidden j2k-types inventory: five forwarding resident-job accessors are removed, and callers use the existing public job.input methods for width, height, component count, bit depth, and signedness." j2k-cuda-runtime: removed_fingerprint: "none" - added_fingerprint: "none" - hidden_count: 918 - hidden_fingerprint: "fnv1a64:61e8d94589aa373a" - rationale: "Reviewed the ordinary j2k-cuda-runtime inventory against 0.7.3; no public API items changed." - hidden_rationale: "Reviewed the complete hidden CUDA runtime inventory; asynchronous cleanup, kernel ABI, pooled allocation, execution, and lifecycle-sensitive unsafe boundaries are unchanged from the published baseline." + added_fingerprint: "fnv1a64:9cec81a67572ffb6" + hidden_count: 1089 + hidden_fingerprint: "fnv1a64:d33ffdcb13c73dd5" + rationale: "Reviewed the additive source-preserving KernelJobStatus error variant and its kernel, job index, status code, and detail fields. It adds batched device failure attribution without removing or changing a baseline item." + hidden_rationale: "Reviewed the complete hidden CUDA runtime inventory, including queued classic/HT/store owners, shared group status, native Gray/RGB/RGBA store ABIs, guarded external writes, stream ordering, event/context diagnostics, and drop-safe completion boundaries." j2k-profile: removed_fingerprint: "none" added_fingerprint: "none" diff --git a/engineering/reviewed-public-api-diff-0.7.4.md b/engineering/reviewed-public-api-diff-0.7.4.md index 4f51de59..91d9f9c1 100644 --- a/engineering/reviewed-public-api-diff-0.7.4.md +++ b/engineering/reviewed-public-api-diff-0.7.4.md @@ -12,29 +12,29 @@ This report is generated by `cargo xtask semver --write-report`. Normal `cargo x | Package | Baseline | Candidate | Computed release type | Added | Removed/changed | Removed fingerprint | Added fingerprint | Rustdoc-hidden items | Hidden inventory fingerprint | | --- | --- | --- | --- | ---: | ---: | --- | --- | ---: | --- | -| `j2k` | `0.7.3` | `0.7.4` | `minor` | 0 | 1 | `fnv1a64:2bb387c8d09725f1` | `none` | 100 | `fnv1a64:41827cac381efc6f` | -| `j2k-core` | `0.7.3` | `0.7.4` | `minor` | 13 | 21 | `fnv1a64:e3cc7ab60eb765cd` | `fnv1a64:166af8cd2f5d3fbd` | 141 | `fnv1a64:d4ca6ced92ff6406` | +| `j2k` | `0.7.3` | `0.7.4` | `minor` | 221 | 1 | `fnv1a64:2bb387c8d09725f1` | `fnv1a64:44c04bf4ec370245` | 103 | `fnv1a64:2439f331dd8532b9` | +| `j2k-core` | `0.7.3` | `0.7.4` | `minor` | 24 | 21 | `fnv1a64:e3cc7ab60eb765cd` | `fnv1a64:d41e22eaacbb6b8c` | 189 | `fnv1a64:b87aa26da547e491` | | `j2k-codec-math` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | | `j2k-jpeg` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 550 | `fnv1a64:d2bd76732f32e066` | | `j2k-tilecodec` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 32 | `fnv1a64:eb677842559d9b0e` | | `j2k-jpeg-metal` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 83 | `fnv1a64:8c83730972447ab6` | -| `j2k-metal` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 287 | `fnv1a64:8ebd9cdd986dadbf` | +| `j2k-metal` | `0.7.3` | `0.7.4` | `minor` | 91 | 0 | `none` | `fnv1a64:2b83a651e326c970` | 299 | `fnv1a64:a496e2469837d3a8` | | `j2k-jpeg-cuda` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 124 | `fnv1a64:0b31ccb3e7974bc4` | -| `j2k-cuda` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 171 | `fnv1a64:56bd9f23e4d2c9fb` | +| `j2k-cuda` | `0.7.3` | `0.7.4` | `minor` | 106 | 0 | `none` | `fnv1a64:4c809347b1ded1da` | 181 | `fnv1a64:6bb25d5241c4e86a` | | `j2k-transcode` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 455 | `fnv1a64:fdd61f6fdda4a9ab` | | `j2k-transcode-cuda` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 17 | `fnv1a64:3e3e69d35c4a62a5` | -| `j2k-metal-support` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 11 | `fnv1a64:671e50d4d121b419` | +| `j2k-metal-support` | `0.7.3` | `0.7.4` | `minor` | 16 | 0 | `none` | `fnv1a64:759663efe6a53439` | 11 | `fnv1a64:671e50d4d121b419` | | `j2k-transcode-metal` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 49 | `fnv1a64:5bf8a22a33753a2d` | -| `j2k-native` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 535 | `fnv1a64:7c931c0e7c4eb27b` | -| `j2k-types` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 54 | `fnv1a64:4b30413a7bfffeb5` | -| `j2k-cuda-runtime` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 918 | `fnv1a64:61e8d94589aa373a` | +| `j2k-native` | `0.7.3` | `0.7.4` | `minor` | 19 | 0 | `none` | `fnv1a64:4313e3ca2306917e` | 661 | `fnv1a64:14c8dc9c142b3008` | +| `j2k-types` | `0.7.3` | `0.7.4` | `minor` | 14 | 0 | `none` | `fnv1a64:cec0bd880fac46b9` | 54 | `fnv1a64:4b30413a7bfffeb5` | +| `j2k-cuda-runtime` | `0.7.3` | `0.7.4` | `minor` | 5 | 0 | `none` | `fnv1a64:9cec81a67572ffb6` | 1089 | `fnv1a64:d33ffdcb13c73dd5` | | `j2k-profile` | `0.7.3` | `0.7.4` | `minor` | 0 | 0 | `none` | `none` | 117 | `fnv1a64:a1546f56ae38f2e3` | ## Published-package details ### `j2k` -Baseline items: 541. Candidate items: 540. Computed release type: `minor`. Rustdoc-hidden candidate items: 100. Full hidden-inventory fingerprint: `fnv1a64:41827cac381efc6f`. +Baseline items: 541. Candidate items: 761. Computed release type: `minor`. Rustdoc-hidden candidate items: 103. Full hidden-inventory fingerprint: `fnv1a64:2439f331dd8532b9`. #### Removed or changed baseline API items @@ -44,11 +44,233 @@ pub use j2k::DecoderContext #### Added candidate API items -None. +```text +#[non_exhaustive] pub enum j2k::BatchAlpha +#[non_exhaustive] pub enum j2k::BatchCodecRoute +#[non_exhaustive] pub enum j2k::BatchErrorStage +#[non_exhaustive] pub enum j2k::BatchItemError +#[non_exhaustive] pub enum j2k::BatchLayout +#[non_exhaustive] pub enum j2k::BatchWaveletTransform +#[non_exhaustive] pub enum j2k::CpuBatchSamples +#[non_exhaustive] pub enum j2k::DecodeRequest +#[non_exhaustive] pub enum j2k::NonRepresentableReason +#[non_exhaustive] pub enum j2k::PreparationDepth +impl core::default::Default for j2k::BatchDecodeOptions +impl core::default::Default for j2k::DecodeSettings +impl core::fmt::Debug for j2k::CpuBatchDecoder +impl core::fmt::Display for j2k::BatchErrorStage +impl core::fmt::Display for j2k::NonRepresentableReason +impl j2k::BatchDecoder for j2k::CpuBatchDecoder +impl j2k::BatchGroupInfo +impl j2k::CpuBatchDecodeResult +impl j2k::CpuBatchDecoder +impl j2k::CpuBatchGroup +impl j2k::CpuBatchSamples +impl j2k::CpuBatchWorkspaceStats +impl j2k::DecodeSettings +impl j2k::EncodedImage +impl j2k::PreparedBatch +impl j2k::PreparedBatchGroup +impl j2k::PreparedClassicPlan +impl j2k::PreparedHtj2kPlan +impl j2k::PreparedImage +pub const fn j2k::CpuBatchDecoder::options(&self) -> j2k::BatchDecodeOptions +pub const fn j2k::CpuBatchGroup::samples(&self) -> &j2k::CpuBatchSamples +pub const fn j2k::CpuBatchSamples::sample_type(&self) -> j2k_core::sample::SampleType +pub const fn j2k::CpuBatchWorkspaceStats::component_owner_reuses(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::compressed_arena_reuses(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::cross_image_entropy_windows(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::decode_calls(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::entropy_job_dispatches(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_classic_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_cleanup_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_group_plans(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_magref_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_payload_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::flattened_sigprop_jobs(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::idwt_owner_reuses(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::output_compaction_copied_samples(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::output_group_allocations(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::preparation_calls(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::preparation_worker_reuses(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::prepared_plan_decode_calls(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::retained_component_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_compressed_arena_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_idwt_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_prepared_plan_classic_workspace_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_prepared_plan_ht_workspace_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_scratch_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::retained_tier1_bytes(self) -> usize +pub const fn j2k::CpuBatchWorkspaceStats::scratch_capacity_retries(self) -> u64 +pub const fn j2k::CpuBatchWorkspaceStats::tier1_owner_reuses(self) -> u64 +pub const fn j2k::DecodeSettings::is_strict(self) -> bool +pub const fn j2k::DecodeSettings::lenient() -> Self +pub const fn j2k::DecodeSettings::lenient_tolerance_enabled(self) -> bool +pub const fn j2k::DecodeSettings::strict() -> Self +pub const fn j2k::PreparedBatch::options(&self) -> j2k::BatchDecodeOptions +pub const fn j2k::PreparedBatchGroup::options(&self) -> j2k::BatchDecodeOptions +pub fn j2k::BatchDecodeOptions::default() -> Self +pub fn j2k::BatchDecoder::decode_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::BatchDecoder::decode_prepared(&mut self, &j2k::PreparedBatch) -> core::result::Result +pub fn j2k::BatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::BatchDecoder::options(&self) -> j2k::BatchDecodeOptions +pub fn j2k::BatchDecoder::prepare_batch(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::BatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::BatchErrorStage::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k::BatchGroupInfo::samples_per_image(&self) -> core::option::Option +pub fn j2k::CpuBatchDecodeResult::errors(&self) -> &[j2k::IndexedBatchError] +pub fn j2k::CpuBatchDecodeResult::groups(&self) -> &[j2k::CpuBatchGroup] +pub fn j2k::CpuBatchDecodeResult::into_parts(self) -> (alloc::vec::Vec, alloc::vec::Vec) +pub fn j2k::CpuBatchDecoder::decode(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_prepared(&mut self, &j2k::PreparedBatch) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_prepared(&mut self, &j2k::PreparedBatch) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k::CpuBatchDecoder::new(j2k::BatchDecodeOptions) -> Self +pub fn j2k::CpuBatchDecoder::options(&self) -> j2k::BatchDecodeOptions +pub fn j2k::CpuBatchDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::prepare_batch(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k::CpuBatchDecoder::retained_worker_count(&self) -> usize +pub fn j2k::CpuBatchDecoder::workspace_stats(&self) -> j2k::CpuBatchWorkspaceStats +pub fn j2k::CpuBatchGroup::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k::CpuBatchGroup::info(&self) -> &j2k::BatchGroupInfo +pub fn j2k::CpuBatchGroup::into_parts(self) -> (j2k::BatchGroupInfo, alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec>, j2k::CpuBatchSamples) +pub fn j2k::CpuBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k::CpuBatchGroup::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k::CpuBatchSamples::is_empty(&self) -> bool +pub fn j2k::CpuBatchSamples::len(&self) -> usize +pub fn j2k::DecodeSettings::default() -> Self +pub fn j2k::EncodedImage::full(alloc::sync::Arc<[u8]>) -> Self +pub fn j2k::EncodedImage::new(alloc::sync::Arc<[u8]>, j2k::DecodeRequest) -> Self +pub fn j2k::NonRepresentableReason::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k::PreparedBatch::errors(&self) -> &[j2k::IndexedBatchError] +pub fn j2k::PreparedBatch::groups(&self) -> &[j2k::PreparedBatchGroup] +pub fn j2k::PreparedBatch::into_parts(self) -> (alloc::sync::Arc<[j2k::PreparedBatchGroup]>, alloc::sync::Arc<[j2k::IndexedBatchError]>, j2k::BatchDecodeOptions) +pub fn j2k::PreparedBatchGroup::images(&self) -> &[j2k::PreparedImage] +pub fn j2k::PreparedBatchGroup::info(&self) -> &j2k::BatchGroupInfo +pub fn j2k::PreparedBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k::PreparedClassicPlan::is_color(&self) -> bool +pub fn j2k::PreparedClassicPlan::is_empty(&self) -> bool +pub fn j2k::PreparedClassicPlan::is_grayscale(&self) -> bool +pub fn j2k::PreparedClassicPlan::is_rgba(&self) -> bool +pub fn j2k::PreparedClassicPlan::payload(&self, usize) -> core::option::Option +pub fn j2k::PreparedClassicPlan::payload_count(&self) -> usize +pub fn j2k::PreparedClassicPlan::payloads(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator + '_ +pub fn j2k::PreparedClassicPlan::range(&self, usize) -> core::option::Option +pub fn j2k::PreparedClassicPlan::range_count(&self) -> usize +pub fn j2k::PreparedClassicPlan::ranges(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator + '_ +pub fn j2k::PreparedHtj2kPlan::is_color(&self) -> bool +pub fn j2k::PreparedHtj2kPlan::is_empty(&self) -> bool +pub fn j2k::PreparedHtj2kPlan::is_grayscale(&self) -> bool +pub fn j2k::PreparedHtj2kPlan::is_rgba(&self) -> bool +pub fn j2k::PreparedHtj2kPlan::payload(&self, usize) -> core::option::Option +pub fn j2k::PreparedHtj2kPlan::payload_count(&self) -> usize +pub fn j2k::PreparedHtj2kPlan::payloads(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator + '_ +pub fn j2k::PreparedImage::bytes(&self) -> &alloc::sync::Arc<[u8]> +pub fn j2k::PreparedImage::classic_plan(&self) -> core::option::Option<&j2k::PreparedClassicPlan> +pub fn j2k::PreparedImage::codestream_range(&self) -> j2k_types::decode_payload::J2kCodestreamRange +pub fn j2k::PreparedImage::decode_settings(&self) -> j2k::DecodeSettings +pub fn j2k::PreparedImage::htj2k_plan(&self) -> core::option::Option<&j2k::PreparedHtj2kPlan> +pub fn j2k::PreparedImage::plan(&self) -> j2k::DeviceDecodePlan +pub fn j2k::PreparedImage::preparation_depth(&self) -> j2k::PreparationDepth +pub fn j2k::PreparedImage::request(&self) -> j2k::DecodeRequest +pub fn j2k::PreparedImage::source_index(&self) -> usize +pub fn j2k::PreparedImage::support(&self) -> &j2k::J2kSupportInfo +pub fn j2k::prepare_batch(alloc::vec::Vec, j2k::BatchDecodeOptions) -> core::result::Result +pub fn j2k::prepare_batch_from_images(alloc::vec::Vec, j2k::BatchDecodeOptions) -> core::result::Result +pub j2k::BatchAlpha::None +pub j2k::BatchAlpha::Premultiplied +pub j2k::BatchAlpha::Straight +pub j2k::BatchCodecRoute::Classic +pub j2k::BatchCodecRoute::Htj2k +pub j2k::BatchDecodeOptions::layout: j2k::BatchLayout +pub j2k::BatchDecodeOptions::settings: j2k::DecodeSettings +pub j2k::BatchDecodeOptions::workers: core::option::Option +pub j2k::BatchErrorStage::Decode +pub j2k::BatchErrorStage::Prepare +pub j2k::BatchGroupInfo::alpha: j2k::BatchAlpha +pub j2k::BatchGroupInfo::color: j2k_core::pixel::PixelLayout +pub j2k::BatchGroupInfo::colorspace: j2k_core::types::Colorspace +pub j2k::BatchGroupInfo::dimensions: (u32, u32) +pub j2k::BatchGroupInfo::layout: j2k::BatchLayout +pub j2k::BatchGroupInfo::payload_kind: j2k_core::passthrough::CompressedPayloadKind +pub j2k::BatchGroupInfo::precision: u8 +pub j2k::BatchGroupInfo::route: j2k::BatchCodecRoute +pub j2k::BatchGroupInfo::sample_type: j2k_core::sample::SampleType +pub j2k::BatchGroupInfo::signed: bool +pub j2k::BatchGroupInfo::transfer_syntax: j2k_core::passthrough::CompressedTransferSyntax +pub j2k::BatchGroupInfo::transform: j2k::BatchWaveletTransform +pub j2k::BatchItemError::Codec +pub j2k::BatchItemError::Codec::source: alloc::sync::Arc +pub j2k::BatchItemError::Codec::stage: j2k::BatchErrorStage +pub j2k::BatchItemError::NonRepresentableBatchOutput +pub j2k::BatchItemError::NonRepresentableBatchOutput::reason: j2k::NonRepresentableReason +pub j2k::BatchItemError::PreparedDecodeSettingsMismatch +pub j2k::BatchItemError::PreparedDecodeSettingsMismatch::prepared: j2k::DecodeSettings +pub j2k::BatchItemError::PreparedDecodeSettingsMismatch::requested: j2k::DecodeSettings +pub j2k::BatchLayout::Nchw +pub j2k::BatchLayout::Nhwc +pub j2k::BatchWaveletTransform::Irreversible97 +pub j2k::BatchWaveletTransform::Reversible53 +pub j2k::CpuBatchSamples::I16(alloc::vec::Vec) +pub j2k::CpuBatchSamples::U16(alloc::vec::Vec) +pub j2k::CpuBatchSamples::U8(alloc::vec::Vec) +pub j2k::DecodeRequest::Full +pub j2k::DecodeRequest::Reduced +pub j2k::DecodeRequest::Reduced::scale: j2k_core::scale::Downscale +pub j2k::DecodeRequest::Region +pub j2k::DecodeRequest::Region::roi: j2k_core::types::Rect +pub j2k::DecodeRequest::RegionReduced +pub j2k::DecodeRequest::RegionReduced::roi: j2k_core::types::Rect +pub j2k::DecodeRequest::RegionReduced::scale: j2k_core::scale::Downscale +pub j2k::EncodedImage::bytes: alloc::sync::Arc<[u8]> +pub j2k::EncodedImage::request: j2k::DecodeRequest +pub j2k::IndexedBatchError::index: usize +pub j2k::IndexedBatchError::source: j2k::BatchItemError +pub j2k::NonRepresentableReason::ComponentSubsampling +pub j2k::NonRepresentableReason::MixedPrecision +pub j2k::NonRepresentableReason::MixedSignedness +pub j2k::NonRepresentableReason::MixedWaveletTransform +pub j2k::NonRepresentableReason::PrecisionAboveSixteen +pub j2k::NonRepresentableReason::UnsupportedColor +pub j2k::NonRepresentableReason::UnsupportedComponentCount +pub j2k::PreparationDepth::ClassicOffsetPlan +pub j2k::PreparationDepth::Htj2kOffsetPlan +pub j2k::PreparationDepth::MetadataOnly +pub struct j2k::BatchDecodeOptions +pub struct j2k::BatchGroupInfo +pub struct j2k::CpuBatchDecodeResult +pub struct j2k::CpuBatchDecoder +pub struct j2k::CpuBatchGroup +pub struct j2k::CpuBatchWorkspaceStats +pub struct j2k::DecodeSettings +pub struct j2k::EncodedImage +pub struct j2k::IndexedBatchError +pub struct j2k::PreparedBatch +pub struct j2k::PreparedBatchGroup +pub struct j2k::PreparedClassicPlan +pub struct j2k::PreparedHtj2kPlan +pub struct j2k::PreparedImage +pub trait j2k::BatchDecoder +pub type j2k::BatchDecoder::Error: core::convert::From +pub type j2k::BatchDecoder::Output +pub type j2k::CpuBatchDecoder::Error = j2k_core::batch::BatchInfrastructureError +pub type j2k::CpuBatchDecoder::Output = j2k::CpuBatchDecodeResult +pub use j2k::BatchColor +pub use j2k::BatchInfrastructureError +pub use j2k::ClassicCodeBlockPayload +pub use j2k::Htj2kPayloadRanges +pub use j2k::J2kCodestreamRange +pub use j2k::NativeSampleType +``` ### `j2k-core` -Baseline items: 484. Candidate items: 476. Computed release type: `minor`. Rustdoc-hidden candidate items: 141. Full hidden-inventory fingerprint: `fnv1a64:d4ca6ced92ff6406`. +Baseline items: 484. Candidate items: 487. Computed release type: `minor`. Rustdoc-hidden candidate items: 189. Full hidden-inventory fingerprint: `fnv1a64:b87aa26da547e491`. #### Removed or changed baseline API items @@ -79,6 +301,11 @@ pub struct j2k_core::DecoderContext #### Added candidate API items ```text +impl j2k_core::PixelLayout +impl j2k_core::Sample for i16 +pub const fn j2k_core::PixelLayout::channels(self) -> usize +pub const i16::BITS: u8 +pub const i16::TYPE: j2k_core::SampleType pub fn j2k_core::TileBatchDecode::decode_tile(&mut Self::Context, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::PixelFormat) -> core::result::Result, Self::Error> pub fn j2k_core::TileBatchDecode::decode_tile_region(&mut Self::Context, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::PixelFormat, j2k_core::Rect) -> core::result::Result, Self::Error> pub fn j2k_core::TileBatchDecode::decode_tile_region_scaled(&mut Self::Context, &mut Self::Pool, j2k_core::PixelFormat, j2k_core::TileRegionScaledDecodeJob<'_, '_>) -> core::result::Result, Self::Error> @@ -92,6 +319,12 @@ pub fn j2k_core::TileBatchDecodeSubmit::submit_tile_region_scaled_to_device(&mut pub fn j2k_core::TileBatchDecodeSubmit::submit_tile_region_to_device(&mut Self::Context, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::PixelFormat, j2k_core::Rect, j2k_core::BackendRequest) -> core::result::Result pub fn j2k_core::TileBatchDecodeSubmit::submit_tile_scaled_to_device(&mut Self::Context, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::PixelFormat, j2k_core::Downscale, j2k_core::BackendRequest) -> core::result::Result pub fn j2k_core::TileBatchDecodeSubmit::submit_tile_to_device(&mut Self::Context, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::PixelFormat, j2k_core::BackendRequest) -> core::result::Result +pub j2k_core::BatchInfrastructureError::UnsupportedContract +pub j2k_core::BatchInfrastructureError::UnsupportedContract::what: &'static str +pub j2k_core::PixelFormat::GrayI16 +pub j2k_core::PixelFormat::RgbI16 +pub j2k_core::PixelFormat::RgbaI16 +pub j2k_core::SampleType::I16 ``` ### `j2k-codec-math` @@ -144,7 +377,7 @@ None. ### `j2k-metal` -Baseline items: 181. Candidate items: 181. Computed release type: `minor`. Rustdoc-hidden candidate items: 287. Full hidden-inventory fingerprint: `fnv1a64:8ebd9cdd986dadbf`. +Baseline items: 181. Candidate items: 272. Computed release type: `minor`. Rustdoc-hidden candidate items: 299. Full hidden-inventory fingerprint: `fnv1a64:a496e2469837d3a8`. #### Removed or changed baseline API items @@ -152,7 +385,99 @@ None. #### Added candidate API items -None. +```text +impl core::fmt::Debug for j2k_metal::MetalBatchDecoder +impl core::fmt::Debug for j2k_metal::MetalBatchGroup +impl core::fmt::Debug for j2k_metal::MetalResidentBatch +impl core::fmt::Debug for j2k_metal::SubmittedMetalGroupDecodeInto +impl core::fmt::Debug for j2k_metal::SubmittedMetalPreparedBatch +impl j2k::owned_batch::prepared::BatchDecoder for j2k_metal::MetalBatchDecoder +impl j2k_core::traits::DeviceSubmission for j2k_metal::SubmittedMetalGroupDecodeInto +impl j2k_core::traits::DeviceSubmission for j2k_metal::SubmittedMetalPreparedBatch +impl j2k_metal::MetalBatchDecodeResult +impl j2k_metal::MetalBatchDecoder +impl j2k_metal::MetalBatchGroup +impl j2k_metal::MetalBatchGroupCompletion +impl j2k_metal::MetalBatchGroupError +impl j2k_metal::MetalResidentBatch +impl j2k_metal::SubmittedMetalGroupDecodeInto +impl j2k_metal::SubmittedMetalPreparedBatch +pub const fn j2k_metal::MetalBatchDecoder::options(&self) -> j2k::owned_batch::contracts::BatchDecodeOptions +pub const fn j2k_metal::MetalResidentBatch::byte_len(&self) -> usize +pub const fn j2k_metal::MetalResidentBatch::byte_offset(&self) -> usize +pub const fn j2k_metal::MetalResidentBatch::image_count(&self) -> usize +pub const fn j2k_metal::MetalResidentBatch::image_stride_bytes(&self) -> usize +pub fn j2k_metal::MetalBackendSession::with_command_queue(metal::device::Device, metal::commandqueue::CommandQueue) -> core::result::Result +pub fn j2k_metal::MetalBatchDecodeResult::errors(&self) -> &[j2k::owned_batch::errors::IndexedBatchError] +pub fn j2k_metal::MetalBatchDecodeResult::group_errors(&self) -> &[j2k_metal::MetalBatchGroupError] +pub fn j2k_metal::MetalBatchDecodeResult::groups(&self) -> &[j2k_metal::MetalBatchGroup] +pub fn j2k_metal::MetalBatchDecodeResult::into_parts(self) -> (alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec) +pub fn j2k_metal::MetalBatchDecoder::backend_session(&self) -> &j2k_metal::MetalBackendSession +pub fn j2k_metal::MetalBatchDecoder::decode_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::decode_prepared_group(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::decode_prepared_group_into(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, &j2k_metal_support::resident::destination::MetalImageDestination) -> core::result::Result<(), j2k_metal::Error> +pub fn j2k_metal::MetalBatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal::MetalBatchDecoder::options(&self) -> j2k::owned_batch::contracts::BatchDecodeOptions +pub fn j2k_metal::MetalBatchDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submissions(&self) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submit_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submit_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submit_prepared_group_into(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, j2k_metal_support::resident::destination::MetalImageDestination) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::submit_prepared_group_into_for_consumer_queue(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, j2k_metal_support::resident::destination::MetalImageDestination, &metal::commandqueue::CommandQueueRef) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::system_default() -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::system_default_with_options(j2k::owned_batch::contracts::BatchDecodeOptions) -> core::result::Result +pub fn j2k_metal::MetalBatchDecoder::validate_destination(&self, &j2k_metal_support::resident::destination::MetalImageDestination, (u32, u32), j2k_core::pixel::PixelFormat) -> core::result::Result<(), j2k_metal::Error> +pub fn j2k_metal::MetalBatchDecoder::with_backend_session(j2k_metal::MetalBackendSession) -> Self +pub fn j2k_metal::MetalBatchDecoder::with_backend_session_and_options(j2k_metal::MetalBackendSession, j2k::owned_batch::contracts::BatchDecodeOptions) -> Self +pub fn j2k_metal::MetalBatchGroup::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k_metal::MetalBatchGroup::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal::MetalBatchGroup::info(&self) -> &j2k::owned_batch::contracts::BatchGroupInfo +pub fn j2k_metal::MetalBatchGroup::into_parts(self) -> j2k_metal::MetalBatchGroupParts +pub fn j2k_metal::MetalBatchGroup::into_resident_batch(self) -> core::option::Option +pub fn j2k_metal::MetalBatchGroup::resident_batch(&self) -> core::option::Option<&j2k_metal::MetalResidentBatch> +pub fn j2k_metal::MetalBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k_metal::MetalBatchGroup::surfaces(&self) -> &[j2k_metal::Surface] +pub fn j2k_metal::MetalBatchGroup::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k_metal::MetalBatchGroupCompletion::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k_metal::MetalBatchGroupCompletion::into_parts(self) -> (alloc::vec::Vec, alloc::vec::Vec>) +pub fn j2k_metal::MetalBatchGroupCompletion::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k_metal::MetalBatchGroupError::into_parts(self) -> (alloc::vec::Vec, j2k_metal::Error) +pub fn j2k_metal::MetalBatchGroupError::source(&self) -> &j2k_metal::Error +pub fn j2k_metal::MetalBatchGroupError::source_indices(&self) -> &[usize] +pub fn j2k_metal::MetalResidentBatch::device_registry_id(&self) -> u64 +pub fn j2k_metal::MetalResidentBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal::SubmittedMetalGroupDecodeInto::enqueue_consumer_wait(&mut self, &metal::commandqueue::CommandQueueRef) -> core::result::Result<(), j2k_metal::Error> +pub fn j2k_metal::SubmittedMetalGroupDecodeInto::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal::SubmittedMetalGroupDecodeInto::wait(self) -> core::result::Result +pub fn j2k_metal::SubmittedMetalGroupDecodeInto::wait(self) -> core::result::Result +pub fn j2k_metal::SubmittedMetalPreparedBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal::SubmittedMetalPreparedBatch::is_empty(&self) -> bool +pub fn j2k_metal::SubmittedMetalPreparedBatch::len(&self) -> usize +pub fn j2k_metal::SubmittedMetalPreparedBatch::wait(self) -> core::result::Result +pub fn j2k_metal::SubmittedMetalPreparedBatch::wait(self) -> core::result::Result +pub struct j2k_metal::MetalBatchDecodeResult +pub struct j2k_metal::MetalBatchDecoder +pub struct j2k_metal::MetalBatchGroup +pub struct j2k_metal::MetalBatchGroupCompletion +pub struct j2k_metal::MetalBatchGroupError +pub struct j2k_metal::MetalResidentBatch +pub struct j2k_metal::SubmittedMetalGroupDecodeInto +pub struct j2k_metal::SubmittedMetalPreparedBatch +pub type j2k_metal::MetalBatchDecoder::Error = j2k_metal::Error +pub type j2k_metal::MetalBatchDecoder::Output = j2k_metal::MetalBatchDecodeResult +pub type j2k_metal::MetalBatchGroupParts = (j2k::owned_batch::contracts::BatchGroupInfo, alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec>, alloc::vec::Vec) +pub type j2k_metal::SubmittedMetalGroupDecodeInto::Error = j2k_metal::Error +pub type j2k_metal::SubmittedMetalGroupDecodeInto::Output = j2k_metal::MetalBatchGroupCompletion +pub type j2k_metal::SubmittedMetalPreparedBatch::Error = j2k_metal::Error +pub type j2k_metal::SubmittedMetalPreparedBatch::Output = j2k_metal::MetalBatchDecodeResult +pub unsafe fn j2k_metal::MetalResidentBatch::metal_buffer(&self) -> &metal::buffer::Buffer +pub use j2k_metal::MetalImageDestination +pub use j2k_metal::MetalImageLayout +``` ### `j2k-jpeg-cuda` @@ -168,7 +493,7 @@ None. ### `j2k-cuda` -Baseline items: 123. Candidate items: 123. Computed release type: `minor`. Rustdoc-hidden candidate items: 171. Full hidden-inventory fingerprint: `fnv1a64:56bd9f23e4d2c9fb`. +Baseline items: 123. Candidate items: 229. Computed release type: `minor`. Rustdoc-hidden candidate items: 181. Full hidden-inventory fingerprint: `fnv1a64:6bb25d5241c4e86a`. #### Removed or changed baseline API items @@ -176,7 +501,114 @@ None. #### Added candidate API items -None. +```text +#[non_exhaustive] pub enum j2k_cuda::CudaBatchError +impl core::fmt::Debug for j2k_cuda::SubmittedCudaExternalBatch +impl core::fmt::Debug for j2k_cuda::SubmittedCudaResidentBatch +impl j2k::owned_batch::prepared::BatchDecoder for j2k_cuda::CudaBatchDecoder +impl j2k_cuda::CudaBatchDecodeResult +impl j2k_cuda::CudaBatchDecoder +impl j2k_cuda::CudaBatchGroup +impl j2k_cuda::CudaBatchGroupError +impl j2k_cuda::CudaDecodePoolDiagnostics +impl j2k_cuda::CudaDecodePoolSnapshot +impl j2k_cuda::CudaExternalBatchGroup +impl j2k_cuda::CudaResidentBatchBuffer +impl j2k_cuda::SubmittedCudaExternalBatch +impl j2k_cuda::SubmittedCudaResidentBatch +pub const fn j2k_cuda::CudaBatchDecoder::options(&self) -> j2k::owned_batch::contracts::BatchDecodeOptions +pub const fn j2k_cuda::CudaBatchDecoder::session(&self) -> &j2k_cuda::CudaSession +pub const fn j2k_cuda::CudaBatchDecoder::with_session_and_options(j2k_cuda::CudaSession, j2k::owned_batch::contracts::BatchDecodeOptions) -> Self +pub const fn j2k_cuda::CudaBatchGroup::dense_output(&self) -> &j2k_cuda::CudaResidentBatchBuffer +pub const fn j2k_cuda::CudaBatchGroup::info(&self) -> &j2k::owned_batch::contracts::BatchGroupInfo +pub const fn j2k_cuda::CudaDecodePoolSnapshot::peak_retained_bytes_upper_bound(self) -> usize +pub const fn j2k_cuda::CudaDecodePoolSnapshot::retained_bytes(self) -> usize +pub const fn j2k_cuda::CudaExternalBatchGroup::info(&self) -> &j2k::owned_batch::contracts::BatchGroupInfo +pub enum j2k_cuda::CudaExternalBatchTryFinish +pub fn j2k_cuda::CudaBatchDecodeResult::errors(&self) -> &[j2k::owned_batch::errors::IndexedBatchError] +pub fn j2k_cuda::CudaBatchDecodeResult::group_errors(&self) -> &[j2k_cuda::CudaBatchGroupError] +pub fn j2k_cuda::CudaBatchDecodeResult::groups(&self) -> &[j2k_cuda::CudaBatchGroup] +pub fn j2k_cuda::CudaBatchDecodeResult::into_parts(self) -> (alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec) +pub fn j2k_cuda::CudaBatchDecoder::decode_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::decode_pool_diagnostics(&self) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::diagnostics(&self) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::new() -> Self +pub fn j2k_cuda::CudaBatchDecoder::options(&self) -> j2k::owned_batch::contracts::BatchDecodeOptions +pub fn j2k_cuda::CudaBatchDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::session_mut(&mut self) -> &mut j2k_cuda::CudaSession +pub fn j2k_cuda::CudaBatchDecoder::submit_batch(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::submit_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_cuda::CudaBatchDecoder::with_options(j2k::owned_batch::contracts::BatchDecodeOptions) -> Self +pub fn j2k_cuda::CudaBatchDecoder::with_session(j2k_cuda::CudaSession) -> Self +pub fn j2k_cuda::CudaBatchGroup::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k_cuda::CudaBatchGroup::into_parts(self) -> (j2k::owned_batch::contracts::BatchGroupInfo, alloc::vec::Vec, alloc::vec::Vec, alloc::vec::Vec>, alloc::vec::Vec, j2k_cuda::CudaResidentBatchBuffer) +pub fn j2k_cuda::CudaBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k_cuda::CudaBatchGroup::surfaces(&self) -> &[j2k_cuda::Surface] +pub fn j2k_cuda::CudaBatchGroup::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k_cuda::CudaBatchGroupError::into_parts(self) -> (alloc::vec::Vec, j2k_cuda::Error) +pub fn j2k_cuda::CudaBatchGroupError::source(&self) -> &j2k_cuda::Error +pub fn j2k_cuda::CudaBatchGroupError::source_indices(&self) -> &[usize] +pub fn j2k_cuda::CudaDecodePoolDiagnostics::peak_retained_bytes_upper_bound(self) -> usize +pub fn j2k_cuda::CudaDecodePoolDiagnostics::retained_bytes(self) -> usize +pub fn j2k_cuda::CudaExternalBatchGroup::decoded_rects(&self) -> &[j2k_core::types::Rect] +pub fn j2k_cuda::CudaExternalBatchGroup::ranges(&self) -> &[j2k_cuda_runtime::memory::CudaDeviceBufferRange] +pub fn j2k_cuda::CudaExternalBatchGroup::source_indices(&self) -> &[usize] +pub fn j2k_cuda::CudaExternalBatchGroup::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k_cuda::CudaResidentBatchBuffer::buffer(&self) -> &j2k_cuda_runtime::memory::CudaDeviceBuffer +pub fn j2k_cuda::CudaResidentBatchBuffer::ranges(&self) -> &[j2k_cuda_runtime::memory::CudaDeviceBufferRange] +pub fn j2k_cuda::CudaSession::decode_pool_diagnostics(&self) -> core::result::Result +pub fn j2k_cuda::CudaSession::diagnostics(&self) -> core::result::Result +pub fn j2k_cuda::SubmittedCudaExternalBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_cuda::SubmittedCudaExternalBatch::group(&self) -> &j2k_cuda::CudaExternalBatchGroup +pub fn j2k_cuda::SubmittedCudaExternalBatch::is_complete(&self) -> core::result::Result +pub fn j2k_cuda::SubmittedCudaExternalBatch::try_finish(self) -> core::result::Result +pub fn j2k_cuda::SubmittedCudaExternalBatch::wait(self) -> core::result::Result +pub fn j2k_cuda::SubmittedCudaResidentBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_cuda::SubmittedCudaResidentBatch::is_complete(&self) -> core::result::Result +pub fn j2k_cuda::SubmittedCudaResidentBatch::pending_group_count(&self) -> usize +pub fn j2k_cuda::SubmittedCudaResidentBatch::wait(self) -> core::result::Result +pub j2k_cuda::CudaBatchError::GroupExecution +pub j2k_cuda::CudaBatchError::GroupExecution::source: alloc::boxed::Box +pub j2k_cuda::CudaBatchError::GroupExecution::source_indices: alloc::vec::Vec +pub j2k_cuda::CudaBatchError::Infrastructure(j2k_core::batch::BatchInfrastructureError) +pub j2k_cuda::CudaDecodePoolDiagnostics::batch_decode: core::option::Option +pub j2k_cuda::CudaDecodePoolDiagnostics::decode: core::option::Option +pub j2k_cuda::CudaDecodePoolSnapshot::cached_buffers: usize +pub j2k_cuda::CudaDecodePoolSnapshot::cached_bytes: usize +pub j2k_cuda::CudaDecodePoolSnapshot::deferred_buffers: usize +pub j2k_cuda::CudaDecodePoolSnapshot::deferred_bytes: usize +pub j2k_cuda::CudaDecodePoolSnapshot::peak_cached_bytes: usize +pub j2k_cuda::CudaDecodePoolSnapshot::peak_deferred_bytes: usize +pub j2k_cuda::CudaDecodePoolSnapshot::reuse_holds: usize +pub j2k_cuda::CudaExternalBatchTryFinish::Complete(j2k_cuda::CudaExternalBatchGroup) +pub j2k_cuda::CudaExternalBatchTryFinish::Pending(j2k_cuda::SubmittedCudaExternalBatch) +pub j2k_cuda::CudaSessionDiagnostics::pools: j2k_cuda::CudaDecodePoolDiagnostics +pub j2k_cuda::CudaSessionDiagnostics::runtime: core::option::Option +pub j2k_cuda::Error::CudaTier1JobFailed +pub j2k_cuda::Error::CudaTier1JobFailed::original_job_index: usize +pub j2k_cuda::Error::CudaTier1JobFailed::source: j2k_cuda_runtime::error::CudaError +pub j2k_cuda::Error::CudaTier1JobFailed::source_index: usize +pub j2k_cuda::Error::HtJobChunkPlan(j2k_core::batch::gpu_job_chunk::HtGpuJobChunkPlanError) +pub struct j2k_cuda::CudaBatchDecodeResult +pub struct j2k_cuda::CudaBatchDecoder +pub struct j2k_cuda::CudaBatchGroup +pub struct j2k_cuda::CudaBatchGroupError +pub struct j2k_cuda::CudaDecodePoolDiagnostics +pub struct j2k_cuda::CudaDecodePoolSnapshot +pub struct j2k_cuda::CudaExternalBatchGroup +pub struct j2k_cuda::CudaResidentBatchBuffer +pub struct j2k_cuda::CudaSessionDiagnostics +pub struct j2k_cuda::SubmittedCudaExternalBatch +pub struct j2k_cuda::SubmittedCudaResidentBatch +pub type j2k_cuda::CudaBatchDecoder::Error = j2k_cuda::CudaBatchError +pub type j2k_cuda::CudaBatchDecoder::Output = j2k_cuda::CudaBatchDecodeResult +pub unsafe fn j2k_cuda::CudaBatchDecoder::decode_batch_into(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, &mut j2k_cuda_runtime::memory::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result +pub unsafe fn j2k_cuda::CudaBatchDecoder::submit_batch_into(&mut self, &j2k::owned_batch::prepared::PreparedBatchGroup, &mut j2k_cuda_runtime::memory::CudaExternalDeviceBufferViewMut<'_>) -> core::result::Result +``` ### `j2k-transcode` @@ -204,7 +636,7 @@ None. ### `j2k-metal-support` -Baseline items: 162. Candidate items: 162. Computed release type: `minor`. Rustdoc-hidden candidate items: 11. Full hidden-inventory fingerprint: `fnv1a64:671e50d4d121b419`. +Baseline items: 162. Candidate items: 178. Computed release type: `minor`. Rustdoc-hidden candidate items: 11. Full hidden-inventory fingerprint: `fnv1a64:671e50d4d121b419`. #### Removed or changed baseline API items @@ -212,7 +644,24 @@ None. #### Added candidate API items -None. +```text +impl core::fmt::Debug for j2k_metal_support::MetalImageDestination +impl j2k_metal_support::MetalImageDestination +pub const fn j2k_metal_support::MetalImageDestination::buffer_len(&self) -> usize +pub const fn j2k_metal_support::MetalImageDestination::device_registry_id(&self) -> u64 +pub const fn j2k_metal_support::MetalImageDestination::layout(&self) -> j2k_metal_support::MetalImageLayout +pub const fn j2k_metal_support::MetalImageLayout::image_count(self) -> usize +pub const fn j2k_metal_support::MetalImageLayout::image_offset_bytes(self, usize) -> core::option::Option +pub const fn j2k_metal_support::MetalImageLayout::image_stride_bytes(self) -> usize +pub fn j2k_metal_support::MetalImageDestination::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_metal_support::MetalImageDestination::validate_batch(&self, (u32, u32), j2k_core::pixel::PixelFormat, usize) -> core::result::Result<(), j2k_metal_support::MetalSupportError> +pub fn j2k_metal_support::MetalImageDestination::validate_device(&self, &metal::device::DeviceRef) -> core::result::Result<(), j2k_metal_support::MetalSupportError> +pub fn j2k_metal_support::MetalImageDestination::validate_image(&self, (u32, u32), j2k_core::pixel::PixelFormat) -> core::result::Result<(), j2k_metal_support::MetalSupportError> +pub fn j2k_metal_support::MetalImageLayout::new_batch(usize, (u32, u32), usize, j2k_core::pixel::PixelFormat, usize, usize) -> core::result::Result +pub struct j2k_metal_support::MetalImageDestination +pub unsafe fn j2k_metal_support::MetalImageDestination::from_exclusive_buffer(metal::buffer::Buffer, j2k_metal_support::MetalImageLayout) -> core::result::Result +pub unsafe fn j2k_metal_support::MetalImageDestination::raw_buffer(&self) -> &metal::buffer::Buffer +``` ### `j2k-transcode-metal` @@ -228,7 +677,7 @@ None. ### `j2k-native` -Baseline items: 358. Candidate items: 358. Computed release type: `minor`. Rustdoc-hidden candidate items: 535. Full hidden-inventory fingerprint: `fnv1a64:7c931c0e7c4eb27b`. +Baseline items: 358. Candidate items: 377. Computed release type: `minor`. Rustdoc-hidden candidate items: 661. Full hidden-inventory fingerprint: `fnv1a64:14c8dc9c142b3008`. #### Removed or changed baseline API items @@ -236,11 +685,31 @@ None. #### Added candidate API items -None. +```text +impl j2k_native::DecoderWorkspace +impl j2k_native::DecoderWorkspaceStats +pub const fn j2k_native::DecoderContext<'_>::workspace_stats(&self) -> j2k_native::DecoderWorkspaceStats +pub const fn j2k_native::DecoderWorkspace::stats(&self) -> j2k_native::DecoderWorkspaceStats +pub const fn j2k_native::DecoderWorkspaceStats::component_owner_reuses(self) -> u64 +pub const fn j2k_native::DecoderWorkspaceStats::decode_calls(self) -> u64 +pub const fn j2k_native::DecoderWorkspaceStats::idwt_owner_reuses(self) -> u64 +pub const fn j2k_native::DecoderWorkspaceStats::retained_component_bytes(self) -> usize +pub const fn j2k_native::DecoderWorkspaceStats::retained_idwt_bytes(self) -> usize +pub const fn j2k_native::DecoderWorkspaceStats::retained_scratch_bytes(self) -> usize +pub const fn j2k_native::DecoderWorkspaceStats::retained_tier1_bytes(self) -> usize +pub const fn j2k_native::DecoderWorkspaceStats::scratch_capacity_retries(self) -> u64 +pub const fn j2k_native::DecoderWorkspaceStats::tier1_owner_reuses(self) -> u64 +pub fn j2k_native::DecoderContext<'_>::from_workspace(j2k_native::DecoderWorkspace) -> Self +pub fn j2k_native::DecoderContext<'_>::into_workspace(self) -> j2k_native::DecoderWorkspace +pub j2k_native::DirectPlanUnsupportedReason::RgbaFourComponentRgbCodestream +pub j2k_native::DirectPlanUnsupportedReason::RgbaRgbImageWithAlpha +pub struct j2k_native::DecoderWorkspace +pub struct j2k_native::DecoderWorkspaceStats +``` ### `j2k-types` -Baseline items: 351. Candidate items: 351. Computed release type: `minor`. Rustdoc-hidden candidate items: 54. Full hidden-inventory fingerprint: `fnv1a64:4b30413a7bfffeb5`. +Baseline items: 351. Candidate items: 365. Computed release type: `minor`. Rustdoc-hidden candidate items: 54. Full hidden-inventory fingerprint: `fnv1a64:4b30413a7bfffeb5`. #### Removed or changed baseline API items @@ -248,11 +717,26 @@ None. #### Added candidate API items -None. +```text +impl j2k_types::J2kClassicCodeBlockPayload +impl j2k_types::J2kCodestreamRange +pub const fn j2k_types::J2kClassicCodeBlockPayload::end_range(self) -> core::option::Option +pub const fn j2k_types::J2kCodestreamRange::end(self) -> core::option::Option +pub j2k_types::HtCodeBlockPayloadRanges::cleanup: j2k_types::J2kCodestreamRange +pub j2k_types::HtCodeBlockPayloadRanges::refinement: core::option::Option +pub j2k_types::J2kClassicCodeBlockPayload::combined_length: usize +pub j2k_types::J2kClassicCodeBlockPayload::first_range: usize +pub j2k_types::J2kClassicCodeBlockPayload::range_count: usize +pub j2k_types::J2kCodestreamRange::length: usize +pub j2k_types::J2kCodestreamRange::offset: usize +pub struct j2k_types::HtCodeBlockPayloadRanges +pub struct j2k_types::J2kClassicCodeBlockPayload +pub struct j2k_types::J2kCodestreamRange +``` ### `j2k-cuda-runtime` -Baseline items: 95. Candidate items: 95. Computed release type: `minor`. Rustdoc-hidden candidate items: 918. Full hidden-inventory fingerprint: `fnv1a64:61e8d94589aa373a`. +Baseline items: 95. Candidate items: 100. Computed release type: `minor`. Rustdoc-hidden candidate items: 1089. Full hidden-inventory fingerprint: `fnv1a64:d33ffdcb13c73dd5`. #### Removed or changed baseline API items @@ -260,7 +744,13 @@ None. #### Added candidate API items -None. +```text +pub j2k_cuda_runtime::CudaError::KernelJobStatus +pub j2k_cuda_runtime::CudaError::KernelJobStatus::code: u32 +pub j2k_cuda_runtime::CudaError::KernelJobStatus::detail: u32 +pub j2k_cuda_runtime::CudaError::KernelJobStatus::job_index: usize +pub j2k_cuda_runtime::CudaError::KernelJobStatus::kernel: &'static str +``` ### `j2k-profile` diff --git a/third_party/cubecl-cuda-0.10.0-patched/.cargo-ok b/third_party/cubecl-cuda-0.10.0-patched/.cargo-ok new file mode 100644 index 00000000..5f8b7958 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/third_party/cubecl-cuda-0.10.0-patched/.cargo_vcs_info.json b/third_party/cubecl-cuda-0.10.0-patched/.cargo_vcs_info.json new file mode 100644 index 00000000..5818811d --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "7cf203735e095e640a2c03b2400d0faa03196bb4" + }, + "path_in_vcs": "crates/cubecl-cuda" +} \ No newline at end of file diff --git a/third_party/cubecl-cuda-0.10.0-patched/Cargo.lock b/third_party/cubecl-cuda-0.10.0-patched/Cargo.lock new file mode 100644 index 00000000..80455105 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/Cargo.lock @@ -0,0 +1,2068 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +dependencies = [ + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "cubecl-common" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc956c2dcc993f16f748d03bfdbd900f75cab4d469f4e5d8924f8ee128e861ad" +dependencies = [ + "backtrace", + "bytemuck", + "cfg-if", + "cfg_aliases", + "ciborium", + "derive-new", + "derive_more", + "dirs", + "embassy-futures", + "embassy-time", + "float4", + "float8", + "futures-lite", + "half", + "hashbrown 0.16.1", + "log", + "num-traits", + "oneshot", + "parking_lot", + "portable-atomic", + "portable-atomic-util", + "rand", + "sanitize-filename", + "serde", + "serde_bytes", + "serde_json", + "spin", + "toml", + "tracing", + "tynm", + "wasm-bindgen-futures", + "web-time", + "xxhash-rust", +] + +[[package]] +name = "cubecl-core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7522e7acf25d7848032c7b5270780f9f2abe84e13c7ed6345aa27ba70c312ca" +dependencies = [ + "bitflags", + "bytemuck", + "cubecl-common", + "cubecl-ir", + "cubecl-macros", + "cubecl-runtime", + "cubecl-zspace", + "derive-new", + "derive_more", + "enumset", + "float-ord", + "half", + "hashbrown 0.16.1", + "log", + "num-traits", + "paste", + "serde", + "serde_json", + "tempfile", + "test-log", + "tracing", + "variadics_please", +] + +[[package]] +name = "cubecl-cpp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411af04828cbf4583ecd388579fb30cd7a644eec19a334bb8f0d9d08522e1458" +dependencies = [ + "bytemuck", + "cubecl-common", + "cubecl-core", + "cubecl-opt", + "cubecl-runtime", + "derive-new", + "half", + "itertools", + "log", +] + +[[package]] +name = "cubecl-cuda" +version = "0.10.0" +dependencies = [ + "bytemuck", + "cubecl-common", + "cubecl-core", + "cubecl-cpp", + "cubecl-runtime", + "cubecl-std", + "cudarc", + "derive-new", + "half", + "log", + "paste", + "pretty_assertions", + "serde", + "test-log", + "tracing", +] + +[[package]] +name = "cubecl-ir" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d28a897a40c8ee6c57a8b8d76d8823ba2e97f4c2b83db9338f17a0752132d486" +dependencies = [ + "cubecl-common", + "cubecl-macros-internal", + "derive-new", + "derive_more", + "enumset", + "float-ord", + "fnv", + "foldhash 0.2.0", + "half", + "hashbrown 0.16.1", + "num-traits", + "portable-atomic", + "serde", + "variadics_please", +] + +[[package]] +name = "cubecl-macros" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77aa6df563e8e6a0926d2e9eeacb968737940a0e1ae9be819f6a65a341006e12" +dependencies = [ + "cubecl-common", + "darling 0.23.0", + "derive-new", + "ident_case", + "inflections", + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cubecl-macros-internal" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a515651a5a91e25d87f71f311f80a088e6cf815798b9922560a97636da6b8740" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cubecl-opt" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a599c6c3efefdcee8636994c90e58ef696c7efbed2d18b5b5ad8d5f29923abb" +dependencies = [ + "cubecl-common", + "cubecl-core", + "cubecl-ir", + "float-ord", + "log", + "num", + "petgraph", + "smallvec", + "stable-vec", + "type-map", +] + +[[package]] +name = "cubecl-runtime" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b68491bf5b3e997ae36bdc4e63b4ccd6d2f0e86b3b596a5d7a48d2b9e92622a0" +dependencies = [ + "ahash", + "async-channel", + "bytemuck", + "cfg-if", + "cfg_aliases", + "cubecl-common", + "cubecl-ir", + "cubecl-zspace", + "derive-new", + "derive_more", + "dirs", + "enumset", + "hashbrown 0.16.1", + "log", + "md5", + "serde", + "serde_json", + "spin", + "thiserror", + "toml", + "tracing", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "cubecl-std" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b391b584a4897683dd9fc0767f96337ac712b733126ce0f68a9ee8a2df073d2d" +dependencies = [ + "cubecl-common", + "cubecl-core", + "cubecl-runtime", + "half", + "num-traits", + "paste", + "serde", + "spin", + "test-log", + "variadics_please", +] + +[[package]] +name = "cubecl-zspace" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04054d95698afab785a4dda9f949e297831dd9e4cc6d036a93e6603f88e88b07" +dependencies = [ + "derive-new", + "serde", + "smallvec", +] + +[[package]] +name = "cudarc" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f071cd6a7b5d51607df76aa2d426aaabc7a74bc6bdb885b8afa63a880572ad9b" +dependencies = [ + "libloading", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn", +] + +[[package]] +name = "derive-new" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", + "unicode-xid", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "embassy-futures" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc2d050bdc5c21e0862a89256ed8029ae6c290a93aecefc73084b3002cdebb01" + +[[package]] +name = "embassy-time" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "592b0c143ec626e821d4d90da51a2bd91d559d6c442b7c74a47d368c9e23d97a" +dependencies = [ + "cfg-if", + "critical-section", + "document-features", + "embassy-time-driver", + "embedded-hal 0.2.7", + "embedded-hal 1.0.0", + "embedded-hal-async", + "futures-core", +] + +[[package]] +name = "embassy-time-driver" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee71af1b3a0deaa53eaf2d39252f83504c853646e472400b763060389b9fcc9" +dependencies = [ + "document-features", +] + +[[package]] +name = "embedded-hal" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff" +dependencies = [ + "nb 0.1.3", + "void", +] + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "embedded-hal-async" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4c685bbef7fe13c3c6dd4da26841ed3980ef33e841cddfa15ce8a8fb3f1884" +dependencies = [ + "embedded-hal 1.0.0", +] + +[[package]] +name = "enumset" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96a4a12fe60ac746ae295a1a4ecb5bb02debc20856506c8635288065f142de" +dependencies = [ + "enumset_derive", + "serde", +] + +[[package]] +name = "enumset_derive" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd536557b58c682b217b8fb199afdff47cd3eff260623f19e77074eb073d63a" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_filter" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +dependencies = [ + "log", +] + +[[package]] +name = "env_logger" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +dependencies = [ + "env_filter", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + +[[package]] +name = "float4" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a5404bf31d22893d61cf24d4dda149d8e6b2ff07601c3cb3be651031f61a4ed" + +[[package]] +name = "float8" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d1f04709a8ac06e8e8042875a3c466cc4832d3c1a18dbcb9dba3c6e83046bc" +dependencies = [ + "half", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", + "wasip2", + "wasip3", +] + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "bytemuck", + "cfg-if", + "crunchy", + "num-traits", + "serde", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "inflections" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +dependencies = [ + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "nb" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f" +dependencies = [ + "nb 1.1.0", +] + +[[package]] +name = "nb" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5439c4ad607c3c23abf66de8c8bf57ba8adcd1f129e699851a6e43935d339d" + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oneshot" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe21416a02c693fb9f980befcb230ecc70b0b3d1cc4abf88b9675c4c1457f0c" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +dependencies = [ + "serde", +] + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "sanitize-filename" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" +dependencies = [ + "regex", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +dependencies = [ + "lock_api", + "portable-atomic", +] + +[[package]] +name = "stable-vec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dac7bc0f7d0d44329b200020effbc25a534d89fa142af95e3ddf76113412a5e" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "test-log" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f46bf474f0a4afebf92f076d54fd5e63423d9438b8c278a3d2ccb0f47f7cdb3" +dependencies = [ + "env_logger", + "test-log-macros", + "tracing-subscriber", +] + +[[package]] +name = "test-log-core" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d4d41320b48bc4a211a9021678fcc0c99569b594ea31c93735b8e517102b4c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "test-log-macros" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9beb9249a81e430dffd42400a49019bcf548444f1968ff23080a625de0d4d320" +dependencies = [ + "syn", + "test-log-core", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "once_cell", + "regex-automata", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tynm" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21cdb0fc8f85c98b1ec812bc4cd69faf6c0fa2fc17d44ea3c2cdd38dc08e999" +dependencies = [ + "nom", +] + +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "variadics_please" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b6d82be61465f97d42bd1d15bf20f3b0a3a0905018f38f9d6f6962055b0b5c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml b/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml new file mode 100644 index 00000000..21a144d9 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml @@ -0,0 +1,170 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +name = "cubecl-cuda" +version = "0.10.0" +authors = ["nathanielsimard "] +build = "build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "CUDA runtime for CubeCL" +readme = "README.md" +keywords = [ + "gpu", + "cuda", +] +categories = ["science"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/tracel-ai/cubecl/tree/main/crates/cubecl-cuda" +resolver = "2" + +[features] +default = [ + "std", + "cubecl-runtime/default", + "cubecl-common/default", + "cubecl-core/default", +] +ptx-wmma = [] +std = [ + "cubecl-runtime/std", + "cubecl-common/std", + "cubecl-core/std", +] +tracing = [ + "dep:tracing", + "cubecl-runtime/tracing", + "cubecl-common/tracing", + "cubecl-core/tracing", +] + +[lib] +name = "cubecl_cuda" +path = "src/lib.rs" + +[dependencies.bytemuck] +version = "1.16.1" + +[dependencies.cubecl-common] +version = "0.10.0" +features = [ + "compilation-cache", + "hash", +] +default-features = false + +[dependencies.cubecl-core] +version = "0.10.0" +default-features = false + +[dependencies.cubecl-cpp] +version = "0.10.0" +features = ["cuda"] +default-features = false + +[dependencies.cubecl-runtime] +version = "0.10.0" +features = [ + "channel-mutex", + "std", +] +default-features = false + +[dependencies.cudarc] +version = "0.19.0" +features = [ + "std", + "driver", + "nvrtc", + "nccl", + "fallback-dynamic-loading", + "cuda-version-from-build-system", + "fallback-latest", +] +default-features = false + +[dependencies.derive-new] +version = "0.7.0" +default-features = false + +[dependencies.half] +version = "2.5" +features = [ + "alloc", + "num-traits", + "serde", +] +default-features = false + +[dependencies.log] +version = "^0.4.22" +default-features = false + +[dependencies.serde] +version = "1.0.204" +features = [ + "derive", + "alloc", +] +default-features = false + +[dependencies.tracing] +version = "^0.1.43" +features = ["attributes"] +optional = true +default-features = false + +[dev-dependencies.cubecl-core] +version = "0.10.0" +features = ["export_tests"] + +[dev-dependencies.cubecl-std] +version = "0.10.0" +features = ["export_tests"] + +[dev-dependencies.paste] +version = "1" + +[dev-dependencies.pretty_assertions] +version = "1.4" + +[dev-dependencies.test-log] +version = "^0.2" +features = ["trace"] +default-features = false + +[build-dependencies.cudarc] +version = "0.19.0" +features = [ + "std", + "driver", + "nvrtc", + "nccl", + "fallback-dynamic-loading", + "cuda-version-from-build-system", + "fallback-latest", +] +default-features = false + +[lints.clippy] +doc_markdown = "warn" + +[lints.rust] +warnings = "warn" + +[lints.rustdoc] +broken_intra_doc_links = "warn" +invalid_html_tags = "warn" diff --git a/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml.orig b/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml.orig new file mode 100644 index 00000000..efe149f6 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml.orig @@ -0,0 +1,73 @@ +[package] +authors = ["nathanielsimard "] +categories = ["science"] +description = "CUDA runtime for CubeCL" +edition.workspace = true +keywords = ["gpu", "cuda"] +license.workspace = true +name = "cubecl-cuda" +readme.workspace = true +repository = "https://github.com/tracel-ai/cubecl/tree/main/crates/cubecl-cuda" +version.workspace = true + + +[lints] +workspace = true + + +[features] +default = [ + "std", + "cubecl-runtime/default", + "cubecl-common/default", + "cubecl-core/default", +] +ptx-wmma = [] +std = ["cubecl-runtime/std", "cubecl-common/std", "cubecl-core/std"] + +tracing = [ + "dep:tracing", + "cubecl-runtime/tracing", + "cubecl-common/tracing", + "cubecl-core/tracing", +] + + +[dependencies] +# Runtime Deps +log = { workspace = true } +tracing = { workspace = true, features = ["attributes"], optional = true } + +bytemuck = { workspace = true } +cubecl-common = { path = "../cubecl-common", version = "0.10.0", default-features = false, features = [ + "compilation-cache", + "hash", +] } +cubecl-core = { path = "../cubecl-core", version = "0.10.0", default-features = false } +cubecl-cpp = { path = "../cubecl-cpp", version = "0.10.0", default-features = false, features = [ + "cuda", +] } +cubecl-runtime = { path = "../cubecl-runtime", version = "0.10.0", default-features = false, features = [ + "channel-mutex", + "std", +] } +cudarc = { workspace = true } + +derive-new = { workspace = true } +half = { workspace = true } +serde = { workspace = true } + + +[dev-dependencies] +cubecl-core = { path = "../cubecl-core", version = "0.10.0", features = [ + "export_tests", +] } +cubecl-std = { path = "../cubecl-std", version = "0.10.0", features = [ + "export_tests", +] } +paste = { workspace = true } +pretty_assertions = { workspace = true } +test-log = { workspace = true, features = ["trace"] } + +[build-dependencies] +cudarc = { workspace = true } diff --git a/third_party/cubecl-cuda-0.10.0-patched/LICENSE-APACHE b/third_party/cubecl-cuda-0.10.0-patched/LICENSE-APACHE new file mode 100644 index 00000000..a18fcc67 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2022 Nathaniel Simard & CubeCl Framework Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/third_party/cubecl-cuda-0.10.0-patched/LICENSE-MIT b/third_party/cubecl-cuda-0.10.0-patched/LICENSE-MIT new file mode 100644 index 00000000..6fca47c9 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Nathaniel Simard & CubeCL Framework Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md b/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md new file mode 100644 index 00000000..2a115570 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md @@ -0,0 +1,12 @@ +# Patch provenance + +Source: `cubecl-cuda` 0.10.0 from crates.io, checksum +`b6b0a69ff45688d322ad8e92c8bf645167b9ca490fa8fa087fc6adac8c5e46be`. + +Local change: implement CubeCL runtime's hidden external-write hook by +resolving the allocation binding on its owning CUDA stream and returning that +stream as an opaque numeric token. `j2k-ml` immediately passes the token to +`j2k-cuda-runtime`, which orders the codec and CubeCL streams with CUDA events. + +Remove this patch when CubeCL exposes an equivalent external-write/event +ordering contract. diff --git a/third_party/cubecl-cuda-0.10.0-patched/README.md b/third_party/cubecl-cuda-0.10.0-patched/README.md new file mode 100644 index 00000000..a02ef40f --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/README.md @@ -0,0 +1,13 @@ +# Cuda runtime + +The runtime uses the lower level primitives from [cudarc](https://github.com/coreylowman/cudarc) to compile generated CUDA code into a ptx and execute it at runtime. + +## Setup + +By default, this runtime uses the latest CUDA version. +If the default features are not enabled, the CUDA version detected during build time will be used instead. +To specify a fixed CUDA version, disable the default features and add `cudarc` to your dependencies with the appropriate feature flag: + +```toml +cudarc = { version = "same as burn", features = ["cuda-11040"] } +``` diff --git a/third_party/cubecl-cuda-0.10.0-patched/build.rs b/third_party/cubecl-cuda-0.10.0-patched/build.rs new file mode 100644 index 00000000..d802e63c --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/build.rs @@ -0,0 +1,13 @@ +use cudarc::driver::sys::CUDA_VERSION; + +fn main() { + println!("cargo::rustc-check-cfg=cfg(cuda_12050)"); + println!("cargo::rustc-check-cfg=cfg(cuda_12080)"); + + if CUDA_VERSION >= 12050 { + println!("cargo:rustc-cfg=cuda_12050"); + } + if CUDA_VERSION >= 12080 { + println!("cargo:rustc-cfg=cuda_12080"); + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/cubecl.toml b/third_party/cubecl-cuda-0.10.0-patched/cubecl.toml new file mode 100644 index 00000000..32defd4c --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/cubecl.toml @@ -0,0 +1,2 @@ +[compilation] +check_mode = "validate" diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/command.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/command.rs new file mode 100644 index 00000000..10d85c12 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/command.rs @@ -0,0 +1,667 @@ +use crate::{ + CudaCompiler, + compute::{ + MB, context::CudaContext, io::controller::PinnedMemoryManagedAllocController, + storage::gpu::GpuResource, stream::CudaStreamBackend, sync::Fence, + }, +}; +use cubecl_common::{ + backtrace::BackTrace, + bytes::{AllocationProperty, Bytes}, + stream_id::StreamId, +}; +#[cfg(debug_assertions)] +use cubecl_core::zspace::striding::try_check_pitched_row_major_strides; +use cubecl_core::{ + MemoryUsage, + future::DynFut, + server::{ + Binding, CopyDescriptor, ExecutionMode, Handle, IoError, LaunchError, ProfileError, + ServerError, + }, + zspace::{Shape, Strides, striding::has_pitched_row_major_strides}, +}; +use cubecl_runtime::{ + compiler::CubeTask, + id::KernelId, + logging::ServerLogger, + memory_management::{ManagedMemoryHandle, MemoryAllocationMode, MemoryHandle}, + stream::ResolvedStreams, +}; +use cudarc::driver::sys::{ + CUDA_MEMCPY2D_st, CUmemorytype, CUstream_st, CUtensorMap, cuMemcpy2DAsync_v2, +}; +use std::{ffi::c_void, ops::DerefMut, sync::Arc}; + +#[derive(new)] +/// The `Command` struct encapsulates a CUDA context and a set of resolved CUDA streams, providing an +/// interface for executing GPU-related operations such as memory allocation, data transfers, kernel +/// registration, and task execution. +pub struct Command<'a> { + ctx: &'a mut CudaContext, + pub(crate) streams: ResolvedStreams<'a, CudaStreamBackend>, +} + +impl<'a> Command<'a> { + pub(super) fn external_write_stream(&mut self) -> u64 { + self.streams.current().sys as usize as u64 + } + + /// Retrieves a GPU resource associated with the provided binding. + /// + /// # Parameters + /// + /// * `binding` - The binding specifying the stream, memory, and offsets for the resource. + /// + /// # Returns + /// + /// * `Ok(GpuResource)` - The GPU resource associated with the binding. + /// * `Err(IoError::InvalidHandle)` - If the binding does not correspond to a valid resource. + pub fn resource(&mut self, binding: Binding) -> Result { + self.streams + .get(&binding.stream) + .memory_management_gpu + .get_resource(binding.memory, binding.offset_start, binding.offset_end) + } + + /// Get the stream cursor. + pub fn cursor(&self) -> u64 { + self.streams.cursor + } + + /// Retrieves the gpu memory usage of the current stream. + /// + /// # Returns + /// + /// * The [`MemoryUsage`] struct. + pub fn memory_usage(&mut self) -> MemoryUsage { + self.streams.current().memory_management_gpu.memory_usage() + } + + /// Explicitly cleanup gpu memory on the current stream. + pub fn memory_cleanup(&mut self) { + self.streams.current().memory_management_gpu.cleanup(true) + } + + /// Set the [`MemoryAllocationMode`] for the current stream. + /// + /// # Parameters + /// + /// * `mode` - The allocation mode to be used. + pub fn allocation_mode(&mut self, mode: MemoryAllocationMode) { + self.streams.current().memory_management_gpu.mode(mode) + } + + /// Allocates a new GPU memory buffer of the specified size. + /// + /// # Parameters + /// + /// * `size` - The size of the memory to allocate (in bytes). + /// + /// # Returns + /// + /// * `Ok(Handle)` - A handle to the newly allocated GPU memory. + /// * `Err(IoError)` - If the allocation fails. + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + pub fn reserve(&mut self, size: u64) -> Result { + let handle = self.streams.current().memory_management_gpu.reserve(size)?; + + Ok(handle) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + pub fn empty(&mut self, size: u64) -> Result { + let handle = Handle::new(self.streams.current, size); + let reserved = self.reserve(size)?; + self.bind(reserved, handle.memory.clone()); + + Ok(handle) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + pub fn bind(&mut self, reserved: ManagedMemoryHandle, new: ManagedMemoryHandle) { + let cursor = self.cursor(); + self.streams + .current() + .memory_management_gpu + .bind(reserved, new, cursor) + .unwrap(); + } + + /// Creates a [Bytes] instance from pinned memory, if suitable for the given size. + /// + /// For small data transfers (<= 100 MB) or when explicitly marked as pinned, this function + /// uses pinned memory to optimize performance. For larger transfers, it falls back to regular memory. + /// + /// # Arguments + /// + /// * `size` - The number of bytes to allocate. + /// * `marked_pinned` - Whether to force the use of pinned memory. + /// + /// # Returns + /// + /// A [Bytes] instance of the correct size. + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + pub fn reserve_cpu( + &mut self, + size: usize, + marked_pinned: bool, + origin: Option, + ) -> Bytes { + // Use pinned memory for small transfers (<= 100 MB) or when explicitly marked. + if !marked_pinned && size > 100 * MB { + return Bytes::from_bytes_vec(vec![0; size]); + } + + self.reserve_pinned(size, origin) + .unwrap_or_else(|| Bytes::from_bytes_vec(vec![0; size])) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + fn reserve_pinned(&mut self, size: usize, origin: Option) -> Option { + let stream = match origin { + Some(id) => self.streams.get(&id), + None => self.streams.current(), + }; + let handle = stream.memory_management_cpu.reserve(size as u64).ok()?; + + let binding = MemoryHandle::binding(handle); + let resource = stream + .memory_management_cpu + .get_resource(binding.clone(), None, None) + .ok()?; + + let controller = Box::new(PinnedMemoryManagedAllocController::init(binding, resource)); + // SAFETY: The binding has initialized memory for at least `size` bytes. + Some(unsafe { Bytes::from_controller(controller, size) }) + } + + /// Asynchronously reads data from GPU memory to host memory based on the provided copy descriptors. + /// + /// # Parameters + /// + /// * `descriptors` - A vector of descriptors specifying the source GPU memory and its layout. + /// + /// # Returns + /// + /// * A `Future` resolving to: + /// * `Ok(Vec)` - The data read from the GPU as a vector of byte arrays. + /// * `Err(IoError)` - If the read operation fails. + pub fn read_async( + &mut self, + descriptors: Vec, + ) -> impl Future, ServerError>> + Send + use<> { + let descriptors_moved = descriptors + .iter() + .map(|b| b.handle.clone()) + .collect::>(); + + let result = self.copies_to_bytes(descriptors, true); + let fence = Fence::new(self.streams.current().sys); + + async move { + let sync = fence.wait_sync(); + // Release memory handle. + core::mem::drop(descriptors_moved); + + sync?; + let bytes = result?; + + Ok(bytes) + } + } + + #[allow(unused)] + /// TODO: Read data using the origin stream where the data was allocated. + pub fn read_async_origin( + &mut self, + descriptors: Vec, + ) -> impl Future, IoError>> + Send + use<> { + let results = self.copies_to_bytes_origin(descriptors, true); + + async move { + let (bytes, fences) = results?; + + for fence in fences { + fence.wait_sync(); + } + Ok(bytes) + } + } + + fn copies_to_bytes( + &mut self, + descriptors: Vec, + pinned: bool, + ) -> Result, IoError> { + let mut result = Vec::with_capacity(descriptors.len()); + + for descriptor in descriptors { + result.push(self.copy_to_bytes(descriptor, pinned, None)?); + } + + Ok(result) + } + + fn copies_to_bytes_origin( + &mut self, + descriptors: Vec, + pinned: bool, + ) -> Result<(Vec, Vec), IoError> { + let mut data = Vec::with_capacity(descriptors.len()); + let mut fences = Vec::with_capacity(descriptors.len()); + let mut fenced = Vec::with_capacity(descriptors.len()); + + for descriptor in descriptors { + let stream = descriptor.handle.stream; + let bytes = self.copy_to_bytes(descriptor, pinned, Some(stream))?; + + if !fenced.contains(&stream) { + let fence = Fence::new(self.streams.get(&stream).sys); + fenced.push(stream); + fences.push(fence); + } + + data.push(bytes); + } + + Ok((data, fences)) + } + + pub fn copy_to_bytes( + &mut self, + descriptor: CopyDescriptor, + pinned: bool, + stream_id: Option, + ) -> Result { + let num_bytes = descriptor.shape.iter().product::() * descriptor.elem_size; + let mut bytes = self.reserve_cpu(num_bytes, pinned, stream_id); + self.write_to_cpu(descriptor, &mut bytes, stream_id)?; + + Ok(bytes) + } + + /// Writes data to the host from the GPU memory as specified by the copy descriptor. + /// + /// # Parameters + /// + /// * `descriptor` - Describes the source GPU memory, its shape, strides, and element size. + /// * `bytes` - The host bytes to write from the GPU. + /// + /// # Returns + /// + /// * `Ok(())` - If the write operation succeeds. + /// * `Err(IoError)` - If the strides are invalid or the resource cannot be accessed. + pub fn write_to_cpu( + &mut self, + descriptor: CopyDescriptor, + bytes: &mut Bytes, + stream_id: Option, + ) -> Result<(), IoError> { + let CopyDescriptor { + handle: binding, + shape, + strides, + elem_size, + } = descriptor; + + if !has_pitched_row_major_strides(&shape, &strides) { + return Err(IoError::UnsupportedStrides { + backtrace: BackTrace::capture(), + }); + } + + let resource = self.resource(binding)?; + let stream = match stream_id { + Some(id) => self.streams.get(&id), + None => self.streams.current(), + }; + + // SAFETY: `resource.ptr` is a valid device pointer obtained from the memory manager, + // `stream.sys` is an initialized CUDA stream, and `bytes` is pre-allocated with + // sufficient capacity for the copy. + unsafe { + write_to_cpu(&shape, &strides, elem_size, bytes, resource.ptr, stream.sys)?; + } + + Ok(()) + } + + /// Registers an error on the stream. + pub fn error(&mut self, error: ServerError) { + let stream = self.streams.current(); + stream.errors.push(error); + } + + /// Writes data from the host to GPU memory as specified by the copy descriptor. + /// + /// # Parameters + /// + /// * `descriptor` - Describes the destination GPU memory, its shape, strides, and element size. + /// * `data` - The host data to write to the GPU. + /// + /// # Returns + /// + /// * `Ok(())` - If the write operation succeeds. + /// * `Err(IoError)` - If the strides are invalid or the resource cannot be accessed. + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, descriptor, data)) + )] + pub fn write_to_gpu(&mut self, descriptor: CopyDescriptor, data: Bytes) -> Result<(), IoError> { + let CopyDescriptor { + handle, + shape, + strides, + elem_size, + } = descriptor; + if !has_pitched_row_major_strides(&shape, &strides) { + return Err(IoError::UnsupportedStrides { + backtrace: BackTrace::capture(), + }); + } + + let resource = self.resource(handle)?; + + let size = data.len(); + let data = match data.property() { + AllocationProperty::File => { + let mut buffer = self.reserve_pinned(size, None).unwrap(); + data.copy_into(&mut buffer); + buffer + } + _ => data, + }; + let current = self.streams.current(); + + // SAFETY: `resource.ptr` is a valid GPU allocation, `data` is a valid host buffer, + // and `current.sys` is an initialized CUDA stream. The shape/strides have been + // validated above to be pitched row-major. + unsafe { + write_to_gpu( + &shape, + &strides, + elem_size, + &data, + resource.ptr, + current.sys, + ) + }?; + + current.drop_queue.push(data); + + Ok(()) + } + + /// Allocates a new GPU memory buffer and immediately copies contiguous host data into it. + /// + /// # Parameters + /// + /// * `data` - The host data to copy to the GPU. + /// + /// # Returns + /// + /// * `Ok(Handle)` - A handle to the newly allocated and populated GPU memory. + /// * `Err(IoError)` - If the allocation or data copy fails. + pub fn create_with_data(&mut self, data: &[u8]) -> Result { + let mut staging = + self.reserve_pinned(data.len(), None) + .ok_or_else(|| IoError::Unknown { + backtrace: BackTrace::capture(), + description: "Unable to reserve pinned memory".into(), + })?; + + staging.copy_from_slice(data); + + let handle = self.empty(staging.len() as u64)?; + + self.write_to_gpu( + CopyDescriptor { + handle: handle.clone().binding(), + shape: [data.len()].into(), + strides: [1].into(), + elem_size: 1, + }, + staging, + )?; + + Ok(handle) + } + + /// Synchronizes the current stream, ensuring all pending operations are complete. + /// + /// # Returns + /// + /// * A `DynFut<()>` future that resolves when the stream is synchronized. + pub fn sync(&mut self) -> DynFut> { + let fence = Fence::new(self.streams.current().sys); + + Box::pin(async { fence.wait_sync() }) + } + + /// Executes a registered CUDA kernel with the specified parameters. + /// + /// # Parameters + /// + /// * `kernel_id` - The identifier of the kernel to execute. + /// * `kernel` - The cube task to compile if not cached. + /// * `mode` - The execution mode for the current kernel. + /// * `dispatch_count` - The number of thread blocks in the x, y, and z dimensions. + /// * `tensor_maps` - Tensor maps for structured memory access. + /// * `resources` - GPU resources (e.g., buffers) used by the kernel. + /// * `scalars` - Scalar arguments passed to the kernel. + /// * `logger` - The logger to use to write compilation & runtime info. + /// + /// # Panics + /// + /// * If the execution fails, with an error message or profiling error. + #[allow(clippy::too_many_arguments)] + pub fn kernel( + &mut self, + kernel_id: KernelId, + kernel: Box>, + mode: ExecutionMode, + dispatch_count: (u32, u32, u32), + tensor_maps: &[CUtensorMap], + resources: &[GpuResource], + const_info: Option<*mut c_void>, + logger: Arc, + ) -> Result<(), LaunchError> { + if !self.ctx.module_names.contains_key(&kernel_id) { + self.ctx.compile_kernel(&kernel_id, kernel, mode, logger)?; + } + + let stream = self.streams.current(); + + let result = self.ctx.execute_task( + stream, + kernel_id, + dispatch_count, + tensor_maps, + resources, + const_info, + ); + + if stream.drop_queue.should_flush() { + stream.drop_queue.flush(|| Fence::new(stream.sys)); + } + + if let Err(err) = result { + match self.ctx.timestamps.is_empty() { + true => return Err(err), + false => self.ctx.timestamps.error(ProfileError::Launch(err)), + } + }; + Ok(()) + } +} + +/// Internal write to GPU command. +/// +/// Writes data from a CPU buffer to a CUDA resource. +/// +/// # Safety +/// +/// - `dst_ptr` must be a valid CUDA device pointer with sufficient space for `data`. +/// - `stream` must be a valid, initialized CUDA stream. +/// - `data` must remain valid until the stream is synchronized. +/// - `shape`/`strides` must describe a valid pitched row-major layout (debug-asserted). +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(strides, data, dst_ptr, stream)) +)] +pub(crate) unsafe fn write_to_gpu( + shape: &Shape, + strides: &Strides, + elem_size: usize, + data: &[u8], + dst_ptr: u64, + stream: *mut CUstream_st, +) -> Result<(), IoError> { + #[cfg(debug_assertions)] + try_check_pitched_row_major_strides(shape, strides).map_err(|e| IoError::Unknown { + description: format!("write_to_gpu: invalid strides: {e}"), + backtrace: BackTrace::capture(), + })?; + + let rank = shape.len(); + if rank <= 1 { + // SAFETY: For rank <= 1 data is contiguous. `dst_ptr` is a valid device pointer + // and `data` is a valid host slice. + unsafe { + cudarc::driver::result::memcpy_htod_async(dst_ptr, data, stream).map_err(|e| { + IoError::Unknown { + description: format!("CUDA memcpy_htod failed: {e}"), + backtrace: BackTrace::capture(), + } + }) + } + } else { + // As we've enforced that the strides are contiguous row-major, + // and we know that the rank >= 2, we can construct a 2D view + // for the aligned GPU pitched memcpy. + + let dim_x_shape = shape[rank - 1]; + let width_bytes = dim_x_shape * elem_size; + + // the second "dim"'s shape is the product of the rest of the space. + let dim_y_shape: usize = shape[..rank - 1].iter().product(); + let pitch = strides[rank - 2] * elem_size; + + let cpy = CUDA_MEMCPY2D_st { + srcMemoryType: CUmemorytype::CU_MEMORYTYPE_HOST, + srcHost: data.as_ptr() as *const c_void, + srcPitch: width_bytes, + dstMemoryType: CUmemorytype::CU_MEMORYTYPE_DEVICE, + dstDevice: dst_ptr, + dstPitch: pitch, + WidthInBytes: width_bytes, + Height: dim_y_shape, + srcXInBytes: Default::default(), + srcY: Default::default(), + srcDevice: Default::default(), + srcArray: Default::default(), + dstXInBytes: Default::default(), + dstY: Default::default(), + dstHost: Default::default(), + dstArray: Default::default(), + }; + + // SAFETY: The `CUDA_MEMCPY2D_st` is fully initialized with valid source/dest + // pointers, memory types, and dimensions derived from the validated shape/strides. + unsafe { + cuMemcpy2DAsync_v2(&cpy, stream) + .result() + .map_err(|e| IoError::Unknown { + description: format!("CUDA memcpy failed: {e}"), + backtrace: BackTrace::capture(), + }) + } + } +} + +/// Internal write to CPU command. +/// +/// Writes data from a CUDA resource to a CPU buffer. +/// +/// # Safety +/// +/// - `resource_ptr` must be a valid CUDA device pointer with at least `bytes.len()` readable bytes. +/// - `stream` must be a valid, initialized CUDA stream. +/// - `bytes` must have sufficient capacity for the copy. +/// - The caller must synchronize the stream before reading from `bytes`. +/// - `shape`/`strides` must describe a valid pitched row-major layout (debug-asserted). +#[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(strides, bytes, resource_ptr, stream)) +)] +pub(crate) unsafe fn write_to_cpu( + shape: &Shape, + strides: &Strides, + elem_size: usize, + bytes: &mut Bytes, + resource_ptr: u64, + stream: *mut CUstream_st, +) -> Result<(), IoError> { + #[cfg(debug_assertions)] + try_check_pitched_row_major_strides(shape, strides).map_err(|e| IoError::Unknown { + description: format!("write_to_cpu: invalid strides: {e}"), + backtrace: BackTrace::capture(), + })?; + + let rank = shape.len(); + let bytes = bytes.deref_mut(); + if rank <= 1 { + // SAFETY: For rank <= 1 data is contiguous. `resource_ptr` is a valid device pointer + // and `bytes` has sufficient capacity. + unsafe { + cudarc::driver::result::memcpy_dtoh_async(bytes, resource_ptr, stream).map_err(|e| { + IoError::Unknown { + description: format!("CUDA memcpy_dtoh failed: {e}"), + backtrace: BackTrace::capture(), + } + }) + } + } else { + // As we've enforced that the strides are contiguous row-major, + // and we know that the rank >= 2, we can construct a 2D view + // for the aligned GPU pitched memcpy. + + let dim_x_shape = shape[rank - 1]; + let width_bytes = dim_x_shape * elem_size; + + // the second "dim"'s shape is the product of the rest of the space. + let dim_y_shape: usize = shape[..rank - 1].iter().product(); + let pitch = strides[rank - 2] * elem_size; + + let cpy = CUDA_MEMCPY2D_st { + srcMemoryType: CUmemorytype::CU_MEMORYTYPE_DEVICE, + srcDevice: resource_ptr, + srcPitch: pitch, + dstMemoryType: CUmemorytype::CU_MEMORYTYPE_HOST, + dstHost: bytes.as_mut_ptr() as *mut c_void, + dstPitch: width_bytes, + WidthInBytes: width_bytes, + Height: dim_y_shape, + srcXInBytes: Default::default(), + srcY: Default::default(), + srcArray: Default::default(), + dstXInBytes: Default::default(), + dstY: Default::default(), + dstArray: Default::default(), + srcHost: Default::default(), + dstDevice: Default::default(), + }; + + // SAFETY: The `CUDA_MEMCPY2D_st` is fully initialized with valid source/dest + // pointers, memory types, and dimensions derived from the validated shape/strides. + unsafe { + cuMemcpy2DAsync_v2(&cpy, stream) + .result() + .map_err(|e| IoError::Unknown { + description: format!("CUDA 2D memcpy failed: {e}"), + backtrace: BackTrace::capture(), + }) + } + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/communication.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/communication.rs new file mode 100644 index 00000000..5cf35a81 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/communication.rs @@ -0,0 +1,106 @@ +use std::{collections::HashMap, sync::OnceLock}; + +use cubecl_core::{ + device::DeviceId, + ir::ElemType, + server::{CommunicationId, ReduceOperation}, + stub::Mutex, +}; + +/// Global state map from [`CommunicationId`] to boxed [`cudarc::nccl::sys::ncclUniqueId`]. +static UNIQUE_IDS_MAP: OnceLock>> = + OnceLock::new(); + +pub(crate) fn get_nccl_comm_id(device_ids: Vec) -> cudarc::nccl::sys::ncclUniqueId { + let mut unique_ids_map = UNIQUE_IDS_MAP.get_or_init(Default::default).lock().unwrap(); + let comm_id = CommunicationId::from(device_ids); + match unique_ids_map.get_mut(&comm_id) { + Some(id) => *id, + None => { + let id = cudarc::nccl::result::get_uniqueid().unwrap(); + unique_ids_map.insert(comm_id, id); + id + } + } +} + +pub(crate) fn to_nccl_op(op: ReduceOperation) -> cudarc::nccl::sys::ncclRedOp_t { + match op { + ReduceOperation::Sum => cudarc::nccl::sys::ncclRedOp_t::ncclSum, + ReduceOperation::Mean => cudarc::nccl::sys::ncclRedOp_t::ncclAvg, + } +} + +pub(crate) fn get_nccl_dtype_count( + dtype: ElemType, + size: u64, +) -> (cudarc::nccl::sys::ncclDataType_t, usize) { + match dtype { + ElemType::Float( + cubecl_core::ir::FloatKind::E2M1 + | cubecl_core::ir::FloatKind::E2M3 + | cubecl_core::ir::FloatKind::E3M2 + | cubecl_core::ir::FloatKind::UE8M0, + ) => panic!("Minifloat not supported in NCCL"), + ElemType::Float(cubecl_core::ir::FloatKind::E4M3) => ( + cudarc::nccl::sys::ncclDataType_t::ncclFloat8e4m3, + size as usize, + ), + ElemType::Float(cubecl_core::ir::FloatKind::E5M2) => ( + cudarc::nccl::sys::ncclDataType_t::ncclFloat8e5m2, + size as usize, + ), + ElemType::Float(cubecl_core::ir::FloatKind::F16) => ( + cudarc::nccl::sys::ncclDataType_t::ncclFloat16, + (size / 2) as usize, + ), + ElemType::Float(cubecl_core::ir::FloatKind::BF16) => ( + cudarc::nccl::sys::ncclDataType_t::ncclBfloat16, + (size / 2) as usize, + ), + ElemType::Float(cubecl_core::ir::FloatKind::Flex32) => { + panic!("NCCL doesn't support Flex32 format.") + } + + ElemType::Float(cubecl_core::ir::FloatKind::F32) => ( + cudarc::nccl::sys::ncclDataType_t::ncclFloat32, + (size / 4) as usize, + ), + ElemType::Float(cubecl_core::ir::FloatKind::TF32) => { + panic!("NCCL doesn't support TF32 format.") + } + ElemType::Float(cubecl_core::ir::FloatKind::F64) => ( + cudarc::nccl::sys::ncclDataType_t::ncclFloat64, + (size / 8) as usize, + ), + ElemType::Int(int_kind) => match int_kind { + cubecl_core::ir::IntKind::I8 => { + (cudarc::nccl::sys::ncclDataType_t::ncclInt8, size as usize) + } + cubecl_core::ir::IntKind::I16 => panic!("NCCL doesn't support Int16 format."), + cubecl_core::ir::IntKind::I32 => ( + cudarc::nccl::sys::ncclDataType_t::ncclInt32, + (size / 4) as usize, + ), + cubecl_core::ir::IntKind::I64 => ( + cudarc::nccl::sys::ncclDataType_t::ncclInt64, + (size / 8) as usize, + ), + }, + ElemType::UInt(uint_kind) => match uint_kind { + cubecl_core::ir::UIntKind::U8 => { + (cudarc::nccl::sys::ncclDataType_t::ncclUint8, size as usize) + } + cubecl_core::ir::UIntKind::U16 => panic!("NCCL doesn't support UInt16 format."), + cubecl_core::ir::UIntKind::U32 => ( + cudarc::nccl::sys::ncclDataType_t::ncclUint32, + (size / 4) as usize, + ), + cubecl_core::ir::UIntKind::U64 => ( + cudarc::nccl::sys::ncclDataType_t::ncclUint64, + (size / 8) as usize, + ), + }, + ElemType::Bool => panic!("NCCL doesn't support Bool format."), + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/context.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/context.rs new file mode 100644 index 00000000..200b0d78 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/context.rs @@ -0,0 +1,344 @@ +use cubecl_common::backtrace::BackTrace; +use cubecl_cpp::formatter::format_cpp; +use cubecl_cpp::{cuda::arch::CudaArchitecture, shared::CompilationOptions}; +use cubecl_runtime::{ + compiler::CompilationError, + validation::{validate_cube_dim, validate_units}, +}; + +use super::storage::gpu::GpuResource; +use crate::{CudaCompiler, compute::stream::Stream}; +use crate::{ + CudaComputeKernel, + install::{cccl_include_path, include_path}, +}; +use cubecl_core::{ + compilation_cache::CompilationCache, + hash::StableHash, + server::ResourceLimitError, + {ir::DeviceProperties, prelude::*}, +}; +use cubecl_runtime::timestamp_profiler::TimestampProfiler; +use cubecl_runtime::{compiler::CubeTask, logging::ServerLogger}; +use cudarc::driver::DriverError; +use cudarc::driver::sys::CUfunc_st; +use cudarc::driver::sys::{CUctx_st, CUfunction_attribute, CUtensorMap}; +use std::collections::HashMap; +use std::ffi::CString; +use std::ffi::c_char; +use std::str::FromStr; +use std::sync::Arc; +use std::{ffi::CStr, os::raw::c_void}; + +use cubecl_common::cache::CacheOption; + +#[derive(Debug)] +pub(crate) struct CudaContext { + pub context: *mut CUctx_st, + pub module_names: HashMap, + ptx_cache: Option>, + pub timestamps: TimestampProfiler, + pub arch: CudaArchitecture, + pub compilation_options: CompilationOptions, + pub properties: DeviceProperties, +} + +#[derive(Debug)] +pub struct CompiledKernel { + cube_dim: CubeDim, + shared_mem_bytes: usize, + func: *mut CUfunc_st, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq, Clone)] +pub struct PtxCacheEntry { + entrypoint_name: String, + shared_mem_bytes: usize, + ptx: Vec, +} + +impl CudaContext { + pub fn new( + compilation_options: CompilationOptions, + properties: DeviceProperties, + context: *mut CUctx_st, + arch: CudaArchitecture, + ) -> Self { + Self { + context, + module_names: HashMap::new(), + ptx_cache: { + use cubecl_runtime::config::RuntimeConfig; + let config = cubecl_runtime::config::CubeClRuntimeConfig::get(); + if let Some(cache) = &config.compilation.cache { + let root = cache.root(); + Some(CompilationCache::new( + "ptx", + CacheOption::default().name("cuda").root(root), + )) + } else { + None + } + }, + arch, + timestamps: TimestampProfiler::default(), + compilation_options, + properties, + } + } + + /// Switches the current CUDA context to this context. + pub fn unsafe_set_current(&self) -> Result<(), DriverError> { + // SAFETY: `self.context` is a valid CUDA context obtained from `primary_ctx::retain` + // during server initialization and remains valid for the server's lifetime. + unsafe { cudarc::driver::result::ctx::set_current(self.context) } + } + + pub fn compile_kernel( + &mut self, + kernel_id: &KernelId, + kernel: Box>, + mode: ExecutionMode, + logger: Arc, + ) -> Result<(), LaunchError> { + let hash = if let Some(cache) = &self.ptx_cache { + let hash = kernel_id.stable_hash(); + + if let Some(entry) = cache.get(&hash) { + log::trace!("Using PTX cache"); + + self.load_ptx( + entry.ptx.clone(), + kernel_id.clone(), + entry.entrypoint_name.clone(), + kernel_id.cube_dim, + entry.shared_mem_bytes, + )?; + return Ok(()); + } + Some(hash) + } else { + None + }; + + log::trace!("Compiling kernel"); + + validate_cube_dim(&self.properties, kernel_id)?; + validate_units(&self.properties, kernel_id)?; + + let mut kernel_compiled = kernel.compile( + &mut Default::default(), + &self.compilation_options, + mode, + kernel.address_type(), + )?; + + self.validate_shared(&kernel_compiled.repr)?; + + if logger.compilation_activated() { + kernel_compiled.debug_info = Some(DebugInformation::new("cpp", kernel_id.clone())); + + if let Ok(formatted) = format_cpp(&kernel_compiled.source) { + kernel_compiled.source = formatted; + } + } + + let cube_dim = kernel_compiled.cube_dim; + let arch = if self.arch.version >= 90 { + format!("--gpu-architecture=sm_{}a", self.arch) + } else { + format!("--gpu-architecture=sm_{}", self.arch) + }; + + let include_path = include_path(); + let include_option = format!("--include-path={}", include_path.to_str().unwrap()); + let cccl_include_path = cccl_include_path(); + let cccl_include_option = format!("--include-path={}", cccl_include_path.to_str().unwrap()); + let mut options = vec![arch.as_str(), include_option.as_str(), "-lineinfo"]; + if cccl_include_path.exists() { + options.push(&cccl_include_option); + } + + logger.log_compilation(&kernel_compiled); + + // SAFETY: Calling NVRTC FFI to create, compile, and extract PTX from a program. + // The `CString` source is null-terminated and outlives the program. On compilation + // failure, the error log is retrieved and reported before returning. + let ptx = unsafe { + // I'd like to set the name to the kernel name, but keep getting UTF-8 errors so let's + // leave it `None` for now + let source = CString::from_str(&kernel_compiled.source).unwrap(); + let program = + cudarc::nvrtc::result::create_program(source.as_c_str(), None).map_err(|err| { + CompilationError::Generic { + reason: format!("{err:?}"), + backtrace: BackTrace::capture(), + } + })?; + if cudarc::nvrtc::result::compile_program(program, &options).is_err() { + let log_raw = cudarc::nvrtc::result::get_program_log(program).map_err(|err| { + CompilationError::Generic { + reason: format!("{err:?}"), + backtrace: BackTrace::capture(), + } + })?; + + let log_ptr = log_raw.as_ptr(); + let log = CStr::from_ptr(log_ptr).to_str().unwrap(); + let mut message = "[Compilation Error] ".to_string(); + for line in log.split('\n') { + if !line.is_empty() { + message += format!("\n {line}").as_str(); + } + } + let source = kernel + .compile( + &mut Default::default(), + &self.compilation_options, + mode, + kernel.address_type(), + )? + .source; + Err(CompilationError::Generic { + reason: format!("{message}\n[Source] \n{source}"), + backtrace: BackTrace::capture(), + })?; + }; + cudarc::nvrtc::result::get_ptx(program).map_err(|err| CompilationError::Generic { + reason: format!("{err:?}"), + backtrace: BackTrace::capture(), + })? + }; + + let repr = kernel_compiled.repr.unwrap(); + + if let Some(cache) = &mut self.ptx_cache { + let result = cache.insert( + hash.unwrap(), + PtxCacheEntry { + entrypoint_name: kernel_compiled.entrypoint_name.clone(), + shared_mem_bytes: repr.shared_memory_size(), + ptx: ptx.clone(), + }, + ); + if let Err(err) = result { + log::warn!("Unable to save the ptx {err:?}"); + } + } + + self.load_ptx( + ptx, + kernel_id.clone(), + kernel_compiled.entrypoint_name, + cube_dim, + repr.shared_memory_size(), + )?; + Ok(()) + } + + fn load_ptx( + &mut self, + ptx: Vec, + kernel_id: KernelId, + entrypoint_name: String, + cube_dim: CubeDim, + shared_mem_bytes: usize, + ) -> Result<(), CompilationError> { + let func_name = CString::new(entrypoint_name).unwrap(); + // SAFETY: `ptx` is a valid null-terminated PTX binary from NVRTC. `func_name` is a + // null-terminated `CString` matching the kernel entry point in the compiled module. + let func = unsafe { + let module = cudarc::driver::result::module::load_data(ptx.as_ptr() as *const _) + .map_err(|err| CompilationError::Generic { + reason: format!("Unable to load the PTX: {err:?}"), + backtrace: BackTrace::capture(), + })?; + + cudarc::driver::result::module::get_function(module, func_name).map_err(|err| { + CompilationError::Generic { + reason: format!("Unable to fetch the function from the module: {err:?}"), + backtrace: BackTrace::capture(), + } + })? + }; + + self.module_names.insert( + kernel_id.clone(), + CompiledKernel { + cube_dim, + shared_mem_bytes, + func, + }, + ); + + Ok(()) + } + + pub fn execute_task( + &mut self, + stream: &mut Stream, + kernel_id: KernelId, + dispatch_count: (u32, u32, u32), + tensor_maps: &[CUtensorMap], + resources: &[GpuResource], + const_info: Option<*mut c_void>, + ) -> Result<(), LaunchError> { + let mut bindings = tensor_maps + .iter() + .map(|map| map as *const _ as *mut c_void) + .collect::>(); + bindings.extend(resources.iter().map(|memory| memory.binding)); + bindings.extend(const_info); + + let kernel = self.module_names.get(&kernel_id).unwrap(); + let cube_dim = kernel.cube_dim; + // SAFETY: `kernel.func` is a valid function handle from a loaded module. + // `stream.sys` is a valid CUDA stream. `bindings` contains valid device pointers + // for all kernel arguments. The dispatch and cube dimensions are validated by + // the caller. + unsafe { + cudarc::driver::result::function::set_function_attribute( + kernel.func, + CUfunction_attribute::CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, + kernel.shared_mem_bytes as i32, + ) + .map_err(|err| LaunchError::Unknown { + reason: format!("{err:?}"), + backtrace: BackTrace::capture(), + })?; + cudarc::driver::result::launch_kernel( + kernel.func, + dispatch_count, + (cube_dim.x, cube_dim.y, cube_dim.z), + // Shared memory is collected into a single buffer, with each shared memory being + // an offset pointer + kernel.shared_mem_bytes as u32, + stream.sys, + &mut bindings, + ) + .map_err(|err| LaunchError::Unknown { + reason: format!("{err:?}"), + backtrace: BackTrace::capture(), + })?; + }; + + Ok(()) + } + + fn validate_shared(&self, repr: &Option) -> Result<(), LaunchError> { + let requested = repr.as_ref().map(|repr| repr.shared_memory_size()); + let max = self.properties.hardware.max_shared_memory_size; + if let Some(requested) = requested + && requested > max + { + Err(ResourceLimitError::SharedMemory { + requested, + max, + backtrace: BackTrace::capture(), + } + .into()) + } else { + Ok(()) + } + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/io/controller.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/io/controller.rs new file mode 100644 index 00000000..3e3fcd18 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/io/controller.rs @@ -0,0 +1,69 @@ +use crate::compute::storage::cpu::{PINNED_MEMORY_ALIGNMENT, PinnedMemoryResource}; +use cubecl_common::bytes::{AllocationController, AllocationProperty}; +use cubecl_runtime::memory_management::ManagedMemoryBinding; + +/// Controller for managing pinned (page-locked) host memory allocations. +/// +/// This struct ensures that the associated memory binding remains alive until +/// explicitly deallocated, allowing the pinned memory to be reused for other memory operations. +pub struct PinnedMemoryManagedAllocController { + resource: PinnedMemoryResource, + /// The memory binding, kept alive until deallocation. + _binding: ManagedMemoryBinding, +} + +impl PinnedMemoryManagedAllocController { + /// Creates a new allocation controller for pinned host memory. + /// + /// # Arguments + /// + /// * `binding` - The memory binding for the pinned memory. + /// * `resource` - The pinned memory resource to manage. + /// + /// # Returns + /// + /// The controller and the corresponding `Allocation`. + pub fn init(binding: ManagedMemoryBinding, resource: PinnedMemoryResource) -> Self { + Self { + _binding: binding, + resource, + } + } +} + +impl AllocationController for PinnedMemoryManagedAllocController { + fn alloc_align(&self) -> usize { + PINNED_MEMORY_ALIGNMENT + } + + unsafe fn memory_mut(&mut self) -> &mut [std::mem::MaybeUninit] { + // SAFETY: + // - The ptr is valid while the binding is alive. + // - The resource is allocated with the size of size. + // - MaybeUninit has the same layout as u8. + // - Caller has to promise to only write initialized data to this slice. + unsafe { + std::slice::from_raw_parts_mut( + self.resource.ptr as *mut std::mem::MaybeUninit, + self.resource.size, + ) + } + } + + fn memory(&self) -> &[std::mem::MaybeUninit] { + // SAFETY: + // - The ptr is valid while the binding is alive. + // - The resource is allocated with the size of size. + // - MaybeUninit has the same layout as u8. + unsafe { + std::slice::from_raw_parts( + self.resource.ptr as *mut std::mem::MaybeUninit, + self.resource.size, + ) + } + } + + fn property(&self) -> AllocationProperty { + AllocationProperty::Pinned + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/io/mod.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/io/mod.rs new file mode 100644 index 00000000..6fb17308 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/io/mod.rs @@ -0,0 +1 @@ +pub(crate) mod controller; diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/mod.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/mod.rs new file mode 100644 index 00000000..c595738e --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/mod.rs @@ -0,0 +1,30 @@ +pub(crate) mod command; +pub(crate) mod communication; +pub(crate) mod context; +pub(crate) mod io; +pub(crate) mod storage; +pub(crate) mod stream; +pub(crate) mod sync; + +mod server; + +pub use server::*; + +/// Creates a `Vec` of the given length with uninitialized elements. +/// +/// # Safety +/// +/// The caller must initialize all elements before reading them. Reading uninitialized +/// memory is undefined behavior. `I` must be valid for any bit pattern (e.g. integer types). +#[allow(clippy::uninit_vec)] +pub unsafe fn uninit_vec(len: usize) -> Vec { + let mut data = Vec::with_capacity(len); + + // SAFETY: The capacity was set to `len` above, so setting the length to `len` is valid. + // The caller is responsible for initializing all elements before reading them. + unsafe { + data.set_len(len); + }; + + data +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/server.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/server.rs new file mode 100644 index 00000000..4ce8dfbf --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/server.rs @@ -0,0 +1,1167 @@ +use super::storage::gpu::{GpuResource, GpuStorage}; +use crate::{ + CudaCompiler, + compute::{ + command::Command, + communication::{get_nccl_comm_id, get_nccl_dtype_count, to_nccl_op}, + context::CudaContext, + stream::CudaStreamBackend, + sync::Fence, + }, +}; +use cubecl_common::{ + backtrace::BackTrace, bytes::Bytes, profile::ProfileDuration, stream_id::StreamId, +}; +use cubecl_core::{ + MemoryConfiguration, + device::DeviceId, + future::{self, DynFut}, + ir::{ElemType, FloatKind, IntKind, MemoryDeviceProperties, StorageType, UIntKind}, + prelude::*, + server::{ + Binding, CommunicationId, CopyDescriptor, Handle, KernelArguments, LaunchError, + ProfileError, ProfilingToken, ReduceOperation, ServerCommunication, ServerError, + ServerUtilities, StreamErrorMode, TensorMapBinding, TensorMapMeta, + }, +}; +use cubecl_runtime::{ + allocator::PitchedMemoryLayoutPolicy, + compiler::CubeTask, + config::{CubeClRuntimeConfig, RuntimeConfig}, + logging::ServerLogger, + memory_management::{ManagedMemoryHandle, MemoryAllocationMode, MemoryUsage}, + server::{ComputeServer, ExternalWriteServer}, + storage::{ComputeStorage, ManagedResource}, + stream::MultiStream, +}; +use cudarc::driver::sys::{ + CUstream_st, CUtensorMapDataType, CUtensorMapFloatOOBfill, CUtensorMapInterleave, + CUtensorMapL2promotion, CUtensorMapSwizzle, cuTensorMapEncodeIm2col, cuTensorMapEncodeTiled, +}; +use std::{ + collections::{HashMap, hash_map::Entry}, + ffi::c_void, + mem::MaybeUninit, + sync::Arc, +}; + +pub(crate) const MB: usize = 1024 * 1024; + +#[derive(Debug)] +pub struct CudaServer { + ctx: CudaContext, + device_id: DeviceId, + streams: MultiStream, + utilities: Arc>, + comm_stream: *mut CUstream_st, + communicators: HashMap, +} + +// SAFETY: `CudaServer` is only accessed from one thread at a time via the `DeviceHandle`, +// which serializes all server access. The CUDA context, streams, and NCCL communicators +// it manages are never shared across threads without synchronization. +unsafe impl Send for CudaServer {} + +impl ExternalWriteServer for CudaServer { + fn external_write_stream( + &mut self, + binding: Binding, + stream_id: StreamId, + ) -> Result { + let mut command = self.command( + stream_id, + core::iter::once(&binding), + StreamErrorMode { + ignore: false, + flush: false, + }, + )?; + Ok(command.external_write_stream()) + } +} + +impl ComputeServer for CudaServer { + type Kernel = Box>; + type Storage = GpuStorage; + type MemoryLayoutPolicy = PitchedMemoryLayoutPolicy; + type Info = (); + + fn logger(&self) -> Arc { + self.streams.logger.clone() + } + + fn staging(&mut self, sizes: &[usize], stream_id: StreamId) -> Result, ServerError> { + let mut command = self.command_no_inputs( + stream_id, + StreamErrorMode { + ignore: true, + flush: false, + }, + )?; + + Ok(sizes + .iter() + .map(|size| command.reserve_cpu(*size, true, None)) + .collect()) + } + + fn utilities(&self) -> Arc> { + self.utilities.clone() + } + + fn read( + &mut self, + descriptors: Vec, + stream_id: StreamId, + ) -> DynFut, ServerError>> { + match self.command( + stream_id, + descriptors.iter().map(|d| &d.handle), + StreamErrorMode { + ignore: false, + flush: true, + }, + ) { + Ok(mut command) => Box::pin(command.read_async(descriptors)), + Err(err) => Box::pin(async move { Err(err) }), + } + } + + fn initialize_memory(&mut self, memory: ManagedMemoryHandle, size: u64, stream_id: StreamId) { + let mut command = match self.command_no_inputs( + stream_id, + StreamErrorMode { + ignore: true, + flush: false, + }, + ) { + Ok(val) => val, + Err(err) => unreachable!("{err:?}"), + }; + + let reserved = command.reserve(size).unwrap(); + command.bind(reserved, memory); + } + + fn write(&mut self, descriptors: Vec<(CopyDescriptor, Bytes)>, stream_id: StreamId) { + let mut command = match self.command( + stream_id, + descriptors.iter().map(|desc| &desc.0.handle), + StreamErrorMode { + ignore: true, + flush: false, + }, + ) { + Ok(val) => val, + Err(err) => unreachable!("{err:?}"), + }; + + for (descriptor, data) in descriptors { + if let Err(err) = command.write_to_gpu(descriptor, data) { + command.error(err.into()); + return; + } + } + } + + unsafe fn launch( + &mut self, + kernel: Self::Kernel, + count: CubeCount, + bindings: KernelArguments, + mode: ExecutionMode, + stream_id: StreamId, + ) { + if let Err(err) = self.launch_checked(kernel, count, bindings, mode, stream_id) { + let mut stream = match self.streams.resolve(stream_id, [].into_iter(), false) { + Ok(stream) => stream, + Err(err) => unreachable!("{err:?}"), + }; + stream.current().errors.push(err); + } + } + + fn flush(&mut self, stream_id: StreamId) -> Result<(), ServerError> { + let mut command = self.command_no_inputs( + stream_id, + StreamErrorMode { + ignore: false, + flush: true, + }, + )?; + + let current = command.streams.current(); + current.drop_queue.flush(|| Fence::new(current.sys)); + current.memory_management_gpu.storage().flush(); + + Ok(()) + } + + fn sync(&mut self, stream_id: StreamId) -> DynFut> { + let command = self.command_no_inputs( + stream_id, + StreamErrorMode { + ignore: false, + flush: true, + }, + ); + + match command { + Ok(mut command) => command.sync(), + Err(err) => Box::pin(async { Err(err) }), + } + } + + fn start_profile(&mut self, stream_id: StreamId) -> Result { + cubecl_common::future::block_on(self.sync(stream_id))?; + Ok(self.ctx.timestamps.start()) + } + + fn end_profile( + &mut self, + stream_id: StreamId, + token: ProfilingToken, + ) -> Result { + if let Err(err) = cubecl_common::future::block_on(self.sync(stream_id)) { + self.ctx + .timestamps + .error(ProfileError::Server(Box::new(err))); + } + self.ctx.timestamps.stop(token) + } + + fn get_resource( + &mut self, + binding: Binding, + stream_id: StreamId, + ) -> Result, ServerError> { + let mut command = self.command( + stream_id, + [&binding].into_iter(), + StreamErrorMode { + ignore: true, + flush: false, + }, + )?; + let memory = binding.memory.clone(); + let resource = command.resource(binding)?; + + Ok(ManagedResource::new(memory, resource)) + } + + fn memory_usage(&mut self, stream_id: StreamId) -> Result { + let mut command = self.command_no_inputs( + stream_id, + StreamErrorMode { + ignore: false, + flush: false, + }, + )?; + Ok(command.memory_usage()) + } + + fn memory_cleanup(&mut self, stream_id: StreamId) { + let mut command = match self.command_no_inputs( + stream_id, + StreamErrorMode { + ignore: true, + flush: false, + }, + ) { + Ok(val) => val, + Err(err) => unreachable!("{err:?}"), + }; + command.memory_cleanup() + } + + fn allocation_mode(&mut self, mode: MemoryAllocationMode, stream_id: StreamId) { + let mut command = match self.command_no_inputs( + stream_id, + StreamErrorMode { + ignore: true, + flush: false, + }, + ) { + Ok(val) => val, + Err(err) => unreachable!("{err:?}"), + }; + command.allocation_mode(mode) + } +} + +impl ServerCommunication for CudaServer { + const SERVER_COMM_ENABLED: bool = true; + + fn comm_init(&mut self, device_ids: Vec) -> Result<(), ServerError> { + let id = CommunicationId::from(device_ids.clone()); + if let Entry::Vacant(e) = self.communicators.entry(id.clone()) { + let mut comm = MaybeUninit::uninit(); + let mut device_ids = device_ids.clone(); + device_ids.sort(); + let rank = device_ids + .iter() + .position(|id| id.index_id == self.device_id.index_id) + .expect("Device's peer id should be in the list of device ids."); + let nccl_comm_id = get_nccl_comm_id(device_ids.clone()); + + // SAFETY: `comm` is a valid `MaybeUninit`. `nccl_comm_id` is a unique communicator ID + // shared across all participating ranks. `rank` is this device's position in the + // group. `comm_init_rank` initializes the communicator, making `assume_init` valid. + unsafe { + cudarc::nccl::result::comm_init_rank( + comm.as_mut_ptr(), + device_ids.len() as i32, + nccl_comm_id, + rank as i32, + ) + .map_err(|e| ServerError::Generic { + reason: format!("NCCL comm_init_rank failed: {e:?}"), + backtrace: BackTrace::capture(), + })?; + e.insert(comm.assume_init()); + } + + let mut initialized_comms = self.utilities.initialized_comms.write().unwrap(); + initialized_comms.insert(id); + } + + Ok(()) + } + + fn all_reduce( + &mut self, + src: Binding, + dst: Binding, + dtype: ElemType, + stream_id: StreamId, + op: ReduceOperation, + device_ids: Vec, + ) -> Result<(), ServerError> { + // We create a command on the server to retrieve the correct resource of the source and the destination + // from the memory pools. + if src.stream != dst.stream { + for stream in [src.stream, dst.stream].iter() { + let mut command = self.command_no_inputs( + *stream, + StreamErrorMode { + ignore: false, + flush: false, + }, + )?; + command.error(ServerError::Generic { + reason: "Source and destination should be on the same stream.".into(), + backtrace: BackTrace::capture(), + }); + } + } + + let mut command_src = self.command( + stream_id, + [&src, &dst].into_iter(), + StreamErrorMode { + ignore: false, + flush: false, + }, + )?; + let resource_src = command_src.resource(src)?; + let resource_dst = command_src.resource(dst)?; + + let stream = command_src.streams.current().sys; + + // We need to free the command before accessing communicators. + core::mem::drop(command_src); + + // Wait for data to be ready on compute stream. + Fence::new(stream).wait_async(self.comm_stream); + + // Get the communicator. + let comm = self + .communicators + .get(&CommunicationId::from(device_ids)) + .expect("Communicator for this ID should be initialized"); + + // Perform the `cudarc::nccl::result::all_reduce` operation. + let (nccl_dtype, count) = get_nccl_dtype_count(dtype, resource_src.size); + // SAFETY: `resource_src.ptr` and `resource_dst.ptr` are valid device pointers. + // `comm` is a valid NCCL communicator initialized via `comm_init_rank`. + // `self.comm_stream` is a valid CUDA stream dedicated to collective operations. + + unsafe { + cudarc::nccl::result::all_reduce( + resource_src.ptr as *const _, + resource_dst.ptr as *mut _, + count, + nccl_dtype, + to_nccl_op(op), + *comm, + self.comm_stream as _, + ) + .map_err(|e| ServerError::Generic { + reason: format!("NCCL all_reduce failed: {e:?}"), + backtrace: BackTrace::capture(), + })?; + } + + Ok(()) + } + + fn sync_collective(&mut self, stream_id: StreamId) -> Result<(), ServerError> { + let mut command = self.command_no_inputs( + stream_id, + StreamErrorMode { + ignore: true, + flush: false, + }, + )?; + let stream = command.streams.current().sys; + + drop(command); + + Fence::new(self.comm_stream).wait_async(stream); + + Ok(()) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(desc)))] + fn send( + &mut self, + desc: CopyDescriptor, + dtype: ElemType, + stream_id: StreamId, + device_id_dst: DeviceId, + ) -> Result<(), ServerError> { + let binding = desc.handle.clone(); + + // We create a command on the source server to retrieve the correct resource from the + // source memory pools. We also make sure the current stream is aligned with the stream of + // the binding, where the data was first allocated. + let mut command = self.command( + stream_id, + [&desc.handle].into_iter(), + StreamErrorMode { + ignore: true, + flush: false, + }, + )?; + let resource = command.resource(binding.clone())?; + let stream = command.streams.current().sys; + + // We need to free the command before creating another one. + core::mem::drop(command); + + // Wait for data to be ready on compute stream. + Fence::new(stream).wait_async(self.comm_stream); + + // Get the communicator. + let mut device_ids = vec![device_id_dst, self.device_id]; + device_ids.sort(); + let comm_id = CommunicationId::from(device_ids.clone()); + let comm = self + .communicators + .get(&comm_id) + .expect("Communicator for this ID should exist"); + + let rank_dst = device_ids + .iter() + .position(|id| id.index_id != self.device_id.index_id) + .unwrap() as i32; + + // Perform the `send` operation. + let (nccl_dtype, count) = get_nccl_dtype_count(dtype, resource.size); + // SAFETY: `resource.ptr` is a valid device pointer. + // `comm` is a valid NCCL communicator initialized via `comm_init_rank`. + // `self.comm_stream` is a valid CUDA stream dedicated to collective operations. + unsafe { + cudarc::nccl::result::send( + resource.ptr as *const _, + count, + nccl_dtype, + rank_dst, + *comm, + self.comm_stream as _, + ) + .map_err(|e| ServerError::Generic { + reason: format!("NCCL send failed: {e:?}"), + backtrace: BackTrace::capture(), + })?; + } + + Ok(()) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace"))] + fn recv( + &mut self, + handle: Handle, + dtype: ElemType, + stream_id: StreamId, + device_id_src: DeviceId, + ) -> Result<(), ServerError> { + // We create a new command on the destination server to reserve the necessary GPU memory. + let mut command_dst = self.command_no_inputs( + stream_id, + StreamErrorMode { + ignore: true, + flush: false, + }, + )?; + + let memory = command_dst.reserve(handle.size()).unwrap(); + command_dst.bind(memory, handle.memory.clone()); + + let resource_dst = command_dst.resource(handle.binding())?; + + core::mem::drop(command_dst); + + // Get the communicator. + let mut device_ids = vec![device_id_src, self.device_id]; + device_ids.sort(); + let comm_id = CommunicationId::from(device_ids.clone()); + let comm = self + .communicators + .get(&comm_id) + .expect("Communicator for this ID should exist"); + + let rank_src = device_ids + .iter() + .position(|id| id.index_id != self.device_id.index_id) + .unwrap() as i32; + + // Perform the `recv` operation. + let (nccl_dtype, count) = get_nccl_dtype_count(dtype, resource_dst.size); + // SAFETY: `resource.ptr` is a valid device pointer. + // `comm` is a valid NCCL communicator initialized via `comm_init_rank`. + // `self.comm_stream` is a valid CUDA stream dedicated to collective operations. + unsafe { + cudarc::nccl::result::recv( + resource_dst.ptr as *mut _, + count, + nccl_dtype, + rank_src, + *comm, + self.comm_stream as _, + ) + .map_err(|e| ServerError::Generic { + reason: format!("NCCL recv failed: {e:?}"), + backtrace: BackTrace::capture(), + })?; + } + + Ok(()) + } +} + +impl CudaServer { + /// Create a new cuda server. + pub(crate) fn new( + ctx: CudaContext, + mem_props: MemoryDeviceProperties, + mem_config: MemoryConfiguration, + mem_alignment: usize, + device_id: DeviceId, + utilities: ServerUtilities, + ) -> Self { + let config = CubeClRuntimeConfig::get(); + let max_streams = config.streaming.max_streams; + + ctx.unsafe_set_current().unwrap(); + + let comm_stream = cudarc::driver::result::stream::create( + cudarc::driver::result::stream::StreamKind::NonBlocking, + ) + .expect("Can create a new stream."); + + Self { + ctx, + device_id, + streams: MultiStream::new( + utilities.logger.clone(), + CudaStreamBackend::new( + mem_props, + mem_config, + mem_alignment, + utilities.logger.clone(), + ), + max_streams, + ), + utilities: Arc::new(utilities), + comm_stream, + communicators: HashMap::default(), + } + } + + fn command_no_inputs( + &mut self, + stream_id: StreamId, + mode: StreamErrorMode, + ) -> Result, ServerError> { + self.command(stream_id, [].into_iter(), mode) + } + + fn unsafe_set_current(&self) { + // TODO: Should check if on the same thread before calling it, since now we don't switch + // thread except for device memory transfer. + self.ctx.unsafe_set_current().unwrap(); + } + + fn command<'a>( + &mut self, + stream_id: StreamId, + handles: impl Iterator, + mode: StreamErrorMode, + ) -> Result, ServerError> { + self.unsafe_set_current(); + + if mode.flush { + let errors = self.flush_errors(stream_id); + + if !mode.ignore && !errors.is_empty() { + return Err(ServerError::ServerUnhealthy { + errors, + backtrace: BackTrace::capture(), + }); + } + } + + let streams = self.streams.resolve(stream_id, handles, !mode.ignore)?; + Ok(Command::new(&mut self.ctx, streams)) + } + + fn flush_errors(&mut self, stream_id: StreamId) -> Vec { + let mut stream = match self.streams.resolve(stream_id, [].into_iter(), false) { + Ok(stream) => stream, + Err(_) => return Vec::new(), + }; + let errors = core::mem::take(&mut stream.current().errors); + + // It is very important to tag current profiles as being wrong. + if !errors.is_empty() { + self.ctx.timestamps.error(ProfileError::Unknown { + reason: alloc::format!("{errors:?}"), + backtrace: BackTrace::capture(), + }); + stream.current().memory_management_gpu.cleanup(false); + } + + core::mem::drop(stream); + errors + } + + fn launch_checked( + &mut self, + kernel: Box>, + count: CubeCount, + bindings: KernelArguments, + mode: ExecutionMode, + stream_id: StreamId, + ) -> Result<(), ServerError> { + let mut kernel_id = kernel.id(); + let logger = self.streams.logger.clone(); + kernel_id.mode(mode); + let grid_constants = self + .ctx + .compilation_options + .supports_features + .grid_constants; + let mut command = self.command( + stream_id, + bindings.buffers.iter(), + StreamErrorMode { + ignore: true, + flush: false, + }, + )?; + + let count = match count { + CubeCount::Static(x, y, z) => (x, y, z), + // TODO: CUDA doesn't have an exact equivalent of dynamic dispatch. Instead, kernels are free to launch other kernels. + // One option is to create a dummy kernel with 1 thread that launches the real kernel with the dynamic dispatch settings. + // For now, just read the dispatch settings from the buffer. + CubeCount::Dynamic(binding) => { + let data = future::block_on(command.read_async(vec![CopyDescriptor::new( + binding, + [3].into(), + [1].into(), + 4, + )]))?; + let data = bytemuck::cast_slice(&data[0]); + assert!( + data.len() == 3, + "Dynamic cube count should contain 3 values" + ); + (data[0], data[1], data[2]) + } + }; + + let (info_const, info_binding) = if grid_constants { + let info = &bindings.info; + + let mut handle = Option::None; + if info.dynamic_metadata_offset < info.data.len() { + let dyn_meta = &bytemuck::cast_slice(&info.data[info.dynamic_metadata_offset..]); + handle = Some(command.create_with_data(dyn_meta)?); + } + + (Some(info.data.as_ptr() as *mut c_void), handle) + } else { + let mut handle = Option::None; + if !bindings.info.data.is_empty() { + handle = Some(command.create_with_data(bytemuck::cast_slice(&bindings.info.data))?); + } + (None, handle) + }; + + let mut resources = bindings + .tensor_maps + .iter() + .map(|it| it.binding.clone()) + .chain(bindings.buffers) + .map(|binding| command.resource(binding).expect("Resource to exist.")) + .collect::>(); + + let mut tensor_maps = Vec::with_capacity(bindings.tensor_maps.len()); + + for TensorMapBinding { map, binding } in bindings.tensor_maps.into_iter() { + let resource = command + .resource(binding) + .expect("Tensor map resource exists."); + let device_ptr = resource.ptr as *mut c_void; + + let mut map_ptr = MaybeUninit::zeroed(); + + let shape: Vec<_> = map + .metadata + .shape() + .iter() + .rev() + .map(|s| *s as u64) + .collect(); + let strides: Vec<_> = map + .metadata + .strides() + .iter() + .rev() + .skip(1) + .map(|s| *s as u64 * map.storage_ty.size() as u64) + .collect(); + let elem_stride: Vec<_> = map.elem_stride.iter().rev().map(|s| *s as u32).collect(); + + match &map.format { + // SAFETY: `map_ptr` is a zeroed `MaybeUninit`. `device_ptr` is a + // valid device pointer. Shape, strides, tile_size, and elem_stride vectors + // are constructed from validated metadata and outlive this call. + TensorMapFormat::Tiled(TiledArgs { tile_size }) => unsafe { + let tile_size: Vec<_> = + tile_size.iter().rev().copied().map(|s| s as u32).collect(); + + cuTensorMapEncodeTiled( + map_ptr.as_mut_ptr(), + elem_to_tensor_map_type(map.storage_ty), + map.metadata.rank() as u32, + device_ptr, + shape.as_ptr(), + strides.as_ptr(), + tile_size.as_ptr(), + elem_stride.as_ptr(), + interleave_to_cuda(map.interleave), + swizzle_to_cuda(map.swizzle), + prefetch_to_cuda(map.prefetch), + oob_to_cuda(map.oob_fill), + ) + .result() + .map_err(|err| { + let generic_err = + check_tma_generic(&map, device_ptr, &shape, &strides, &elem_stride) + .err(); + let tiled_err = check_tma_tiled(&map, &tile_size).err(); + generic_err + .or(tiled_err) + .unwrap_or_else(|| LaunchError::Unknown { + reason: format!("{err}"), + backtrace: BackTrace::capture(), + }) + })?; + }, + // SAFETY: Same invariants as `Tiled` above. Additionally, `lower_corner` and + // `upper_corner` are valid pixel box bounds derived from the tensor map args. + TensorMapFormat::Im2col(args) => unsafe { + let lower_corner: Vec<_> = + args.pixel_box_lower_corner.iter().rev().copied().collect(); + let upper_corner: Vec<_> = + args.pixel_box_upper_corner.iter().rev().copied().collect(); + + cuTensorMapEncodeIm2col( + map_ptr.as_mut_ptr(), + elem_to_tensor_map_type(map.storage_ty), + map.metadata.rank() as u32, + device_ptr, + shape.as_ptr(), + strides.as_ptr(), + lower_corner.as_ptr(), + upper_corner.as_ptr(), + args.channels_per_pixel, + args.pixels_per_column, + elem_stride.as_ptr(), + interleave_to_cuda(map.interleave), + swizzle_to_cuda(map.swizzle), + prefetch_to_cuda(map.prefetch), + oob_to_cuda(map.oob_fill), + ) + .result() + .map_err(|err| { + let generic_err = + check_tma_generic(&map, device_ptr, &shape, &strides, &elem_stride) + .err(); + let tiled_err = check_tma_im2col( + &map, + &lower_corner, + &upper_corner, + args.channels_per_pixel, + args.pixels_per_column, + ) + .err(); + generic_err + .or(tiled_err) + .unwrap_or_else(|| LaunchError::Unknown { + reason: format!("{err}"), + backtrace: BackTrace::capture(), + }) + })?; + }, + // SAFETY: Same invariants as `Im2col` above. Requires CUDA 12.8+. + #[cfg(cuda_12080)] + TensorMapFormat::Im2colWide(args) => unsafe { + use cudarc::driver::sys::{ + CUtensorMapIm2ColWideMode, cuTensorMapEncodeIm2colWide, + }; + cuTensorMapEncodeIm2colWide( + map_ptr.as_mut_ptr(), + elem_to_tensor_map_type(map.storage_ty), + map.metadata.rank() as u32, + device_ptr, + shape.as_ptr(), + strides.as_ptr(), + args.pixel_box_lower_corner_width, + args.pixel_box_upper_corner_width, + args.channels_per_pixel, + args.pixels_per_column, + elem_stride.as_ptr(), + interleave_to_cuda(map.interleave), + CUtensorMapIm2ColWideMode::CU_TENSOR_MAP_IM2COL_WIDE_MODE_W, + swizzle_to_cuda(map.swizzle), + prefetch_to_cuda(map.prefetch), + oob_to_cuda(map.oob_fill), + ) + .result() + .map_err(|err| { + let generic_err = + check_tma_generic(&map, device_ptr, &shape, &strides, &elem_stride) + .err(); + generic_err.unwrap_or_else(|| LaunchError::Unknown { + reason: format!("{err}"), + backtrace: BackTrace::capture(), + }) + })?; + }, + #[cfg(not(cuda_12080))] + TensorMapFormat::Im2colWide(_) => { + return Err(LaunchError::Unknown { + reason: "CUDA version 12.8 required for tensor map format Im2colWide" + .into(), + backtrace: BackTrace::capture(), + } + .into()); + } + }; + // SAFETY: `map_ptr` was fully initialized by one of the `cuTensorMapEncode*` + // calls above, which all succeeded (errors are propagated before reaching here). + let binding = unsafe { map_ptr.assume_init() }; + tensor_maps.push(binding); + } + + resources.extend( + info_binding + .into_iter() + .map(|s| command.resource(s.binding()).expect("Resource to exist")), + ); + + command.kernel( + kernel_id, + kernel, + mode, + count, + &tensor_maps, + &resources, + info_const, + logger, + )?; + + Ok(()) + } + + pub(crate) fn utilities(&self) -> Arc> { + self.utilities.clone() + } +} + +fn elem_to_tensor_map_type(ty: StorageType) -> CUtensorMapDataType { + use cudarc::driver::sys::CUtensorMapDataType::*; + match ty { + // packed fp4 should be treated as single 4-bit values to simplify indexing/shape handling + // So a tile of width 16 with fp4 elements is 8 x fp4x2 elements wide. + #[cfg(cuda_12080)] + StorageType::Packed(ty, 2) if ty.size_bits() == 4 => CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B, + StorageType::Scalar(ElemType::Float(kind)) => match kind { + // There's no special handling for FP8, so load as u8. `0u8 == 0.0` when reinterpreting. + FloatKind::E2M1 // single fp4s are padded to a full byte + | FloatKind::E4M3 + | FloatKind::E5M2 + | FloatKind::UE8M0 + | FloatKind::E2M3 + | FloatKind::E3M2 => CU_TENSOR_MAP_DATA_TYPE_UINT8, + FloatKind::F16 => CU_TENSOR_MAP_DATA_TYPE_FLOAT16, + FloatKind::BF16 => CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, + FloatKind::Flex32 | FloatKind::F32 => CU_TENSOR_MAP_DATA_TYPE_FLOAT32, + FloatKind::TF32 => CU_TENSOR_MAP_DATA_TYPE_TFLOAT32, + FloatKind::F64 => CU_TENSOR_MAP_DATA_TYPE_FLOAT64, + }, + StorageType::Scalar(ElemType::Int(kind)) => match kind { + // UInt is fine because zero bits and size is the same between both + IntKind::I8 => CU_TENSOR_MAP_DATA_TYPE_UINT8, + IntKind::I16 => CU_TENSOR_MAP_DATA_TYPE_UINT16, + IntKind::I32 => CU_TENSOR_MAP_DATA_TYPE_INT32, + IntKind::I64 => CU_TENSOR_MAP_DATA_TYPE_INT64, + }, + StorageType::Scalar(ElemType::UInt(kind)) => match kind { + UIntKind::U8 => CU_TENSOR_MAP_DATA_TYPE_UINT8, + UIntKind::U16 => CU_TENSOR_MAP_DATA_TYPE_UINT16, + UIntKind::U32 => CU_TENSOR_MAP_DATA_TYPE_UINT32, + UIntKind::U64 => CU_TENSOR_MAP_DATA_TYPE_UINT64, + }, + _ => unimplemented!("Not supported for tensor map type"), + } +} + +fn interleave_to_cuda(interleave: TensorMapInterleave) -> CUtensorMapInterleave { + use cudarc::driver::sys::CUtensorMapInterleave::*; + match interleave { + TensorMapInterleave::None => CU_TENSOR_MAP_INTERLEAVE_NONE, + TensorMapInterleave::B16 => CU_TENSOR_MAP_INTERLEAVE_16B, + TensorMapInterleave::B32 => CU_TENSOR_MAP_INTERLEAVE_32B, + } +} + +fn swizzle_to_cuda(swizzle: TensorMapSwizzle) -> CUtensorMapSwizzle { + use cudarc::driver::sys::CUtensorMapSwizzle::*; + match swizzle { + TensorMapSwizzle::None => CU_TENSOR_MAP_SWIZZLE_NONE, + TensorMapSwizzle::B32 => CU_TENSOR_MAP_SWIZZLE_32B, + TensorMapSwizzle::B64 => CU_TENSOR_MAP_SWIZZLE_64B, + TensorMapSwizzle::B128 => CU_TENSOR_MAP_SWIZZLE_128B, + #[cfg(cuda_12080)] + TensorMapSwizzle::B128Atom32B => CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B, + #[cfg(cuda_12080)] + TensorMapSwizzle::B128Atom32BFlip8B => CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B_FLIP_8B, + #[cfg(cuda_12080)] + TensorMapSwizzle::B128Atom64B => CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B, + #[cfg(not(cuda_12080))] + _ => unimplemented!("Swizzle atomicity requires CUDA 12.8 or higher"), + } +} + +fn prefetch_to_cuda(prefetch: TensorMapPrefetch) -> CUtensorMapL2promotion { + use cudarc::driver::sys::CUtensorMapL2promotion::*; + match prefetch { + TensorMapPrefetch::None => CU_TENSOR_MAP_L2_PROMOTION_NONE, + TensorMapPrefetch::B64 => CU_TENSOR_MAP_L2_PROMOTION_L2_64B, + TensorMapPrefetch::B128 => CU_TENSOR_MAP_L2_PROMOTION_L2_128B, + TensorMapPrefetch::B256 => CU_TENSOR_MAP_L2_PROMOTION_L2_256B, + } +} + +fn oob_to_cuda(fill: OobFill) -> CUtensorMapFloatOOBfill { + use cudarc::driver::sys::CUtensorMapFloatOOBfill::*; + match fill { + OobFill::Zero => CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE, + OobFill::NaN => CU_TENSOR_MAP_FLOAT_OOB_FILL_NAN_REQUEST_ZERO_FMA, + } +} + +macro_rules! launch_check { + ($assertion: expr, $($arg:tt)+) => { + if $assertion { + Ok(()) + } else { + Err(LaunchError::Unknown { + reason: format!($($arg)*), + backtrace: BackTrace::capture(), + }) + } + }; +} + +fn check_tma_generic( + map: &TensorMapMeta, + device_ptr: *mut c_void, + shape: &[u64], + strides: &[u64], + elem_strides: &[u32], +) -> Result<(), LaunchError> { + // globalAddress invariants + launch_check!( + (device_ptr as usize).is_multiple_of(16), + "Tensor pointer must be 16 byte aligned" + )?; + if !matches!(map.interleave, TensorMapInterleave::None) { + launch_check!( + (device_ptr as usize).is_multiple_of(32), + "Tensor pointer must be 32 byte aligned" + )?; + } + + // tensorRank invariants + launch_check!( + (1..=5).contains(&map.metadata.rank()), + "Rank must be between 1 and 5" + )?; + launch_check!( + matches!(map.interleave, TensorMapInterleave::None) || map.metadata.rank() >= 3, + "When interleave is enabled, rank must be >= 3" + )?; + + // globalDim invariants + launch_check!( + shape.iter().all(|it| *it <= u32::MAX as u64), + "Shape must be <= u32::MAX" + )?; + #[cfg(cuda_12080)] + if matches!(map.storage_ty, StorageType::Packed(ty, 2) if ty.size_bits() == 4) { + launch_check!( + shape[0].is_multiple_of(2), + "Packed tensor map must have multiple of 2 for the innermost dimension" + )?; + } + + // globalStrides invariants + launch_check!( + strides.iter().all(|it| it.is_multiple_of(16)), + "Strides must be 16 byte aligned" + )?; + if matches!(map.interleave, TensorMapInterleave::B32) { + launch_check!( + strides.iter().all(|it| it.is_multiple_of(32)), + "Strides must be 32 byte aligned when interleave is B32" + )?; + } + + // elementStrides invariants + launch_check!( + elem_strides.iter().all(|it| *it > 0 && *it <= 8), + "Element strides must be non-zero and <= 8" + )?; + if matches!(map.interleave, TensorMapInterleave::None) { + launch_check!( + elem_strides[0] == 1, + "Innermost element stride is ignored without interleaving" + )?; + } + + // oobFill invariants + if matches!(map.oob_fill, OobFill::NaN) { + launch_check!( + map.storage_ty.is_float(), + "NaN fill is only supported for float types" + )?; + } + + Ok(()) +} + +fn check_tma_tiled(map: &TensorMapMeta, tile_size: &[u32]) -> Result<(), LaunchError> { + launch_check!( + tile_size.len() == map.metadata.rank(), + "Tile shape should match rank" + )?; + launch_check!( + tile_size.iter().all(|it| *it > 0 && *it <= 256), + "Tile shape must be non-zero and <= 256" + )?; + let tile_size_0_bytes = tile_size[0] as usize * map.storage_ty.size(); + if matches!(map.interleave, TensorMapInterleave::None) { + let max_tile_bytes = match map.swizzle { + TensorMapSwizzle::None => usize::MAX, + TensorMapSwizzle::B32 => 32, + TensorMapSwizzle::B64 => 64, + TensorMapSwizzle::B128 + | TensorMapSwizzle::B128Atom32B + | TensorMapSwizzle::B128Atom32BFlip8B + | TensorMapSwizzle::B128Atom64B => 128, + }; + launch_check!( + tile_size_0_bytes <= max_tile_bytes, + "Innermost tile dim must be <= swizzle size" + )?; + } + if matches!(map.interleave, TensorMapInterleave::B32) { + launch_check!( + map.swizzle == TensorMapSwizzle::B32, + "If interleave is B32, swizzle must be B32" + )?; + } + + Ok(()) +} + +fn check_tma_im2col( + map: &TensorMapMeta, + lower_corner: &[i32], + upper_corner: &[i32], + channels_per_pixel: u32, + pixels_per_column: u32, +) -> Result<(), LaunchError> { + launch_check!( + lower_corner.len() == map.metadata.rank() - 2, + "Lower corner must be rank - 2 elements" + )?; + launch_check!( + upper_corner.len() == map.metadata.rank() - 2, + "Upper corner must be rank - 2 elements" + )?; + + launch_check!( + map.metadata.rank() >= 3 && map.metadata.rank() <= 5, + "im2col requires rank to be between 3 and 5" + )?; + + let (range_lower, range_upper) = match map.metadata.rank() { + 3 => (-32768, 32767), + 4 => (-128, 127), + 5 => (-16, 15), + _ => unreachable!(), + }; + launch_check!( + lower_corner + .iter() + .all(|it| *it >= range_lower && *it <= range_upper), + "Lower corner must be in range [{range_lower}, {range_upper}] for {}D im2col", + map.metadata.rank() + )?; + launch_check!( + upper_corner + .iter() + .all(|it| *it >= range_lower && *it <= range_upper), + "Upper corner must be in range [{range_lower}, {range_upper}] for {}D im2col", + map.metadata.rank() + )?; + + launch_check!( + channels_per_pixel <= 256, + "Channels per pixel must be <= 256" + )?; + launch_check!( + pixels_per_column <= 1024, + "Pixels per column must be <= 1024" + )?; + + Ok(()) +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/cpu.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/cpu.rs new file mode 100644 index 00000000..81bdded1 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/cpu.rs @@ -0,0 +1,132 @@ +use cubecl_common::backtrace::BackTrace; +use cubecl_core::server::IoError; +use cubecl_runtime::storage::{ComputeStorage, StorageHandle, StorageId, StorageUtilization}; +use std::{collections::HashMap, ffi::c_void}; + +/// Memory alignment for pinned host memory, set to the size of `u128` for optimal performance. +pub const PINNED_MEMORY_ALIGNMENT: usize = core::mem::size_of::(); + +/// Manages pinned host memory for CUDA operations. +/// +/// This storage handles allocation and deallocation of pinned (page-locked) host memory, +/// which is optimized for fast data transfers between host and GPU in CUDA applications. +pub struct PinnedMemoryStorage { + memory: HashMap, + mem_alignment: usize, +} + +/// A pinned memory resource allocated on the host. +#[derive(Debug)] +pub struct PinnedMemoryResource { + /// Pointer to the pinned memory buffer. + pub ptr: *mut u8, + /// Size of the memory resource in bytes. + pub size: usize, +} + +/// Internal representation of pinned memory with associated pointers. +#[derive(Debug)] +struct PinnedMemory { + /// Pointer to the pinned memory buffer. + ptr: *mut c_void, + /// Pointer-to-pointer for CUDA allocation, kept alive for async operations. + #[allow(unused)] + ptr2ptr: *mut *mut c_void, +} + +impl PinnedMemoryStorage { + /// Creates a new [`PinnedMemoryStorage`] instance. + /// + /// Initializes the storage with the default pinned memory alignment + /// defined by [`PINNED_MEMORY_ALIGNMENT`]. + pub fn new() -> Self { + Self { + memory: HashMap::new(), + mem_alignment: PINNED_MEMORY_ALIGNMENT, + } + } +} + +// SAFETY: `PinnedMemoryResource` contains a raw pointer to page-locked host memory. +// It is safe to send between threads because the memory remains valid and pinned +// regardless of which thread accesses it, and access is serialized by the `DeviceHandle`. +unsafe impl Send for PinnedMemoryResource {} +// SAFETY: `PinnedMemoryStorage` is only accessed from one thread at a time via the +// `DeviceHandle`, which serializes all server access. The pinned memory it manages +// is never shared across threads without synchronization. +unsafe impl Send for PinnedMemoryStorage {} + +impl ComputeStorage for PinnedMemoryStorage { + type Resource = PinnedMemoryResource; + + fn alignment(&self) -> usize { + self.mem_alignment + } + + fn get(&mut self, handle: &StorageHandle) -> Self::Resource { + let memory = self + .memory + .get(&handle.id) + .expect("Storage handle not found"); + + let offset = handle.offset() as usize; + let size = handle.size() as usize; + + // SAFETY: `memory.ptr` was allocated by `cuMemAllocHost_v2` with at least + // `offset + size` bytes. The `add(offset)` produces a pointer within the allocation + // bounds as guaranteed by the storage handle's offset/size validation. + unsafe { + PinnedMemoryResource { + ptr: memory.ptr.cast::().add(offset), + size, + } + } + } + + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, size)) + )] + fn alloc(&mut self, size: u64) -> Result { + // SAFETY: Calling CUDA driver FFI to allocate page-locked (pinned) host memory. + // The returned pointer is stored and freed via `cuMemFreeHost` on deallocation. + let resource = unsafe { + let mut ptr: *mut c_void = std::ptr::null_mut(); + let ptr2ptr: *mut *mut c_void = &mut ptr; + + // Allocate pinned host memory using cuMemAllocHost_v2 + let result = cudarc::driver::sys::cuMemAllocHost_v2(ptr2ptr, size as usize); + + if result != cudarc::driver::sys::CUresult::CUDA_SUCCESS { + return Err(IoError::Unknown { + description: format!("cuMemAllocHost_v2 failed with error code: {result:?}"), + backtrace: BackTrace::capture(), + }); + } + + PinnedMemory { ptr, ptr2ptr } + }; + + let id = StorageId::new(); + self.memory.insert(id, resource); + Ok(StorageHandle::new( + id, + StorageUtilization { offset: 0, size }, + )) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + fn dealloc(&mut self, id: StorageId) { + if let Some(resource) = self.memory.remove(&id) { + // SAFETY: `resource.ptr` was allocated by `cuMemAllocHost_v2` and has not been + // freed yet. After this call, the pointer is invalid and removed from `self.memory`. + unsafe { + cudarc::driver::sys::cuMemFreeHost(resource.ptr); + } + } + } + + fn flush(&mut self) { + // We don't wait for dealloc. + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/gpu.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/gpu.rs new file mode 100644 index 00000000..afbc6004 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/gpu.rs @@ -0,0 +1,212 @@ +use crate::compute::uninit_vec; +use cubecl_common::backtrace::BackTrace; +use cubecl_core::server::IoError; +use cubecl_runtime::storage::{ComputeStorage, StorageHandle, StorageId, StorageUtilization}; +use cudarc::driver::DriverError; +use std::collections::HashMap; + +#[derive(Debug, Clone, Copy)] +enum AllocationKind { + Async, + Sync, +} + +/// Buffer storage for NVIDIA GPUs. +/// +/// This struct manages memory resources for CUDA kernels, allowing them to be used as bindings +/// for launching kernels. +pub struct GpuStorage { + memory: HashMap, + deallocations: Vec, + ptr_bindings: PtrBindings, + stream: cudarc::driver::sys::CUstream, + mem_alignment: usize, +} + +/// A GPU memory resource allocated for CUDA using [`GpuStorage`]. +#[derive(Debug)] +pub struct GpuResource { + /// The GPU memory pointer. + pub ptr: u64, + /// The CUDA binding pointer. + pub binding: *mut std::ffi::c_void, + /// The size of the resource. + pub size: u64, +} + +impl GpuResource { + /// Creates a new [`GpuResource`]. + pub fn new(ptr: u64, binding: *mut std::ffi::c_void, size: u64) -> Self { + Self { ptr, binding, size } + } +} + +impl GpuStorage { + /// Creates a new [`GpuStorage`] instance for the specified CUDA stream. + /// + /// # Arguments + /// + /// * `mem_alignment` - The memory alignment requirement in bytes. + pub fn new(mem_alignment: usize, stream: cudarc::driver::sys::CUstream) -> Self { + Self { + memory: HashMap::new(), + deallocations: Vec::new(), + ptr_bindings: PtrBindings::new(), + stream, + mem_alignment, + } + } + + /// Deallocates buffers marked for deallocation. + /// + /// This method processes all pending deallocations by freeing the associated GPU memory. + fn perform_deallocations(&mut self) { + self.deallocations + .drain(..) + .filter_map(|id| self.memory.remove(&id)) + // SAFETY: Each `ptr` was obtained from a prior `malloc_async` or `malloc_sync` + // call and has not been freed yet. The deallocation method matches the allocation kind. + .for_each(|(ptr, kind)| unsafe { + match kind { + AllocationKind::Async => { + let _ = cudarc::driver::result::free_async(ptr, self.stream); + } + AllocationKind::Sync => { + if let Err(e) = cudarc::driver::result::free_sync(ptr) { + eprintln!("CUDA free error: {}", e); + } + } + } + }); + } +} + +// SAFETY: `GpuResource` contains CUDA device pointers that are safe to send between +// threads as long as proper stream synchronization is maintained by the caller. +unsafe impl Send for GpuResource {} +// SAFETY: `GpuStorage` is only accessed from one thread at a time via the `DeviceHandle`, +// which serializes all server access. The raw CUDA pointers it contains are never shared +// across threads without synchronization. +unsafe impl Send for GpuStorage {} + +impl core::fmt::Debug for GpuStorage { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("GpuStorage").finish() + } +} + +/// Manages active CUDA buffer bindings in a ring buffer. +/// +/// This ensures that pointers remain valid during kernel execution, preventing use-after-free errors. +struct PtrBindings { + slots: Vec, + cursor: usize, +} + +impl PtrBindings { + /// Creates a new [`PtrBindings`] instance with a fixed-size ring buffer. + fn new() -> Self { + Self { + // SAFETY: `CUdeviceptr` is a `u64`, valid for any bit pattern. All slots are + // written via `register()` before being read, so uninitialized values are never observed. + slots: unsafe { uninit_vec(crate::device::CUDA_MAX_BINDINGS as usize) }, + cursor: 0, + } + } + + /// Registers a new pointer in the ring buffer. + /// + /// # Arguments + /// + /// * `ptr` - The CUDA device pointer to register. + /// + /// # Returns + /// + /// A reference to the registered pointer. + fn register(&mut self, ptr: u64) -> &u64 { + self.slots[self.cursor] = ptr; + let ptr_ref = self.slots.get(self.cursor).unwrap(); + + self.cursor += 1; + + // Reset the cursor when the ring buffer is full. + if self.cursor >= self.slots.len() { + self.cursor = 0; + } + + ptr_ref + } +} + +impl ComputeStorage for GpuStorage { + type Resource = GpuResource; + + fn alignment(&self) -> usize { + self.mem_alignment + } + + fn get(&mut self, handle: &StorageHandle) -> Self::Resource { + let (ptr, _) = self + .memory + .get(&handle.id) + .expect("Storage handle not found"); + + let offset = handle.offset(); + let size = handle.size(); + let ptr = self.ptr_bindings.register(ptr + offset); + + GpuResource::new( + *ptr, + ptr as *const cudarc::driver::sys::CUdeviceptr as *mut std::ffi::c_void, + size, + ) + } + + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, size)) + )] + fn alloc(&mut self, size: u64) -> Result { + let id = StorageId::new(); + // SAFETY: Calling CUDA driver FFI to allocate device memory. First tries async + // allocation on the stream; falls back to synchronous allocation if that fails. + // The returned pointer is stored in `self.memory` and freed on deallocation. + let ptr = unsafe { cudarc::driver::result::malloc_async(self.stream, size as usize) }; + let (ptr, kind) = match ptr { + Ok(ptr) => (ptr, AllocationKind::Async), + Err(_) => unsafe { + match cudarc::driver::result::malloc_sync(size as usize) { + Ok(ptr) => (ptr, AllocationKind::Sync), + Err(DriverError(cudarc::driver::sys::CUresult::CUDA_ERROR_OUT_OF_MEMORY)) => { + return Err(IoError::BufferTooBig { + size, + backtrace: BackTrace::capture(), + }); + } + Err(other) => { + return Err(IoError::Unknown { + description: format!("CUDA allocation error: {other}"), + backtrace: BackTrace::capture(), + }); + } + } + }, + }; + + self.memory.insert(id, (ptr, kind)); + Ok(StorageHandle::new( + id, + StorageUtilization { offset: 0, size }, + )) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + fn dealloc(&mut self, id: StorageId) { + self.deallocations.push(id); + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + fn flush(&mut self) { + self.perform_deallocations(); + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/mod.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/mod.rs new file mode 100644 index 00000000..7dce56f6 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/storage/mod.rs @@ -0,0 +1,2 @@ +pub mod cpu; +pub mod gpu; diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/stream.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/stream.rs new file mode 100644 index 00000000..976ff3a9 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/stream.rs @@ -0,0 +1,107 @@ +use crate::compute::{ + storage::{ + cpu::{PINNED_MEMORY_ALIGNMENT, PinnedMemoryStorage}, + gpu::GpuStorage, + }, + sync::Fence, +}; +use cubecl_core::{ + MemoryConfiguration, + ir::MemoryDeviceProperties, + server::{Binding, ServerError}, +}; +use cubecl_runtime::{ + logging::ServerLogger, + memory_management::{ + MemoryAllocationMode, MemoryManagement, MemoryManagementOptions, drop_queue, + }, + stream::EventStreamBackend, +}; +use std::sync::Arc; + +#[derive(Debug)] +pub struct Stream { + pub sys: cudarc::driver::sys::CUstream, + pub memory_management_gpu: MemoryManagement, + pub memory_management_cpu: MemoryManagement, + pub errors: Vec, + pub drop_queue: drop_queue::PendingDropQueue, +} + +impl drop_queue::Fence for Fence { + fn sync(self) { + let _ = self.wait_sync().ok(); + } +} + +#[derive(new, Debug)] +pub struct CudaStreamBackend { + mem_props: MemoryDeviceProperties, + mem_config: MemoryConfiguration, + mem_alignment: usize, + logger: Arc, +} + +impl EventStreamBackend for CudaStreamBackend { + type Stream = Stream; + type Event = Fence; + + fn create_stream(&self) -> Self::Stream { + let stream = cudarc::driver::result::stream::create( + cudarc::driver::result::stream::StreamKind::NonBlocking, + ) + .expect("Can create a new stream."); + + let storage = GpuStorage::new(self.mem_alignment, stream); + let memory_management_gpu = MemoryManagement::from_configuration( + storage, + &self.mem_props, + self.mem_config.clone(), + self.logger.clone(), + MemoryManagementOptions::new("Main GPU Memory"), + ); + // We use the same page size and memory pools configuration for CPU pinned memory, since we + // expect the CPU to have at least the same amount of RAM as GPU memory. + let memory_management_cpu = MemoryManagement::from_configuration( + PinnedMemoryStorage::new(), + &MemoryDeviceProperties { + max_page_size: self.mem_props.max_page_size, + alignment: PINNED_MEMORY_ALIGNMENT as u64, + }, + self.mem_config.clone(), + self.logger.clone(), + MemoryManagementOptions::new("Pinned CPU Memory").mode(MemoryAllocationMode::Auto), + ); + + Stream { + sys: stream, + memory_management_gpu, + memory_management_cpu, + errors: Vec::new(), + drop_queue: Default::default(), + } + } + + fn flush(stream: &mut Self::Stream) -> Self::Event { + Fence::new(stream.sys) + } + + fn wait_event(stream: &mut Self::Stream, event: Self::Event) { + event.wait_async(stream.sys); + } + + fn wait_event_sync(event: Self::Event) -> Result<(), ServerError> { + event.wait_sync() + } + + fn handle_cursor(stream: &Self::Stream, binding: &Binding) -> u64 { + stream + .memory_management_gpu + .get_cursor(binding.memory.clone()) + .unwrap() + } + + fn is_healthy(stream: &Self::Stream) -> bool { + stream.errors.is_empty() + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/sync/fence.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/sync/fence.rs new file mode 100644 index 00000000..68313d35 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/sync/fence.rs @@ -0,0 +1,87 @@ +use cubecl_common::backtrace::BackTrace; +use cubecl_core::server::ServerError; +use cudarc::driver::sys::{CUevent_flags, CUevent_st, CUevent_wait_flags, CUstream_st}; + +/// A fence is simply an [event](CUevent_st) created on a [stream](CUevent_st) that you can wait +/// until completion. +/// +/// This is useful for doing synchronization outside of the compute server, which is normally +/// locked by a mutex or a channel. This allows the server to continue accepting other tasks. +#[derive(Debug)] +pub struct Fence { + event: *mut CUevent_st, +} + +// If we don't close the stream or destroy the event, it is safe. +// +// # Safety +// +// Since streams are never closed and we destroy the event after waiting, which consumes the +// [Fence], it is safe. +unsafe impl Send for Fence {} + +#[allow(unused)] +impl Fence { + /// Create a new [Fence] on the given stream. + /// + /// # Notes + /// + /// The [stream](CUevent_st) must be initialized. + pub fn new(stream: *mut CUstream_st) -> Self { + // SAFETY: `stream` must be a valid, initialized CUDA stream (enforced by the doc + // contract). The event is created and immediately recorded on the stream. + unsafe { + let event = + cudarc::driver::result::event::create(CUevent_flags::CU_EVENT_DEFAULT).unwrap(); + cudarc::driver::result::event::record(event, stream).unwrap(); + + Self { event } + } + } + + /// Wait for the [Fence] to be reached, ensuring that all previous tasks enqueued to the + /// [stream](CUstream_st) are completed. + pub fn wait_sync(self) -> Result<(), ServerError> { + // SAFETY: `self.event` is a valid event created in `Fence::new`. We synchronize + // (block) until the event completes, then destroy it. `self` is consumed so the + // event cannot be double-freed. + unsafe { + cudarc::driver::result::event::synchronize(self.event).map_err(|err| { + ServerError::Generic { + reason: format!("{err:?}"), + backtrace: BackTrace::capture(), + } + })?; + cudarc::driver::result::event::destroy(self.event).map_err(|err| { + ServerError::Generic { + reason: format!("{err:?}"), + backtrace: BackTrace::capture(), + } + })?; + } + + Ok(()) + } + + /// Wait for the [Fence] to be reached, ensuring that all previous tasks enqueued to the + /// [stream](CUstream_st) are completed on the [original stream](CUstream_st) before new tasks + /// are registered on the [provided stream](CUstream_st). + /// + /// # Notes + /// + /// The [stream](CUevent_st) must be initialized. + pub fn wait_async(self, stream: *mut CUstream_st) { + // SAFETY: `self.event` is a valid event created in `Fence::new`. `stream` must be + // a valid CUDA stream. The event is destroyed after the wait, and `self` is consumed + // so the event cannot be used again. + unsafe { + cudarc::driver::result::stream::wait_event( + stream, + self.event, + CUevent_wait_flags::CU_EVENT_WAIT_DEFAULT, + ) + .unwrap(); + cudarc::driver::result::event::destroy(self.event).unwrap(); + } + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/compute/sync/mod.rs b/third_party/cubecl-cuda-0.10.0-patched/src/compute/sync/mod.rs new file mode 100644 index 00000000..a3be1adb --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/compute/sync/mod.rs @@ -0,0 +1,3 @@ +mod fence; + +pub use fence::*; diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/device.rs b/third_party/cubecl-cuda-0.10.0-patched/src/device.rs new file mode 100644 index 00000000..0d03e76f --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/device.rs @@ -0,0 +1,33 @@ +use cubecl_common::device::{Device, DeviceId}; + +// It is not clear if CUDA has a limit on the number of bindings it can hold at +// any given time, but it's highly unlikely that it's more than this. We can +// also assume that we'll never have more than this many bindings in flight, +// so it's 'safe' to store only this many bindings. +pub const CUDA_MAX_BINDINGS: u32 = 1024; + +#[derive(new, Clone, PartialEq, Eq, Default, Hash)] +pub struct CudaDevice { + pub index: usize, +} + +impl core::fmt::Debug for CudaDevice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Cuda({})", self.index) + } +} + +impl Device for CudaDevice { + fn from_id(device_id: DeviceId) -> Self { + Self { + index: device_id.index_id as usize, + } + } + + fn to_id(&self) -> DeviceId { + DeviceId { + type_id: 0, + index_id: self.index as u16, + } + } +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/lib.rs b/third_party/cubecl-cuda-0.10.0-patched/src/lib.rs new file mode 100644 index 00000000..409db3af --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/lib.rs @@ -0,0 +1,80 @@ +#[macro_use] +extern crate derive_new; +extern crate alloc; + +mod compute; +mod device; +mod runtime; + +pub use device::*; +pub use runtime::*; + +#[cfg(feature = "ptx-wmma")] +pub(crate) type WmmaCompiler = cubecl_cpp::cuda::mma::PtxWmmaCompiler; + +#[cfg(not(feature = "ptx-wmma"))] +pub(crate) type WmmaCompiler = cubecl_cpp::cuda::mma::CudaWmmaCompiler; + +pub mod install { + use std::path::PathBuf; + + pub fn include_path() -> PathBuf { + let mut path = cuda_path().expect(" + CUDA installation not found. + Please ensure that CUDA is installed and the CUDA_PATH environment variable is set correctly. + Note: Default paths are used for Linux (/usr/local/cuda) and Windows (C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/), which may not be correct. + "); + path.push("include"); + path + } + + pub fn cccl_include_path() -> PathBuf { + let mut path = include_path(); + path.push("cccl"); + path + } + + pub fn cuda_path() -> Option { + if let Ok(path) = std::env::var("CUDA_PATH") { + return Some(PathBuf::from(path)); + } + + #[cfg(target_os = "linux")] + { + // If it is installed as part of the distribution + return if std::fs::exists("/usr/local/cuda").is_ok_and(|exists| exists) { + Some(PathBuf::from("/usr/local/cuda")) + } else if std::fs::exists("/opt/cuda").is_ok_and(|exists| exists) { + Some(PathBuf::from("/opt/cuda")) + } else if std::fs::exists("/usr/bin/nvcc").is_ok_and(|exists| exists) { + // Maybe the compiler was installed within the user path. + Some(PathBuf::from("/usr")) + } else { + None + }; + } + + #[cfg(target_os = "windows")] + { + return Some(PathBuf::from( + "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/", + )); + } + + #[allow(unreachable_code)] + None + } +} + +#[cfg(test)] +#[allow(unexpected_cfgs)] +mod tests { + pub type TestRuntime = crate::CudaRuntime; + + pub use half::{bf16, f16}; + + cubecl_core::testgen_all!(f32: [f16, bf16, f32, f64], i32: [i8, i16, i32, i64], u32: [u8, u16, u32, u64]); + cubecl_std::testgen!(); + cubecl_std::testgen_tensor_identity!([f16, bf16, f32, u32]); + cubecl_std::testgen_quantized_view!(f16); +} diff --git a/third_party/cubecl-cuda-0.10.0-patched/src/runtime.rs b/third_party/cubecl-cuda-0.10.0-patched/src/runtime.rs new file mode 100644 index 00000000..be53d933 --- /dev/null +++ b/third_party/cubecl-cuda-0.10.0-patched/src/runtime.rs @@ -0,0 +1,392 @@ +use crate::{ + WmmaCompiler, + compute::{CudaServer, context::CudaContext}, + device::CudaDevice, +}; +use cubecl_common::{ + device::{Device, DeviceService}, + profile::TimingMethod, +}; +use cubecl_core::{ + MemoryConfiguration, Runtime, + device::{DeviceId, ServerUtilitiesHandle}, + ir::{ + BarrierLevel, ContiguousElements, DeviceProperties, ElemType, FloatKind, + HardwareProperties, MatrixLayout, MemoryDeviceProperties, MmaProperties, OpaqueType, + SemanticType, StorageType, TargetProperties, Type, VectorSize, + features::{AtomicUsage, Plane, Tma, TypeUsage}, + }, + server::ServerUtilities, + zspace::{Shape, Strides, striding::has_pitched_row_major_strides}, +}; +use cubecl_cpp::{ + ComputeKernel, DialectWmmaCompiler, + cuda::{CudaDialect, arch::CudaArchitecture, mma::contiguous_elements_cuda}, + register_supported_types, + shared::{ + CompilationOptions, CppCompiler, CppSupportedFeatures, register_mma_features, + register_scaled_mma_features, register_wmma_features, + }, +}; +use cubecl_runtime::{ + allocator::PitchedMemoryLayoutPolicy, client::ComputeClient, logging::ServerLogger, +}; +use cudarc::driver::sys::{CUDA_VERSION, cuDeviceTotalMem_v2}; +use std::{mem::MaybeUninit, sync::Arc}; + +/// Options configuring the CUDA runtime. +#[derive(Default)] +pub struct RuntimeOptions { + /// Configures the memory management. + pub memory_config: MemoryConfiguration, +} + +#[derive(Debug, Clone)] +pub struct CudaRuntime; + +impl DeviceService for CudaServer { + fn init(device_id: cubecl_common::device::DeviceId) -> Self { + let options = RuntimeOptions::default(); + let device = CudaDevice::from_id(device_id); + + // To get the supported WMMA features, and memory properties, we have to initialize the server immediately. + cudarc::driver::result::init().unwrap(); + let device_index = device.index as i32; + let device_ptr = cudarc::driver::result::device::get(device_index).unwrap(); + let arch_major; + // SAFETY: Calling CUDA driver FFI to query compute capability attributes. + // `device_ptr` is a valid device handle obtained from `cudarc::driver::result::device::get`. + let arch_version = unsafe { + arch_major = cudarc::driver::result::device::get_attribute( + device_ptr, + cudarc::driver::sys::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, + ) + .unwrap(); + let minor = cudarc::driver::result::device::get_attribute( + device_ptr, + cudarc::driver::sys::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, + ) + .unwrap(); + arch_major * 10 + minor + } as u32; + + // This is the alignment returned by `cuMallocPitched`, so it's the one considered optimal + // for row alignment by CUDA. This hasn't changed since at least the GTX 700 series. + // Querying texture row align is a heuristic, but also not guaranteed to be the same. + let mem_alignment = 512; + + // Ask the wmma compiler for its supported combinations + let arch = CudaArchitecture { + version: arch_version, + }; + let supported_wmma_combinations = WmmaCompiler::supported_wmma_combinations(&arch); + let supported_mma_combinations = WmmaCompiler::supported_mma_combinations(&arch); + let supported_scaled_mma_combinations = + WmmaCompiler::supported_scaled_mma_combinations(&arch); + + // SAFETY: `device_ptr` is a valid CUDA device. `primary_ctx::retain` returns the + // primary context which is then set as current for the calling thread. + let ctx = unsafe { + let ctx = cudarc::driver::result::primary_ctx::retain(device_ptr).unwrap(); + cudarc::driver::result::ctx::set_current(ctx).unwrap(); + ctx + }; + + // SAFETY: `device_ptr` is valid. `cuDeviceTotalMem_v2` writes the total device memory + // into the `MaybeUninit`, making `assume_init()` valid on success. + let max_memory = unsafe { + let mut bytes = MaybeUninit::uninit(); + cuDeviceTotalMem_v2(bytes.as_mut_ptr(), device_ptr); + bytes.assume_init() as u64 + }; + let mem_properties = MemoryDeviceProperties { + max_page_size: max_memory / 4, + alignment: mem_alignment as u64, + }; + + let mut comp_opts = CompilationOptions { + supports_features: CppSupportedFeatures { + fast_math: true, + ..Default::default() + }, + ..Default::default() + }; + + // SAFETY: `device_ptr` is a valid CUDA device. All `get_attribute` calls query + // read-only device properties via the CUDA driver API. + let hardware_props = unsafe { + use cudarc::driver::{result::device::get_attribute, sys::CUdevice_attribute::*}; + let warp_size = + get_attribute(device_ptr, CU_DEVICE_ATTRIBUTE_WARP_SIZE).unwrap() as u32; + let max_shared = get_attribute( + device_ptr, + CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, + ) + .unwrap() as usize; + let max_threads = get_attribute(device_ptr, CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK) + .unwrap() as u32; + let block_dim_x = + get_attribute(device_ptr, CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X).unwrap(); + let block_dim_y = + get_attribute(device_ptr, CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Y).unwrap(); + let block_dim_z = + get_attribute(device_ptr, CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z).unwrap(); + let max_cube_dim = (block_dim_x as u32, block_dim_y as u32, block_dim_z as u32); + + let grid_dim_x = get_attribute(device_ptr, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X).unwrap(); + let grid_dim_y = get_attribute(device_ptr, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Y).unwrap(); + let grid_dim_z = get_attribute(device_ptr, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Z).unwrap(); + let max_cube_count = (grid_dim_x as u32, grid_dim_y as u32, grid_dim_z as u32); + + let num_streaming_multiprocessors = Some( + get_attribute(device_ptr, CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT).unwrap() as u32, + ); + let num_tensor_cores = tensor_cores_per_sm(arch_version); + + comp_opts.warp_size = warp_size; + + HardwareProperties { + load_width: 128, + plane_size_min: warp_size, + plane_size_max: warp_size, + max_bindings: crate::device::CUDA_MAX_BINDINGS, + max_shared_memory_size: max_shared, + max_cube_count, + max_units_per_cube: max_threads, + max_cube_dim, + num_streaming_multiprocessors, + num_tensor_cores, + min_tensor_cores_dim: if supported_wmma_combinations.is_empty() { + None + } else { + Some(8) + }, + num_cpu_cores: None, + max_vector_size: VectorSize::MAX, + } + }; + + let mut device_props = DeviceProperties::new( + Default::default(), + mem_properties.clone(), + hardware_props, + TimingMethod::System, + ); + register_supported_types(&mut device_props); + device_props.register_type_usage(ElemType::Float(FloatKind::TF32), TypeUsage::Conversion); + if arch_version >= 60 { + device_props.register_atomic_type_usage( + Type::new(StorageType::Atomic(ElemType::Float(FloatKind::F64))), + AtomicUsage::Add | AtomicUsage::LoadStore, + ); + } + if arch_version >= 70 { + device_props.register_atomic_type_usage( + Type::new(StorageType::Atomic(ElemType::Float(FloatKind::F16))), + AtomicUsage::Add | AtomicUsage::LoadStore, + ); + device_props.register_atomic_type_usage( + Type::new(StorageType::Atomic(ElemType::Float(FloatKind::F16))).with_vector_size(2), + AtomicUsage::Add | AtomicUsage::LoadStore, + ); + device_props.register_semantic_type(SemanticType::Pipeline); + device_props + .register_type_usage(OpaqueType::Barrier(BarrierLevel::Unit), TypeUsage::Buffer); + device_props + .register_type_usage(OpaqueType::Barrier(BarrierLevel::Cube), TypeUsage::Buffer); + device_props.features.plane.insert(Plane::Sync); + comp_opts.supports_features.grid_constants = true; + } + + if arch_version >= 75 { + device_props + .features + .matmul + .ldmatrix + .insert(ElemType::Float(FloatKind::F16).into()); + device_props + .features + .matmul + .ldmatrix + .insert(ElemType::Float(FloatKind::BF16).into()); + comp_opts.supports_features.fast_tanh = CUDA_VERSION >= 12080; + } + + if arch_version >= 80 { + device_props.features.copy_async = true; + } + + // NOTE: I commented that since I observed synchronisation issues with atomic add for bf16. + // if arch.get_version() >= 80 { + // device_props.register_feature(Feature::Type(Elem::AtomicFloat(FloatKind::BF16))); + // } + + if arch_version >= 89 { + device_props.register_type_usage( + ElemType::Float(FloatKind::E4M3), + TypeUsage::Conversion | TypeUsage::Buffer, + ); + device_props.register_type_usage( + ElemType::Float(FloatKind::E5M2), + TypeUsage::Conversion | TypeUsage::Buffer, + ); + } + if arch_version >= 90 { + device_props.features.tma.insert(Tma::Base); + device_props.register_semantic_type(SemanticType::TensorMap); + device_props.features.cube_cluster = true; + comp_opts.supports_features.clusters = true; + comp_opts.supports_features.elect_sync = true; + device_props + .features + .matmul + .stmatrix + .insert(ElemType::Float(FloatKind::F16).into()); + device_props + .features + .matmul + .stmatrix + .insert(ElemType::Float(FloatKind::BF16).into()); + device_props.register_atomic_type_usage( + Type::new(StorageType::Atomic(ElemType::Float(FloatKind::F32))).with_vector_size(2), + AtomicUsage::LoadStore | AtomicUsage::Add, + ); + device_props.register_atomic_type_usage( + Type::new(StorageType::Atomic(ElemType::Float(FloatKind::F32))).with_vector_size(4), + AtomicUsage::LoadStore | AtomicUsage::Add, + ); + } + + if arch_version >= 100 { + device_props.features.tma.insert(Tma::Im2colWide); + // Breaks swizzle so disable for now and fix in a PR specifically for this + // if CUDA_VERSION >= 12090 { + // device_props.hardware.load_width = 256; + // } + } + + // NOTE: FP6/FP4 is explicitly not marked as forward compatible, but is compatible within a + // major version. Try to keep this up to date with new arch major revisions if they also + // implement it. + if arch_major == 10 || arch_major == 11 || arch_major == 12 { + device_props + .register_type_usage(ElemType::Float(FloatKind::E2M1), TypeUsage::Conversion); + device_props.register_type_usage( + StorageType::Packed(ElemType::Float(FloatKind::E2M1), 2), + TypeUsage::Conversion | TypeUsage::Buffer, + ); + device_props.register_type_usage( + ElemType::Float(FloatKind::E2M3), + TypeUsage::Conversion | TypeUsage::Buffer, + ); + device_props.register_type_usage( + ElemType::Float(FloatKind::E3M2), + TypeUsage::Conversion | TypeUsage::Buffer, + ); + device_props.register_type_usage( + ElemType::Float(FloatKind::UE8M0), + TypeUsage::Conversion | TypeUsage::Buffer, + ); + + if CUDA_VERSION >= 12080 { + device_props.features.tma.insert(Tma::SwizzleAtomicity); + } + } + + device_props.features.memory_reinterpret = true; + device_props.features.alignment = true; + device_props.features.plane.insert(Plane::Ops); + device_props + .features + .plane + .insert(Plane::NonUniformControlFlow); + + register_wmma_features(supported_wmma_combinations, &mut device_props); + register_mma_features(supported_mma_combinations, &mut device_props); + register_scaled_mma_features(supported_scaled_mma_combinations, &mut device_props); + + let cuda_ctx = CudaContext::new(comp_opts, device_props.clone(), ctx, arch); + let logger = Arc::new(ServerLogger::default()); + let policy = PitchedMemoryLayoutPolicy::new(device_props.memory.alignment as usize); + let utilities = ServerUtilities::new(device_props, logger, (), policy); + + CudaServer::new( + cuda_ctx, + mem_properties, + options.memory_config, + mem_alignment, + device_id, + utilities, + ) + } + + fn utilities(&self) -> ServerUtilitiesHandle { + self.utilities() as ServerUtilitiesHandle + } +} + +pub type CudaCompiler = CppCompiler>; +pub type CudaComputeKernel = ComputeKernel>; + +fn tensor_cores_per_sm(version: u32) -> Option { + match version { + 70 | 75 => Some(8), // Volta, Turing + 80 | 86 | 89 | 90 | 91 | 92 | 100 => Some(4), // Ampere, Hopper, Blackwell + _ => None, // Unknown or unsupported architecture + } +} + +impl Runtime for CudaRuntime { + type Compiler = CudaCompiler; + type Server = CudaServer; + type Device = CudaDevice; + + fn client(device: &Self::Device) -> ComputeClient { + ComputeClient::load(device) + } + + fn name(_client: &ComputeClient) -> &'static str { + "cuda" + } + + fn require_array_lengths() -> bool { + true + } + + fn max_cube_count() -> (u32, u32, u32) { + (i32::MAX as u32, u16::MAX as u32, u16::MAX as u32) + } + + fn can_read_tensor(shape: &Shape, strides: &Strides) -> bool { + has_pitched_row_major_strides(shape, strides) + } + + fn target_properties() -> TargetProperties { + TargetProperties { + mma: MmaProperties { + register_size_bits: 32, + const_plane_size: 32, + register_layout_a: MatrixLayout::RowMajor, + register_layout_b: MatrixLayout::ColMajor, + register_layout_acc: MatrixLayout::RowMajor, + register_duplication_a: 1, + register_duplication_b: 1, + register_duplication_acc: 1, + contiguous_elements: ContiguousElements::new(contiguous_elements_cuda), + }, + } + } + + fn enumerate_devices( + _: u16, + _: &::Info, + ) -> Vec { + let count = cudarc::driver::CudaContext::device_count().unwrap_or(0) as usize; + (0..count) + .map(|i| DeviceId { + type_id: 0, + index_id: i as u16, + }) + .collect() + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/.cargo-ok b/third_party/cubecl-runtime-0.10.0-patched/.cargo-ok new file mode 100644 index 00000000..5f8b7958 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/third_party/cubecl-runtime-0.10.0-patched/.cargo_vcs_info.json b/third_party/cubecl-runtime-0.10.0-patched/.cargo_vcs_info.json new file mode 100644 index 00000000..7d3d6874 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "7cf203735e095e640a2c03b2400d0faa03196bb4" + }, + "path_in_vcs": "crates/cubecl-runtime" +} \ No newline at end of file diff --git a/third_party/cubecl-runtime-0.10.0-patched/Cargo.lock b/third_party/cubecl-runtime-0.10.0-patched/Cargo.lock new file mode 100644 index 00000000..33b51673 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/Cargo.lock @@ -0,0 +1,1897 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cc" +version = "1.2.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "cubecl-common" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc956c2dcc993f16f748d03bfdbd900f75cab4d469f4e5d8924f8ee128e861ad" +dependencies = [ + "backtrace", + "bytemuck", + "cfg-if", + "cfg_aliases", + "derive-new", + "derive_more", + "dirs", + "embassy-futures", + "embassy-time", + "float4", + "float8", + "futures-lite", + "half", + "hashbrown 0.16.1", + "log", + "num-traits", + "oneshot", + "parking_lot", + "portable-atomic", + "portable-atomic-util", + "rand", + "sanitize-filename", + "serde", + "serde_bytes", + "serde_json", + "spin", + "toml", + "tracing", + "tynm", + "wasm-bindgen-futures", + "web-time", + "xxhash-rust", +] + +[[package]] +name = "cubecl-ir" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d28a897a40c8ee6c57a8b8d76d8823ba2e97f4c2b83db9338f17a0752132d486" +dependencies = [ + "cubecl-common", + "cubecl-macros-internal", + "derive-new", + "derive_more", + "enumset", + "float-ord", + "fnv", + "foldhash 0.2.0", + "half", + "hashbrown 0.16.1", + "num-traits", + "portable-atomic", + "serde", + "variadics_please", +] + +[[package]] +name = "cubecl-macros-internal" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a515651a5a91e25d87f71f311f80a088e6cf815798b9922560a97636da6b8740" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cubecl-runtime" +version = "0.10.0" +dependencies = [ + "ahash", + "async-channel", + "bytemuck", + "cfg-if", + "cfg_aliases", + "cubecl-common", + "cubecl-ir", + "cubecl-zspace", + "derive-new", + "derive_more", + "dirs", + "enumset", + "hashbrown 0.16.1", + "log", + "md5", + "rand", + "serde", + "serde_json", + "serial_test", + "spin", + "test-log", + "thiserror", + "toml", + "tracing", + "tracy-client", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "cubecl-zspace" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04054d95698afab785a4dda9f949e297831dd9e4cc6d036a93e6603f88e88b07" +dependencies = [ + "derive-new", + "serde", + "smallvec", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn", +] + +[[package]] +name = "derive-new" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", + "unicode-xid", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "embassy-futures" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc2d050bdc5c21e0862a89256ed8029ae6c290a93aecefc73084b3002cdebb01" + +[[package]] +name = "embassy-time" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "592b0c143ec626e821d4d90da51a2bd91d559d6c442b7c74a47d368c9e23d97a" +dependencies = [ + "cfg-if", + "critical-section", + "document-features", + "embassy-time-driver", + "embedded-hal 0.2.7", + "embedded-hal 1.0.0", + "embedded-hal-async", + "futures-core", +] + +[[package]] +name = "embassy-time-driver" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee71af1b3a0deaa53eaf2d39252f83504c853646e472400b763060389b9fcc9" +dependencies = [ + "document-features", +] + +[[package]] +name = "embedded-hal" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff" +dependencies = [ + "nb 0.1.3", + "void", +] + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "embedded-hal-async" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4c685bbef7fe13c3c6dd4da26841ed3980ef33e841cddfa15ce8a8fb3f1884" +dependencies = [ + "embedded-hal 1.0.0", +] + +[[package]] +name = "enumset" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96a4a12fe60ac746ae295a1a4ecb5bb02debc20856506c8635288065f142de" +dependencies = [ + "enumset_derive", + "serde", +] + +[[package]] +name = "enumset_derive" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd536557b58c682b217b8fb199afdff47cd3eff260623f19e77074eb073d63a" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + +[[package]] +name = "float4" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a5404bf31d22893d61cf24d4dda149d8e6b2ff07601c3cb3be651031f61a4ed" + +[[package]] +name = "float8" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d1f04709a8ac06e8e8042875a3c466cc4832d3c1a18dbcb9dba3c6e83046bc" +dependencies = [ + "half", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f04ae4152da20c76fe800fa48659201d5cf627c5149ca0b707b69d7eef6cf9" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", + "wasip2", + "wasip3", +] + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "serde", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +dependencies = [ + "libc", +] + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "nb" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f" +dependencies = [ + "nb 1.1.0", +] + +[[package]] +name = "nb" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5439c4ad607c3c23abf66de8c8bf57ba8adcd1f129e699851a6e43935d339d" + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oneshot" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe21416a02c693fb9f980befcb230ecc70b0b3d1cc4abf88b9675c4c1457f0c" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +dependencies = [ + "serde", +] + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "sanitize-filename" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" +dependencies = [ + "regex", +] + +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serial_test" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +dependencies = [ + "lock_api", + "portable-atomic", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test-log" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f46bf474f0a4afebf92f076d54fd5e63423d9438b8c278a3d2ccb0f47f7cdb3" +dependencies = [ + "test-log-macros", + "tracing-subscriber", +] + +[[package]] +name = "test-log-core" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d4d41320b48bc4a211a9021678fcc0c99569b594ea31c93735b8e517102b4c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "test-log-macros" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9beb9249a81e430dffd42400a49019bcf548444f1968ff23080a625de0d4d320" +dependencies = [ + "syn", + "test-log-core", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tracy-client" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f6fc3baeac5d86ab90c772e9e30620fc653bf1864295029921a15ef478e6a5" +dependencies = [ + "loom", + "once_cell", + "tracy-client-sys", +] + +[[package]] +name = "tracy-client-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f7c95348f20c1c913d72157b3c6dee6ea3e30b3d19502c5a7f6d3f160dacbf" +dependencies = [ + "cc", + "windows-targets", +] + +[[package]] +name = "tynm" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21cdb0fc8f85c98b1ec812bc4cd69faf6c0fa2fc17d44ea3c2cdd38dc08e999" +dependencies = [ + "nom", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "variadics_please" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b6d82be61465f97d42bd1d15bf20f3b0a3a0905018f38f9d6f6962055b0b5c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml b/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml new file mode 100644 index 00000000..869c82a8 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml @@ -0,0 +1,224 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +name = "cubecl-runtime" +version = "0.10.0" +authors = [ + "louisfd ", + "Nathaniel Simard", +] +build = "build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Crate that helps creating high performance async runtimes for CubeCL." +readme = "README.md" +keywords = [ + "deep-learning", + "machine-learning", + "data", +] +categories = ["science"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/tracel-ai/cubecl/tree/main/crates/cubecl-runtime" +resolver = "2" + +[features] +autotune-checks = [] +channel-cell = [] +channel-mpsc = [] +channel-mutex = [] +default = [ + "std", + "channel-mutex", + "channel-mpsc", + "channel-cell", + "storage-bytes", + "cubecl-common/default", +] +exclusive-memory-only = [] +profile-tracy = ["dep:tracy-client"] +std = [ + "cubecl-common/std", + "toml", + "dirs", + "thiserror/std", +] +storage-bytes = [] +tracing = [ + "dep:tracing", + "cubecl-common/tracing", + "cubecl-ir/tracing", +] + +[lib] +name = "cubecl_runtime" +path = "src/lib.rs" + +[[test]] +name = "integration_test" +path = "tests/integration_test.rs" + +[[bench]] +name = "dynamic" +path = "benches/dynamic.rs" +harness = false + +[dependencies.ahash] +version = "0.8.12" +default-features = false + +[dependencies.async-channel] +version = "2.3" + +[dependencies.bytemuck] +version = "1.16.1" + +[dependencies.cfg-if] +version = "1.0.0" + +[dependencies.cubecl-common] +version = "0.10.0" +default-features = false + +[dependencies.cubecl-ir] +version = "0.10.0" +features = ["serde"] +default-features = false + +[dependencies.cubecl-zspace] +version = "0.10.0" +default-features = false + +[dependencies.derive-new] +version = "0.7.0" +default-features = false + +[dependencies.derive_more] +version = "2" +features = ["eq"] +default-features = false + +[dependencies.dirs] +version = "6.0.0" +optional = true + +[dependencies.enumset] +version = "1.1.12" +default-features = false + +[dependencies.hashbrown] +version = "0.16" + +[dependencies.log] +version = "^0.4.22" +default-features = false + +[dependencies.md5] +version = "0.8.0" + +[dependencies.serde] +version = "1.0.204" +features = [ + "derive", + "alloc", +] +default-features = false + +[dependencies.thiserror] +version = "2" +default-features = false + +[dependencies.toml] +version = "1" +optional = true + +[dependencies.tracing] +version = "^0.1.43" +features = ["attributes"] +optional = true +default-features = false + +[dependencies.web-time] +version = "1.1.0" + +[dev-dependencies.rand] +version = "0.10.0" +features = [ + "std_rng", + "thread_rng", +] +default-features = false + +[dev-dependencies.serial_test] +version = "3.1.1" + +[dev-dependencies.test-log] +version = "^0.2" +features = ["trace"] +default-features = false + +[build-dependencies.cfg_aliases] +version = "0.2.1" + +[target.'cfg(any(target_os = "windows", target_os = "linux", target_os = "macos", target_os = "android"))'.dependencies.cubecl-common] +version = "0.10.0" +features = [ + "cache", + "serde", + "hash", +] +default-features = false + +[target.'cfg(any(target_os = "windows", target_os = "linux", target_os = "macos", target_os = "android"))'.dependencies.serde_json] +version = "1.0.119" +features = ["std"] +default-features = false + +[target.'cfg(any(target_os = "windows", target_os = "linux", target_os = "macos", target_os = "android"))'.dependencies.tracy-client] +version = "0.18.0" +optional = true + +[target.'cfg(not(target_has_atomic = "ptr"))'.dependencies.spin] +version = "0.10.0" +features = [ + "mutex", + "spin_mutex", + "mutex", + "spin_mutex", + "portable_atomic", +] + +[target.'cfg(target_family = "wasm")'.dependencies.wasm-bindgen-futures] +version = "0.4.45" + +[target.'cfg(target_has_atomic = "ptr")'.dependencies.spin] +version = "0.10.0" +features = [ + "mutex", + "spin_mutex", + "mutex", + "spin_mutex", +] + +[lints.clippy] +doc_markdown = "warn" + +[lints.rust] +warnings = "warn" + +[lints.rustdoc] +broken_intra_doc_links = "warn" +invalid_html_tags = "warn" diff --git a/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml.orig b/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml.orig new file mode 100644 index 00000000..01c3efcf --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml.orig @@ -0,0 +1,101 @@ +[package] +authors = ["louisfd ", "Nathaniel Simard"] +categories = ["science"] +description = "Crate that helps creating high performance async runtimes for CubeCL." +edition.workspace = true +keywords = ["deep-learning", "machine-learning", "data"] +license.workspace = true +name = "cubecl-runtime" +readme.workspace = true +repository = "https://github.com/tracel-ai/cubecl/tree/main/crates/cubecl-runtime" +version.workspace = true + + +[lints] +workspace = true + + +[features] +autotune-checks = [] +channel-cell = [] +channel-mpsc = [] +channel-mutex = [] +default = [ + "std", + "channel-mutex", + "channel-mpsc", + "channel-cell", + "storage-bytes", + "cubecl-common/default", +] +exclusive-memory-only = [] +profile-tracy = ["dep:tracy-client"] +std = ["cubecl-common/std", "toml", "dirs", "thiserror/std"] +storage-bytes = [] + +tracing = ["dep:tracing", "cubecl-common/tracing", "cubecl-ir/tracing"] + +[dependencies] +# Runtime Deps +log = { workspace = true } +tracing = { workspace = true, features = ["attributes"], optional = true } + +async-channel = { workspace = true } # Assume std +bytemuck = { workspace = true } +cfg-if = { workspace = true } +cubecl-common = { path = "../cubecl-common", version = "0.10.0", default-features = false } +cubecl-ir = { path = "../cubecl-ir", version = "0.10.0", default-features = false, features = [ + "serde", +] } +cubecl-zspace = { path = "../cubecl-zspace", version = "0.10.0", default-features = false } +derive-new = { workspace = true } +derive_more = { workspace = true, features = ["eq"] } +dirs = { workspace = true, optional = true } +enumset = { workspace = true } +hashbrown = { workspace = true } +serde = { workspace = true } +toml = { workspace = true, optional = true } +web-time = { workspace = true } + +thiserror = { workspace = true } +ahash = { version = "0.8.12", default-features = false } +md5 = { workspace = true } + +# Persistent cache deps - has to match the cfg(std_io) cfg. +[target.'cfg(any(target_os = "windows", target_os = "linux", target_os = "macos", target_os = "android"))'.dependencies] +cubecl-common = { path = "../cubecl-common", version = "0.10.0", default-features = false, features = [ + "cache", + "serde", + "hash", +] } +serde_json = { workspace = true, features = ["std"] } + +# Tracy if enabled. +tracy-client = { workspace = true, optional = true } + +[target.'cfg(target_has_atomic = "ptr")'.dependencies] +spin = { workspace = true, features = ["mutex", "spin_mutex"] } + + +[target.'cfg(not(target_has_atomic = "ptr"))'.dependencies] +spin = { workspace = true, features = [ + "mutex", + "spin_mutex", + "portable_atomic", +] } + + +[target.'cfg(target_family = "wasm")'.dependencies] +wasm-bindgen-futures = { workspace = true } + +[dev-dependencies] +rand = { workspace = true, features = ["thread_rng"] } +serial_test = { workspace = true } +test-log = { workspace = true, features = ["trace"] } + +[build-dependencies] +cfg_aliases = { workspace = true } + +[[bench]] +harness = false +name = "dynamic" diff --git a/third_party/cubecl-runtime-0.10.0-patched/LICENSE-APACHE b/third_party/cubecl-runtime-0.10.0-patched/LICENSE-APACHE new file mode 100644 index 00000000..a18fcc67 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2022 Nathaniel Simard & CubeCl Framework Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/third_party/cubecl-runtime-0.10.0-patched/LICENSE-MIT b/third_party/cubecl-runtime-0.10.0-patched/LICENSE-MIT new file mode 100644 index 00000000..6fca47c9 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Nathaniel Simard & CubeCL Framework Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md b/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md new file mode 100644 index 00000000..e692dd1b --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md @@ -0,0 +1,13 @@ +# Patch provenance + +Source: `cubecl-runtime` 0.10.0 from crates.io, checksum +`b68491bf5b3e997ae36bdc4e63b4ccd6d2f0e86b3b596a5d7a48d2b9e92622a0`. + +Local change: add one hidden `ExternalWriteServer` hook and one unsafe +`ComputeClient::external_write_stream` method. The hook resolves the normal +CubeCL binding dependencies before returning a backend-native stream token. +The token is used only inside `j2k-ml`'s lifetime-guarded CUDA event bridge; +it is never part of a public J2K or Burn adapter API. + +Remove this patch when CubeCL exposes an equivalent external-write/event +ordering contract. diff --git a/third_party/cubecl-runtime-0.10.0-patched/README.md b/third_party/cubecl-runtime-0.10.0-patched/README.md new file mode 100644 index 00000000..a3fe250e --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/README.md @@ -0,0 +1,7 @@ +# CubeCL Runtime + +This crate helps creating high performance async runtimes. + +- [x] Asynchronous kernel executions +- [x] Memory allocation management +- [x] Autotuning diff --git a/third_party/cubecl-runtime-0.10.0-patched/benches/dynamic.rs b/third_party/cubecl-runtime-0.10.0-patched/benches/dynamic.rs new file mode 100644 index 00000000..539af9de --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/benches/dynamic.rs @@ -0,0 +1,36 @@ +use cubecl_ir::MemoryDeviceProperties; +use cubecl_runtime::{ + logging::ServerLogger, + memory_management::{MemoryConfiguration, MemoryManagement, MemoryManagementOptions}, + storage::BytesStorage, +}; +use std::{collections::LinkedList, sync::Arc}; + +const MB: u64 = 1024 * 1024; + +fn main() { + let start = std::time::Instant::now(); + let storage = BytesStorage::default(); + let config = MemoryConfiguration::default(); + let mem_props = MemoryDeviceProperties { + max_page_size: 2048 * MB, + alignment: 32, + }; + let logger = Arc::new(ServerLogger::default()); + let mut mm = MemoryManagement::from_configuration( + storage, + &mem_props, + config, + logger, + MemoryManagementOptions::new("test"), + ); + let mut handles = LinkedList::new(); + for _ in 0..100 * 2048 { + if handles.len() >= 4000 { + handles.pop_front(); + } + let handle = mm.reserve(MB); + handles.push_back(handle); + } + println!("{:?}", start.elapsed()); +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/build.rs b/third_party/cubecl-runtime-0.10.0-patched/build.rs new file mode 100644 index 00000000..38ddbc69 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/build.rs @@ -0,0 +1,12 @@ +use cfg_aliases::cfg_aliases; + +fn main() { + // Setup cfg aliases + cfg_aliases! { + // Some features like autotune caching, compilation caching, and config loading + // require std with OS-level filesystem and environment access. + std_io: { all(feature = "std", any(target_os = "windows", target_os = "linux", target_os = "macos", target_os = "android")) }, + exclusive_memory_only: { any(feature = "exclusive-memory-only", target_family = "wasm") }, + multi_threading: { all(feature = "std", not(target_family = "wasm")) }, + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/allocator.rs b/third_party/cubecl-runtime-0.10.0-patched/src/allocator.rs new file mode 100644 index 00000000..63efcc06 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/allocator.rs @@ -0,0 +1,146 @@ +use crate::{ + memory_management::optimal_align, + server::{ + Handle, MemoryLayout, MemoryLayoutDescriptor, MemoryLayoutPolicy, MemoryLayoutStrategy, + }, +}; +use alloc::vec::Vec; +use cubecl_common::stream_id::StreamId; +use cubecl_zspace::{Shape, Strides, strides}; + +/// Allocators where every allocations is with contiguous memory. +pub struct ContiguousMemoryLayoutPolicy { + mem_alignment: usize, +} + +/// Allocators where some allocations can leverage a pitched layout. +pub struct PitchedMemoryLayoutPolicy { + mem_alignment: usize, +} + +impl MemoryLayoutPolicy for PitchedMemoryLayoutPolicy { + fn apply( + &self, + stream_id: StreamId, + descriptors: &[MemoryLayoutDescriptor], + ) -> (Handle, Vec) { + let mut total_size = 0u64; + + let (sizes, strides): (Vec<_>, Vec<_>) = descriptors + .iter() + .map(|descriptor| { + let last_dim = descriptor.shape.last().copied().unwrap_or(1); + let pitch_align = match descriptor.strategy { + MemoryLayoutStrategy::Contiguous => 1, + MemoryLayoutStrategy::Optimized => { + optimal_align(last_dim, descriptor.elem_size, self.mem_alignment) + } + }; + + let rank = descriptor.shape.len(); + let width = *descriptor.shape.last().unwrap_or(&1); + let height: usize = descriptor.shape.iter().rev().skip(1).product(); + let height = Ord::max(height, 1); + + let width_bytes = width * descriptor.elem_size; + let pitch = width_bytes.next_multiple_of(pitch_align); + let size = height * pitch; + + let mut strides = strides![1; rank]; + if rank > 1 { + strides[rank - 2] = pitch / descriptor.elem_size; + } + if rank > 2 { + for i in (0..rank - 2).rev() { + strides[i] = strides[i + 1] * descriptor.shape[i + 1]; + } + } + total_size += size.next_multiple_of(self.mem_alignment) as u64; + (size, strides) + }) + .unzip(); + + let base_handle = Handle::new(stream_id, total_size); + + let layouts = offset_handles(base_handle.clone(), &sizes, self.mem_alignment) + .into_iter() + .zip(strides) + .map(|(handle, strides)| MemoryLayout::new(handle, strides)) + .collect(); + (base_handle, layouts) + } +} + +impl ContiguousMemoryLayoutPolicy { + /// Creates a new allocator with the given memory alignment. + pub fn new(mem_alignment: usize) -> Self { + Self { mem_alignment } + } +} + +impl PitchedMemoryLayoutPolicy { + /// Creates a new allocator with the given memory alignment. + pub fn new(mem_alignment: usize) -> Self { + Self { mem_alignment } + } +} + +impl MemoryLayoutPolicy for ContiguousMemoryLayoutPolicy { + fn apply( + &self, + stream_id: StreamId, + descriptors: &[MemoryLayoutDescriptor], + ) -> (Handle, Vec) { + let mut total_size = 0u64; + let (sizes, strides): (Vec<_>, Vec<_>) = descriptors + .iter() + .map(|desc| { + let size = desc.shape.iter().product::() * desc.elem_size; + total_size += size.next_multiple_of(self.mem_alignment) as u64; + (size, contiguous_strides(&desc.shape)) + }) + .unzip(); + + let base_handle = Handle::new(stream_id, total_size); + + let layouts = offset_handles(base_handle.clone(), &sizes, self.mem_alignment) + .into_iter() + .zip(strides) + .map(|(handle, stride)| MemoryLayout::new(handle, stride)) + .collect(); + + (base_handle, layouts) + } +} + +pub(crate) fn contiguous_strides(shape: &Shape) -> Strides { + let rank = shape.len(); + let mut strides = strides![1; rank]; + for i in (0..rank - 1).rev() { + strides[i] = strides[i + 1] * shape[i + 1]; + } + strides +} + +/// Take a list of sub-slices of a buffer and create a list of offset handles. +/// Sizes must be in bytes and handles will be aligned to the memory alignment. +pub fn offset_handles( + base_handle: Handle, + sizes_bytes: &[usize], + buffer_align: usize, +) -> Vec { + let total_size = base_handle.size() as usize; + let mut offset = 0; + let mut out = Vec::new(); + + for size in sizes_bytes { + let handle = base_handle + .clone() + .offset_start(offset as u64) + .offset_end((total_size - offset - size) as u64); + out.push(handle); + offset += size.next_multiple_of(buffer_align); + } + + out +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/client.rs b/third_party/cubecl-runtime-0.10.0-patched/src/client.rs new file mode 100644 index 00000000..4f99d22a --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/client.rs @@ -0,0 +1,1076 @@ +use crate::{ + config::{TypeNameFormatLevel, type_name_format}, + kernel::KernelMetadata, + logging::ProfileLevel, + memory_management::{MemoryAllocationMode, MemoryUsage}, + runtime::Runtime, + server::{ + CommunicationId, ComputeServer, CopyDescriptor, CubeCount, ExecutionMode, + ExternalWriteServer, Handle, IoError, KernelArguments, MemoryLayout, + MemoryLayoutDescriptor, MemoryLayoutPolicy, MemoryLayoutStrategy, ProfileError, + ReduceOperation, ServerCommunication, ServerError, ServerUtilities, + }, + storage::{ComputeStorage, ManagedResource}, +}; +use alloc::{format, sync::Arc, vec, vec::Vec}; +use cubecl_common::{ + backtrace::BackTrace, + bytes::{AllocationProperty, Bytes}, + device::{Device, DeviceId}, + device_handle::DeviceHandle, + future::DynFut, + profile::ProfileDuration, +}; +use cubecl_ir::{DeviceProperties, ElemType, VectorSize, features::Features}; +use cubecl_zspace::Shape; + +#[allow(unused)] +use cubecl_common::profile::TimingMethod; +use cubecl_common::stream_id::StreamId; + +/// The `ComputeClient` is the entry point to require tasks from the `ComputeServer`. +/// It should be obtained for a specific device via the Compute struct. +pub struct ComputeClient { + device: DeviceHandle, + utilities: Arc>, + stream_id: Option, +} + +impl Clone for ComputeClient { + fn clone(&self) -> Self { + Self { + device: self.device.clone(), + utilities: self.utilities.clone(), + stream_id: self.stream_id, + } + } +} + +impl ComputeClient { + /// Get the info of the current backend. + pub fn info(&self) -> &::Info { + &self.utilities.info + } + + /// Create a new client with a new server. + pub fn init(device: &D, server: R::Server) -> Self { + let utilities = server.utilities(); + let context = DeviceHandle::::insert(device.to_id(), server) + .expect("Can't create a new client on an already registered server"); + + Self { + device: context, + utilities, + stream_id: None, + } + } + + /// Load the client for the given device. + pub fn load(device: &D) -> Self { + let context = DeviceHandle::::new(device.to_id()); + + // This is safe because we now know the return type of [`DeviceHandle::utilities()`]. + let utilities = context + .utilities() + .downcast::>() + .expect("Can downcast to `ServerUtilities`"); + + Self { + device: context, + utilities, + stream_id: None, + } + } + + fn stream_id(&self) -> StreamId { + match self.stream_id { + Some(val) => val, + None => StreamId::current(), + } + } + + /// Set the stream in which the current client is operating on. + /// + /// # Safety + /// + /// This is highly unsafe and should probably only be used by the CubeCL/Burn projects for now. + pub unsafe fn set_stream(&mut self, stream_id: StreamId) { + self.stream_id = Some(stream_id); + } + + fn do_read(&self, descriptors: Vec) -> DynFut, ServerError>> { + let stream_id = self.stream_id(); + self.device + .submit_blocking(move |server| server.read(descriptors, stream_id)) + .unwrap() + } + + /// Given bindings, returns owned resources as bytes. + pub fn read_async( + &self, + handles: Vec, + ) -> impl Future, ServerError>> + Send { + let shapes = handles + .iter() + .map(|it| [it.size_in_used() as usize].into()) + .collect::>(); + let descriptors = handles + .into_iter() + .zip(shapes) + .map(|(handle, shape)| CopyDescriptor::new(handle.binding(), shape, [1].into(), 1)) + .collect(); + + self.do_read(descriptors) + } + + /// Given bindings, returns owned resources as bytes. + /// + /// # Remarks + /// + /// Panics if the read operation fails. + pub fn read(&self, handles: Vec) -> Vec { + cubecl_common::reader::read_sync(self.read_async(handles)).expect("TODO") + } + + /// Given a binding, returns owned resource as bytes. + pub fn read_one(&self, handle: Handle) -> Result { + Ok(cubecl_common::reader::read_sync(self.read_async(vec![handle]))?.remove(0)) + } + + /// Given a binding, returns owned resource as bytes. + /// + /// # Remarks + /// + /// Panics if the read operation fails. Useful for tests. + pub fn read_one_unchecked(&self, handle: Handle) -> Bytes { + cubecl_common::reader::read_sync(self.read_async(vec![handle])) + .unwrap() + .remove(0) + } + + /// Given bindings, returns owned resources as bytes. + pub fn read_tensor_async( + &self, + descriptors: Vec, + ) -> impl Future, ServerError>> + Send { + self.do_read(descriptors) + } + + /// Given bindings, returns owned resources as bytes. + /// + /// # Remarks + /// + /// Panics if the read operation fails. + /// + /// The tensor must be in the same layout as created by the runtime, or more strict. + /// Contiguous tensors are always fine, strided tensors are only ok if the stride is similar to + /// the one created by the runtime (i.e. padded on only the last dimension). A way to check + /// stride compatibility on the runtime will be added in the future. + /// + /// Also see [`ComputeClient::create_tensor`]. + pub fn read_tensor(&self, descriptors: Vec) -> Vec { + cubecl_common::reader::read_sync(self.read_tensor_async(descriptors)).expect("TODO") + } + + /// Given a binding, returns owned resource as bytes. + /// See [`ComputeClient::read_tensor`] + pub fn read_one_tensor_async( + &self, + descriptor: CopyDescriptor, + ) -> impl Future> + Send { + let fut = self.read_tensor_async(vec![descriptor]); + + async { Ok(fut.await?.remove(0)) } + } + + /// Given a binding, returns owned resource as bytes. + /// + /// # Remarks + /// + /// Panics if the read operation fails. + /// See [`ComputeClient::read_tensor`] + pub fn read_one_unchecked_tensor(&self, descriptor: CopyDescriptor) -> Bytes { + self.read_tensor(vec![descriptor]).remove(0) + } + + /// Given a resource handle, returns the storage resource. + pub fn get_resource( + &self, + handle: Handle, + ) -> Result< + ManagedResource<<::Storage as ComputeStorage>::Resource>, + ServerError, + > { + let stream_id = self.stream_id(); + let binding = handle.binding(); + + self.device + .submit_blocking(move |state| state.get_resource(binding, stream_id)) + .unwrap() + } + + /// Resolve the native stream owning `handle` for a guarded external write. + /// + /// # Safety + /// + /// The returned numeric handle must stay internal to an audited backend + /// bridge. The caller must retain this client, the managed allocation, and + /// every submitted external resource until cross-stream dependencies have + /// been registered in both directions. It must not synchronize, destroy, + /// cache, or expose the stream handle. + #[doc(hidden)] + pub unsafe fn external_write_stream(&self, handle: &Handle) -> Result + where + R::Server: ExternalWriteServer, + { + let stream_id = self.stream_id(); + let binding = handle.clone().binding(); + self.device + .submit_blocking(move |server| server.external_write_stream(binding, stream_id)) + .unwrap() + } + + fn do_create_from_slices( + &self, + descriptors: Vec, + slices: Vec>, + ) -> Result, IoError> { + let stream_id = self.stream_id(); + let (handle_base, layouts) = self.utilities.layout_policy.apply(stream_id, &descriptors); + + let descriptors = descriptors + .into_iter() + .zip(layouts.iter()) + .zip(slices) + .map(|((desc, alloc), data)| { + ( + CopyDescriptor::new( + alloc.memory.clone().binding(), + desc.shape, + alloc.strides.clone(), + desc.elem_size, + ), + Bytes::from_bytes_vec(data.to_vec()), + ) + }) + .collect::>(); + + let (size, memory) = (handle_base.size(), handle_base.memory); + self.device.submit(move |server| { + server.initialize_memory(memory, size, stream_id); + server.write(descriptors, stream_id); + }); + + Ok(layouts) + } + + fn do_create( + &self, + descriptors: Vec, + mut data: Vec, + ) -> Result, IoError> { + self.staging(data.iter_mut(), true); + + let stream_id = self.stream_id(); + let (handle_base, layouts) = self.utilities.layout_policy.apply(stream_id, &descriptors); + + let descriptors = descriptors + .into_iter() + .zip(layouts.iter()) + .zip(data) + .map(|((desc, layout), data)| { + ( + CopyDescriptor::new( + layout.memory.clone().binding(), + desc.shape, + layout.strides.clone(), + desc.elem_size, + ), + Bytes::from_bytes_vec(data.to_vec()), + ) + }) + .collect::>(); + + let (size, memory) = (handle_base.size(), handle_base.memory); + self.device.submit(move |server| { + server.initialize_memory(memory, size, stream_id); + server.write(descriptors, stream_id); + }); + + Ok(layouts) + } + + /// Returns a resource handle containing the given data. + /// + /// # Notes + /// + /// Prefer using the more efficient [`Self::create`] function. + pub fn create_from_slice(&self, slice: &[u8]) -> Handle { + let shape: Shape = [slice.len()].into(); + + self.do_create_from_slices( + vec![MemoryLayoutDescriptor::new( + MemoryLayoutStrategy::Contiguous, + shape, + 1, + )], + vec![slice.to_vec()], + ) + .unwrap() + .remove(0) + .memory + } + + /// todo: docs + pub fn exclusive<'a, Re: Send + 'static, F: FnOnce() -> Re + Send + 'a>( + &'a self, + task: F, + ) -> Result { + // We then launch the task. + self.device + .exclusive(task) + .map_err(|err| ServerError::Generic { + reason: format!("Communication channel with the server is down: {err:?}"), + backtrace: BackTrace::capture(), + }) + } + + /// dodo: Docs + pub fn memory_persistent_allocation< + 'a, + Re: Send, + Input: Send, + F: FnOnce(Input) -> Re + Send + 'a, + >( + &'a self, + input: Input, + task: F, + ) -> Result { + let stream_id = StreamId::current(); + + self.device.submit(move |server| { + server.allocation_mode(MemoryAllocationMode::Persistent, stream_id); + }); + + // All tasks created on the same stream will have persistent memory. + let output = task(input); + + self.device.submit(move |server| { + server.allocation_mode(MemoryAllocationMode::Auto, stream_id); + }); + + Ok(output) + } + + /// Returns a resource handle containing the given [Bytes]. + pub fn create(&self, data: Bytes) -> Handle { + let shape = [data.len()].into(); + + self.do_create( + vec![MemoryLayoutDescriptor::new( + MemoryLayoutStrategy::Contiguous, + shape, + 1, + )], + vec![data], + ) + .unwrap() + .remove(0) + .memory + } + + /// Given a resource and shape, stores it and returns the tensor handle and strides. + /// This may or may not return contiguous strides. The layout is up to the runtime, and care + /// should be taken when indexing. + /// + /// Currently the tensor may either be contiguous (most runtimes), or "pitched", to use the CUDA + /// terminology. This means the last (contiguous) dimension is padded to fit a certain alignment, + /// and the strides are adjusted accordingly. This can make memory accesses significantly faster + /// since all rows are aligned to at least 16 bytes (the maximum load width), meaning the GPU + /// can load as much data as possible in a single instruction. It may be aligned even more to + /// also take cache lines into account. + /// + /// However, the stride must be taken into account when indexing and reading the tensor + /// (also see [`ComputeClient::read_tensor`]). + /// + /// # Notes + /// + /// Prefer using [`Self::create_tensor`] for better performance. + pub fn create_tensor_from_slice( + &self, + slice: &[u8], + shape: Shape, + elem_size: usize, + ) -> MemoryLayout { + self.do_create_from_slices( + vec![MemoryLayoutDescriptor::new( + MemoryLayoutStrategy::Optimized, + shape, + elem_size, + )], + vec![slice.to_vec()], + ) + .unwrap() + .remove(0) + } + + /// Given a resource and shape, stores it and returns the tensor handle and strides. + /// This may or may not return contiguous strides. The layout is up to the runtime, and care + /// should be taken when indexing. + /// + /// Currently the tensor may either be contiguous (most runtimes), or "pitched", to use the CUDA + /// terminology. This means the last (contiguous) dimension is padded to fit a certain alignment, + /// and the strides are adjusted accordingly. This can make memory accesses significantly faster + /// since all rows are aligned to at least 16 bytes (the maximum load width), meaning the GPU + /// can load as much data as possible in a single instruction. It may be aligned even more to + /// also take cache lines into account. + /// + /// However, the stride must be taken into account when indexing and reading the tensor + /// (also see [`ComputeClient::read_tensor`]). + pub fn create_tensor(&self, bytes: Bytes, shape: Shape, elem_size: usize) -> MemoryLayout { + self.do_create( + vec![MemoryLayoutDescriptor::new( + MemoryLayoutStrategy::Optimized, + shape, + elem_size, + )], + vec![bytes], + ) + .unwrap() + .remove(0) + } + + /// Reserves all `shapes` in a single storage buffer, copies the corresponding `data` into each + /// handle, and returns the handles for them. + /// See [`ComputeClient::create_tensor`] + /// + /// # Notes + /// + /// Prefer using [`Self::create_tensors`] for better performance. + pub fn create_tensors_from_slices( + &self, + descriptors: Vec<(MemoryLayoutDescriptor, &[u8])>, + ) -> Vec { + let mut data = Vec::with_capacity(descriptors.len()); + let mut descriptors_ = Vec::with_capacity(descriptors.len()); + for (a, b) in descriptors { + data.push(b.to_vec()); + descriptors_.push(a); + } + + self.do_create_from_slices(descriptors_, data).unwrap() + } + + /// Reserves all `shapes` in a single storage buffer, copies the corresponding `data` into each + /// handle, and returns the handles for them. + /// See [`ComputeClient::create_tensor`] + pub fn create_tensors( + &self, + descriptors: Vec<(MemoryLayoutDescriptor, Bytes)>, + ) -> Vec { + let (descriptors, data) = descriptors.into_iter().unzip(); + + self.do_create(descriptors, data).unwrap() + } + + fn do_empty( + &self, + descriptors: Vec, + ) -> Result, IoError> { + let stream_id = self.stream_id(); + let (handle_base, layouts) = self.utilities.layout_policy.apply(stream_id, &descriptors); + + let (size, memory) = (handle_base.size(), handle_base.memory); + self.device.submit(move |server| { + server.initialize_memory(memory, size, stream_id); + }); + + Ok(layouts) + } + + /// Reserves `size` bytes in the storage, and returns a handle over them. + pub fn empty(&self, size: usize) -> Handle { + let shape: Shape = [size].into(); + let descriptor = MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Contiguous, shape, 1); + self.do_empty(vec![descriptor]).unwrap().remove(0).memory + } + + /// Reserves `shape` in the storage, and returns a tensor handle for it. + /// See [`ComputeClient::create_tensor`] + pub fn empty_tensor(&self, shape: Shape, elem_size: usize) -> MemoryLayout { + let descriptor = + MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Optimized, shape, elem_size); + self.do_empty(vec![descriptor]).unwrap().remove(0) + } + + /// Reserves all `shapes` in a single storage buffer, and returns the handles for them. + /// See [`ComputeClient::create_tensor`] + pub fn empty_tensors(&self, descriptors: Vec) -> Vec { + self.do_empty(descriptors).unwrap() + } + + /// Marks the given [Bytes] as being a staging buffer, maybe transferring it to pinned memory + /// for faster data transfer with compute device. + /// + /// TODO: This blocks the compute queue, so it will drop the compute utilization. + pub fn staging<'a, I>(&self, bytes: I, file_only: bool) + where + I: Iterator, + { + let has_staging = |b: &Bytes| match b.property() { + AllocationProperty::Pinned => false, + AllocationProperty::File => true, + AllocationProperty::Native | AllocationProperty::Other => !file_only, + }; + + let mut to_be_updated = Vec::new(); + let sizes = bytes + .filter_map(|b| match has_staging(b) { + true => { + let len = b.len(); + to_be_updated.push(b); + Some(len) + } + false => None, + }) + .collect::>(); + + if sizes.is_empty() { + return; + } + + let stream_id = self.stream_id(); + let sizes = sizes.to_vec(); + let stagings = self + .device + .submit_blocking(move |server| server.staging(&sizes, stream_id)) + .unwrap(); + + let stagings = match stagings { + Ok(val) => val, + Err(_) => return, + }; + + to_be_updated + .into_iter() + .zip(stagings) + .for_each(|(b, mut staging)| { + b.copy_into(&mut staging); + core::mem::swap(b, &mut staging); + }); + } + + /// Transfer data from one client to another + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, src, dst_server)) + )] + pub fn to_client(&mut self, src: Handle, dst_server: &Self, dtype: ElemType) -> Handle { + let shape = [src.size_in_used() as usize]; + let src_descriptor = src.copy_descriptor(shape.into(), [1].into(), 1); + + if R::Server::SERVER_COMM_ENABLED { + self.to_client_tensor(src_descriptor, dst_server, dtype) + } else { + let alloc_desc = MemoryLayoutDescriptor::new( + MemoryLayoutStrategy::Contiguous, + src_descriptor.shape.clone(), + src_descriptor.elem_size, + ); + self.change_client_sync(src_descriptor, alloc_desc, dst_server) + .memory + } + } + + /// Perform an `all_reduce` operation on the given devices. + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, device_ids)) + )] + pub fn ensure_init_collective(&mut self, device_ids: Vec) { + let comm_id = CommunicationId::from(device_ids.clone()); + let is_comms_init = self + .utilities + .initialized_comms + .read() + .unwrap() + .contains(&comm_id); + if !is_comms_init { + self.device + .submit(move |server| server.comm_init(device_ids).unwrap()); + let mut initialized_comms = self.utilities.initialized_comms.write().unwrap(); + initialized_comms.insert(comm_id); + // Flush immediately so other devices aren't blocked waiting on this initialization. + self.device.flush_queue(); + } + } + + /// Wait on the communication stream. + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + pub fn sync_collective(&self) { + if DeviceHandle::::is_blocking() { + panic!("Can't use `sync_collective` with a blocking device handle"); + } + let stream_id = self.stream_id(); + + self.device.submit(move |server| { + server.sync_collective(stream_id).unwrap(); + }); + + // We don't actually need or want to sync the server here, but we need to make sure any + // task enqueued on the communication channel is done. + self.device.flush_queue(); + } + + /// Perform an `all_reduce` operation on the given devices. + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, src, dst, dtype, device_ids, op)) + )] + pub fn all_reduce( + &mut self, + src: Handle, + dst: Handle, + dtype: ElemType, + device_ids: Vec, + op: ReduceOperation, + ) { + if DeviceHandle::::is_blocking() { + panic!("Can't use `all_reduce` with a blocking device handle"); + } + + let stream_id = self.stream_id(); + let src = src.binding(); + let dst = dst.binding(); + + self.ensure_init_collective(device_ids.clone()); + + self.device.submit(move |server| { + server + .all_reduce(src, dst, dtype, stream_id, op, device_ids) + .unwrap(); + }); + } + + /// Transfer data from one client to another + /// + /// Make sure the source description can be read in a contiguous manner. + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, src_descriptor, dst_server)) + )] + pub fn to_client_tensor( + &mut self, + src_descriptor: CopyDescriptor, + dst_server: &Self, + dtype: ElemType, + ) -> Handle { + let stream_id_src = self.stream_id(); + let stream_id_dst = dst_server.stream_id(); + + let device_id_src = self.device.device_id(); + let device_id_dst = dst_server.device.device_id(); + + let mut dst_server = dst_server.clone(); + let handle = Handle::new(stream_id_dst, src_descriptor.handle.size_in_used()); + let handle_cloned = handle.clone(); + + let device_ids = vec![device_id_src, device_id_dst]; + self.ensure_init_collective(device_ids.clone()); + dst_server.ensure_init_collective(device_ids); + + self.device.submit(move |server_src| { + server_src + .send(src_descriptor, dtype, stream_id_src, device_id_dst) + .unwrap() + }); + + dst_server.device.submit(move |server_dst| { + server_dst + .recv(handle_cloned, dtype, stream_id_dst, device_id_src) + .unwrap(); + server_dst.sync_collective(stream_id_dst).unwrap(); + }); + + // `ServerCommunication::send` and`ServerCommunication::recv` are blocking: they each wait for the corresponding recv/send + // call to be made. We flush the operations right away so that the neither server ends up in a deadlock. + // The actual data transfer is still executed asynchronously on the communication stream. + self.device.flush_queue(); + dst_server.device.flush_queue(); + + handle + } + + #[track_caller] + #[cfg_attr(feature = "tracing", tracing::instrument(level="trace", + skip(self, kernel, bindings), + fields( + kernel.name = %kernel.name(), + kernel.id = %kernel.id(), + ) + ))] + unsafe fn launch_inner( + &self, + kernel: ::Kernel, + count: CubeCount, + bindings: KernelArguments, + mode: ExecutionMode, + stream_id: StreamId, + ) { + let level = self.utilities.logger.profile_level(); + + match level { + None | Some(ProfileLevel::ExecutionOnly) => { + let utilities = self.utilities.clone(); + self.device.submit(move |state| { + let name = kernel.name(); + unsafe { state.launch(kernel, count, bindings, mode, stream_id) }; + + if matches!(level, Some(ProfileLevel::ExecutionOnly)) { + let info = type_name_format(name, TypeNameFormatLevel::Balanced); + utilities.logger.register_execution(info); + } + }); + } + Some(level) => { + let name = kernel.name(); + let kernel_id = kernel.id(); + let context = self.device.clone(); + let count_moved = count.clone(); + let (result, profile) = self + .profile( + move || { + context + .submit_blocking(move |state| unsafe { + state.launch(kernel, count_moved, bindings, mode, stream_id) + }) + .unwrap() + }, + name, + ) + .unwrap(); + let info = match level { + ProfileLevel::Full => { + format!("{name}: {kernel_id} CubeCount {count:?}") + } + _ => type_name_format(name, TypeNameFormatLevel::Balanced), + }; + self.utilities.logger.register_profiled(info, profile); + result + } + } + } + + /// Launches the `kernel` with the given `bindings`. + #[track_caller] + pub fn launch( + &self, + kernel: ::Kernel, + count: CubeCount, + bindings: KernelArguments, + ) { + // SAFETY: Using checked execution mode. + unsafe { + self.launch_inner( + kernel, + count, + bindings, + ExecutionMode::Checked, + self.stream_id(), + ) + } + } + + /// Launches the `kernel` with the given `bindings` without performing any bound checks. + /// + /// # Safety + /// + /// To ensure this is safe, you must verify your kernel: + /// - Has no out-of-bound reads and writes that can happen. + /// - Has no infinite loops that might never terminate. + #[track_caller] + pub unsafe fn launch_unchecked( + &self, + kernel: ::Kernel, + count: CubeCount, + bindings: KernelArguments, + ) { + // SAFETY: Caller has to uphold kernel being safe. + unsafe { + self.launch_inner( + kernel, + count, + bindings, + match self.utilities.check_mode { + crate::config::compilation::BoundsCheckMode::Enforce => ExecutionMode::Checked, + crate::config::compilation::BoundsCheckMode::Validate => { + ExecutionMode::Validate + } + crate::config::compilation::BoundsCheckMode::Auto => ExecutionMode::Unchecked, + }, + self.stream_id(), + ) + } + } + + /// Flush all outstanding commands. + pub fn flush(&self) -> Result<(), ServerError> { + let stream_id = self.stream_id(); + + self.device + .submit_blocking(move |server| server.flush(stream_id)) + .unwrap() + } + + /// Wait for the completion of every task in the server. + pub fn sync(&self) -> DynFut> { + let stream_id = self.stream_id(); + + let fut = self + .device + .submit_blocking(move |server| server.sync(stream_id)) + .unwrap(); + + self.utilities.logger.profile_summary(); + + fut + } + + /// Get the features supported by the compute server. + pub fn properties(&self) -> &DeviceProperties { + &self.utilities.properties + } + + /// Get the features supported by the compute server. + pub fn features(&self) -> &Features { + &self.utilities.properties.features + } + + /// # Warning + /// + /// For private use only. + pub fn properties_mut(&mut self) -> Option<&mut DeviceProperties> { + Arc::get_mut(&mut self.utilities).map(|state| &mut state.properties) + } + + /// Get the current memory usage of this client. + pub fn memory_usage(&self) -> Result { + let stream_id = self.stream_id(); + self.device + .submit_blocking(move |server| server.memory_usage(stream_id)) + .unwrap() + } + + /// Get all devices of a specific type available to this runtime + pub fn enumerate_devices(&self, type_id: u16) -> Vec { + R::enumerate_devices(type_id, self.info()) + } + + /// Get all devices available to this runtime + pub fn enumerate_all_devices(&self) -> Vec { + R::enumerate_all_devices(self.info()) + } + + /// Get the number of devices of a specific type available to this runtime + pub fn device_count(&self, type_id: u16) -> usize { + self.enumerate_devices(type_id).len() + } + + /// Get the number of devices of a specific type available to this runtime + pub fn device_count_total(&self) -> usize { + self.enumerate_all_devices().len() + } + + /// Change the memory allocation mode. + /// + /// # Safety + /// + /// This function isn't thread safe and might create memory leaks. + pub unsafe fn allocation_mode(&self, mode: MemoryAllocationMode) { + let stream_id = self.stream_id(); + self.device + .submit(move |server| server.allocation_mode(mode, stream_id)); + } + + /// Ask the client to release memory that it can release. + /// + /// Nb: Results will vary on what the memory allocator deems beneficial, + /// so it's not guaranteed any memory is freed. + pub fn memory_cleanup(&self) { + let stream_id = self.stream_id(); + self.device + .submit(move |server| server.memory_cleanup(stream_id)); + } + + /// Measure the execution time of some inner operations. + #[track_caller] + pub fn profile( + &self, + func: impl FnOnce() -> O + Send, + #[allow(unused)] func_name: &str, + ) -> Result<(O, ProfileDuration), ProfileError> { + // Get the outer caller. For execute() this points straight to the + // cube kernel. For general profiling it points to whoever calls profile. + #[cfg(feature = "profile-tracy")] + let location = std::panic::Location::caller(); + + // Make a CPU span. If the server has system profiling this is all you need. + #[cfg(feature = "profile-tracy")] + let _span = tracy_client::Client::running().unwrap().span_alloc( + None, + func_name, + location.file(), + location.line(), + 0, + ); + + let stream_id = self.stream_id(); + + #[cfg(feature = "profile-tracy")] + let gpu_span = if self.utilities.properties.timing_method == TimingMethod::Device { + let gpu_span = self + .utilities + .gpu_client + .span_alloc(func_name, "profile", location.file(), location.line()) + .unwrap(); + Some(gpu_span) + } else { + None + }; + + let device = self.device.clone(); + #[allow(unused_mut, reason = "Used in profile-tracy")] + let mut result = self + .device + .exclusive(move || { + // We first get mut access to the server to create a token. + // Then we free to server, since it's going to be accessed in `func()`. + let token = + match device.submit_blocking(move |server| server.start_profile(stream_id)) { + Ok(token) => match token { + Ok(token) => token, + Err(err) => return Err(err), + }, + Err(err) => { + return Err(ServerError::Generic { + reason: alloc::format!( + "Can't start profiling because of a call error: {err:?}" + ), + backtrace: BackTrace::capture(), + }); + } + }; + + // We execute `func()` which will recursibly access the server. + let out = func(); + + // Finally we get the result from the token. + let result = device + .submit_blocking(move |server| { + let mut result = server.end_profile(stream_id, token); + + match result { + Ok(result) => Ok((out, result)), + Err(err) => Err(err), + } + }) + .unwrap(); + + Ok(result) + }) + .unwrap() + .map_err(|err| ProfileError::Unknown { + reason: alloc::format!("{err:?}"), + backtrace: BackTrace::capture(), + })?; + + #[cfg(feature = "profile-tracy")] + if let Some(mut gpu_span) = gpu_span { + gpu_span.end_zone(); + let epoch = self.utilities.epoch_time; + // Add in the work to upload the timestamp data. + result = result.map(|(o, result)| { + ( + o, + ProfileDuration::new( + alloc::boxed::Box::pin(async move { + let ticks = result.resolve().await; + let start_duration = + ticks.start_duration_since(epoch).as_nanos() as i64; + let end_duration = ticks.end_duration_since(epoch).as_nanos() as i64; + gpu_span.upload_timestamp_start(start_duration); + gpu_span.upload_timestamp_end(end_duration); + ticks + }), + TimingMethod::Device, + ), + ) + }); + } + + result + } + + /// Transfer data from one client to another + #[cfg_attr( + feature = "tracing", + tracing::instrument( + level = "trace", + skip(self, src_descriptor, alloc_descriptor, dst_server) + ) + )] + fn change_client_sync( + &self, + src_descriptor: CopyDescriptor, + alloc_descriptor: MemoryLayoutDescriptor, + dst_server: &Self, + ) -> MemoryLayout { + let shape = src_descriptor.shape.clone(); + let elem_size = src_descriptor.elem_size; + let stream_id = self.stream_id(); + + let read = self + .device + .submit_blocking(move |server| server.read(vec![src_descriptor], stream_id)) + .unwrap(); + + let mut data = cubecl_common::future::block_on(read).unwrap(); + + let (handle_base, mut layouts) = self + .utilities + .layout_policy + .apply(stream_id, &[alloc_descriptor]); + let alloc = layouts.remove(0); + + let desc_descriptor = CopyDescriptor { + handle: handle_base.clone().binding(), + shape, + strides: alloc.strides.clone(), + elem_size, + }; + + let (size, memory) = (handle_base.size(), handle_base.memory); + dst_server.device.submit(move |server| { + server.initialize_memory(memory, size, stream_id); + server.write(vec![(desc_descriptor, data.remove(0))], stream_id) + }); + + alloc + } + + /// Returns all vector sizes that are useful to perform optimal IO operation on the given element. + pub fn io_optimized_vector_sizes( + &self, + size: usize, + ) -> impl Iterator + Clone { + let load_width = self.properties().hardware.load_width as usize; + let size_bits = size * 8; + let max = load_width / size_bits; + let max = usize::min(self.properties().hardware.max_vector_size, max); + + // If the max is 8, we want to test 1, 2, 4, 8 which is log2(8) + 1. + let num_candidates = max.trailing_zeros() + 1; + + (0..num_candidates).map(|i| 2usize.pow(i)).rev() + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/compiler.rs b/third_party/cubecl-runtime-0.10.0-patched/src/compiler.rs new file mode 100644 index 00000000..f7e95a64 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/compiler.rs @@ -0,0 +1,91 @@ +use crate::{ + kernel::{CompiledKernel, KernelDefinition, KernelMetadata}, + server::ExecutionMode, +}; +use alloc::string::String; +use cubecl_common::backtrace::BackTrace; +use cubecl_ir::{ElemType, StorageType}; +use thiserror::Error; + +/// Kernel trait with the `ComputeShader` that will be compiled and cached based on the +/// provided id. +pub trait CubeTask: KernelMetadata + Send + Sync { + /// Compile a kernel and return the compiled form with an optional non-text representation + fn compile( + &self, + compiler: &mut C, + compilation_options: &C::CompilationOptions, + mode: ExecutionMode, + address_type: StorageType, + ) -> Result, CompilationError>; +} + +/// JIT compilation error. +#[derive(Error, Clone)] +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +pub enum CompilationError { + /// An instruction isn't supported. + #[error( + "An unsupported instruction caused the compilation to fail\nCaused by:\n {reason}\nBacktrace:\n{backtrace}" + )] + UnsupportedInstruction { + /// The caused of the error. + reason: String, + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// A generic compilation error. + #[error( + "An error caused the compilation to fail\nCaused by:\n {reason}\nBacktrace:\n{backtrace}" + )] + Generic { + /// The error context. + reason: String, + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + /// A generic compilation error. + #[error( + "A validation error caused the compilation to fail\nCaused by:\n {reason}\nBacktrace:\n{backtrace}" + )] + Validation { + /// The error context. + reason: String, + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, +} + +impl core::fmt::Debug for CompilationError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{self}") + } +} + +/// Compiles the representation into its own representation that can be formatted into tokens. +pub trait Compiler: Sync + Send + 'static + Clone + core::fmt::Debug { + /// The representation for the compiled code. + type Representation: core::fmt::Display; + /// The compilation options used to configure the compiler + type CompilationOptions: Send + Default + core::fmt::Debug; + + /// Compiles the [kernel definition](KernelDefinition) into the compiler's representation. + fn compile( + &mut self, + kernel: KernelDefinition, + compilation_options: &Self::CompilationOptions, + mode: ExecutionMode, + addr_type: StorageType, + ) -> Result; + + /// The size of the given element in bytes. + fn elem_size(&self, elem: ElemType) -> usize; + + /// The default extension for the runtime's kernel/shader code. + /// Might change based on which compiler is used. + fn extension(&self) -> &'static str; +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/config/autotune.rs b/third_party/cubecl-runtime-0.10.0-patched/src/config/autotune.rs new file mode 100644 index 00000000..17b47677 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/config/autotune.rs @@ -0,0 +1,61 @@ +#[cfg(std_io)] +use super::cache::CacheConfig; +use super::logger::{LogLevel, LoggerConfig}; + +/// Configuration for autotuning in `CubeCL`. +#[derive(Default, Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct AutotuneConfig { + /// Logger configuration for autotune logs, using autotune-specific log levels. + #[serde(default)] + pub logger: LoggerConfig, + + /// Autotune level, controlling the intensity of autotuning. + #[serde(default)] + pub level: AutotuneLevel, + + /// Cache location for storing autotune results. + #[serde(default)] + #[cfg(std_io)] + pub cache: CacheConfig, +} + +/// Log levels for autotune logging in `CubeCL`. +#[derive(Default, Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] +pub enum AutotuneLogLevel { + /// Autotune logging is disabled. + #[serde(rename = "disabled")] + Disabled, + + /// Minimal autotune information is logged such as the fastest kernel selected and a few + /// statistics (default). + #[default] + #[serde(rename = "minimal")] + Minimal, + + /// Full autotune details are logged. + #[serde(rename = "full")] + Full, +} + +impl LogLevel for AutotuneLogLevel {} + +/// Autotune levels controlling the intensity of autotuning. +#[derive(Default, Clone, Debug, serde::Serialize, serde::Deserialize)] +pub enum AutotuneLevel { + /// Minimal autotuning effort. + #[serde(rename = "minimal")] + Minimal, + + /// Balanced autotuning effort (default). + #[default] + #[serde(rename = "balanced")] + Balanced, + + /// Increased autotuning effort. + #[serde(rename = "extensive")] + Extensive, + + /// Maximum autotuning effort. + #[serde(rename = "full")] + Full, +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/config/base.rs b/third_party/cubecl-runtime-0.10.0-patched/src/config/base.rs new file mode 100644 index 00000000..8f8cac36 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/config/base.rs @@ -0,0 +1,219 @@ +use crate::config::memory::MemoryConfig; +use crate::config::streaming::StreamingConfig; + +use super::{autotune::AutotuneConfig, compilation::CompilationConfig, profiling::ProfilingConfig}; +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::sync::Arc; +use cubecl_common::config::RuntimeConfig; + +/// Static mutex holding the global configuration, initialized as `None`. +static CUBE_GLOBAL_CONFIG: spin::Mutex>> = spin::Mutex::new(None); + +/// Represents the global configuration for `CubeCL`, combining profiling, autotuning, and compilation settings. +#[derive(Default, Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct CubeClRuntimeConfig { + /// Configuration for profiling `CubeCL` operations. + #[serde(default)] + pub profiling: ProfilingConfig, + + /// Configuration for autotuning performance parameters. + #[serde(default)] + pub autotune: AutotuneConfig, + + /// Configuration for compilation settings. + #[serde(default)] + pub compilation: CompilationConfig, + + /// Configuration for streaming settings. + #[serde(default)] + pub streaming: StreamingConfig, + + /// Configuration for memory settings. + #[serde(default)] + pub memory: MemoryConfig, +} + +impl RuntimeConfig for CubeClRuntimeConfig { + fn storage() -> &'static spin::Mutex>> { + &CUBE_GLOBAL_CONFIG + } + + fn file_names() -> &'static [&'static str] { + &["cubecl.toml", "CubeCL.toml"] + } + + fn section_file_names() -> &'static [(&'static str, &'static str)] { + &[("burn.toml", "cubecl"), ("Burn.toml", "cubecl")] + } + + #[cfg(std_io)] + fn override_from_env(mut self) -> Self { + use super::compilation::CompilationLogLevel; + use crate::config::{ + autotune::{AutotuneLevel, AutotuneLogLevel}, + profiling::ProfilingLogLevel, + }; + + if let Ok(val) = std::env::var("CUBECL_DEBUG_LOG") { + self.compilation.logger.level = CompilationLogLevel::Full; + self.profiling.logger.level = ProfilingLogLevel::Medium; + self.autotune.logger.level = AutotuneLogLevel::Full; + + match val.as_str() { + "stdout" => { + self.compilation.logger.stdout = true; + self.profiling.logger.stdout = true; + self.autotune.logger.stdout = true; + } + "stderr" => { + self.compilation.logger.stderr = true; + self.profiling.logger.stderr = true; + self.autotune.logger.stderr = true; + } + "1" | "true" => { + let file_path = "/tmp/cubecl.log"; + self.compilation.logger.file = Some(file_path.into()); + self.profiling.logger.file = Some(file_path.into()); + self.autotune.logger.file = Some(file_path.into()); + } + "0" | "false" => { + self.compilation.logger.level = CompilationLogLevel::Disabled; + self.profiling.logger.level = ProfilingLogLevel::Disabled; + self.autotune.logger.level = AutotuneLogLevel::Disabled; + } + file_path => { + self.compilation.logger.file = Some(file_path.into()); + self.profiling.logger.file = Some(file_path.into()); + self.autotune.logger.file = Some(file_path.into()); + } + } + }; + + if let Ok(val) = std::env::var("CUBECL_DEBUG_OPTION") { + match val.as_str() { + "debug" => { + self.compilation.logger.level = CompilationLogLevel::Full; + self.profiling.logger.level = ProfilingLogLevel::Medium; + self.autotune.logger.level = AutotuneLogLevel::Full; + } + "debug-full" => { + self.compilation.logger.level = CompilationLogLevel::Full; + self.profiling.logger.level = ProfilingLogLevel::Full; + self.autotune.logger.level = AutotuneLogLevel::Full; + } + "profile" => { + self.profiling.logger.level = ProfilingLogLevel::Basic; + } + "profile-medium" => { + self.profiling.logger.level = ProfilingLogLevel::Medium; + } + "profile-full" => { + self.profiling.logger.level = ProfilingLogLevel::Full; + } + _ => {} + } + }; + + if let Ok(val) = std::env::var("CUBECL_AUTOTUNE_LEVEL") { + match val.as_str() { + "minimal" | "0" => { + self.autotune.level = AutotuneLevel::Minimal; + } + "balanced" | "1" => { + self.autotune.level = AutotuneLevel::Balanced; + } + "extensive" | "2" => { + self.autotune.level = AutotuneLevel::Extensive; + } + "full" | "3" => { + self.autotune.level = AutotuneLevel::Full; + } + _ => {} + } + } + + self + } +} + +#[derive(Clone, Copy, Debug)] +/// How to format cubecl type names. +pub enum TypeNameFormatLevel { + /// No formatting apply, full information is included. + Full, + /// Most information is removed for a small formatted name. + Short, + /// Balanced info is kept. + Balanced, +} + +/// Format a type name with different options. +pub fn type_name_format(name: &str, level: TypeNameFormatLevel) -> String { + match level { + TypeNameFormatLevel::Full => name.to_string(), + TypeNameFormatLevel::Short => { + if let Some(val) = name.split("<").next() { + val.split("::").last().unwrap_or(name).to_string() + } else { + name.to_string() + } + } + TypeNameFormatLevel::Balanced => { + let mut split = name.split("<"); + let before_generic = split.next(); + let after_generic = split.next(); + + let before_generic = match before_generic { + None => return name.to_string(), + Some(val) => val + .split("::") + .last() + .unwrap_or(val) + .trim() + .replace(">", "") + .to_string(), + }; + let inside_generic = match after_generic { + None => return before_generic.to_string(), + Some(val) => { + let mut val = val.to_string(); + for s in split { + val += "<"; + val += s; + } + val + } + }; + + let inside = type_name_list_format(&inside_generic, level); + + format!("{before_generic}{inside}") + } + } +} + +fn type_name_list_format(name: &str, level: TypeNameFormatLevel) -> String { + let mut acc = String::new(); + let splits = name.split(", "); + + for a in splits { + acc += " | "; + acc += &type_name_format(a, level); + } + + acc +} + +#[cfg(test)] +mod test { + use super::*; + + #[test_log::test] + fn test_format_name() { + let full_name = "burn_cubecl::kernel::unary_numeric::unary_numeric::UnaryNumeric::copy::Copy, cubecl_cuda::runtime::CudaRuntime>"; + let name = type_name_format(full_name, TypeNameFormatLevel::Balanced); + + assert_eq!(name, "UnaryNumeric | f32 | CubeTensor | Copy | CudaRuntime"); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/config/cache.rs b/third_party/cubecl-runtime-0.10.0-patched/src/config/cache.rs new file mode 100644 index 00000000..43ba62aa --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/config/cache.rs @@ -0,0 +1,61 @@ +/// Cache location options. +#[derive(Default, Clone, Debug, serde::Serialize, serde::Deserialize)] +pub enum CacheConfig { + /// Stores cache in the current working directory. + #[serde(rename = "local")] + Local, + + /// Stores cache in the project's `target` directory (default). + #[default] + #[serde(rename = "target")] + Target, + + /// Stores cache in the system's local configuration directory. + #[serde(rename = "global")] + Global, + + /// Stores cache in a user-specified file path. + #[serde(rename = "file")] + File(std::path::PathBuf), +} + +impl CacheConfig { + /// Returns the root directory for the cache. + pub fn root(&self) -> std::path::PathBuf { + match self { + Self::Local => std::env::current_dir().unwrap(), + Self::Target => { + let dir_original = + std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("/")); + let mut dir = dir_original.clone(); + + // Search for Cargo.toml in parent directories to locate project root. + loop { + if let Ok(true) = std::fs::exists(dir.join("Cargo.toml")) { + return dir.join("target"); + } + + if !dir.pop() { + break; + } + } + + // No Cargo.toml anywhere above cwd — this is a bundled or + // installed application (Tauri, GUI app, CLI installed via + // cargo install, etc.) running outside a workspace. The + // previous fallback of `dir_original.join("target")` became + // `/target` when cwd was `/`, which fails on most platforms + // with EROFS (read-only system volume on macOS, root-owned + // on Linux) and cascaded a `CacheFile::new` directory + // failure into the whole autotune pipeline. Use the + // platform-appropriate user cache directory instead. + if let Some(cache) = dirs::cache_dir() { + return cache.join("cubecl"); + } + dir_original.join("target") + } + Self::Global => dirs::config_local_dir().unwrap(), + Self::File(path_buf) => path_buf.clone(), + } + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/config/compilation.rs b/third_party/cubecl-runtime-0.10.0-patched/src/config/compilation.rs new file mode 100644 index 00000000..19ff1d94 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/config/compilation.rs @@ -0,0 +1,53 @@ +#[cfg(std_io)] +use super::cache::CacheConfig; +use super::logger::{LogLevel, LoggerConfig}; + +/// Configuration for compilation settings in `CubeCL`. +#[derive(Default, Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct CompilationConfig { + /// Logger configuration for compilation logs, using binary log levels. + #[serde(default)] + pub logger: LoggerConfig, + /// Cache location for storing compiled kernels. + #[serde(default)] + #[cfg(std_io)] + pub cache: Option, + /// Controls whether kernel launches enforce bounds checks. + #[serde(default)] + pub check_mode: BoundsCheckMode, +} + +/// Bounds checks options. +#[derive(Default, Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] +pub enum BoundsCheckMode { + #[serde(rename = "enforce")] + /// Always enforce bounds checks on every kernel launch. + Enforce, + #[serde(rename = "validate")] + /// Always enforce bounds checks on every kernel launch, and validate unchecked kernels for OOB. + Validate, + /// Enforce bounds checking on standard launches, but skip checks on + /// explicitly unchecked launches for better performance. + #[default] + #[serde(rename = "auto")] + Auto, +} + +/// Log levels for compilation in `CubeCL`. +#[derive(Default, Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] +pub enum CompilationLogLevel { + /// Compilation logging is disabled. + #[default] + #[serde(rename = "disabled")] + Disabled, + + /// Basic compilation information is logged such as when kernels are compiled. + #[serde(rename = "basic")] + Basic, + + /// Full compilation details are logged including source code. + #[serde(rename = "full")] + Full, +} + +impl LogLevel for CompilationLogLevel {} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/config/logger.rs b/third_party/cubecl-runtime-0.10.0-patched/src/config/logger.rs new file mode 100644 index 00000000..2f331978 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/config/logger.rs @@ -0,0 +1,135 @@ +use super::{CubeClRuntimeConfig, RuntimeConfig}; +use crate::config::{ + autotune::AutotuneLogLevel, compilation::CompilationLogLevel, memory::MemoryLogLevel, + profiling::ProfilingLogLevel, streaming::StreamingLogLevel, +}; +use alloc::{sync::Arc, vec::Vec}; +use core::fmt::Display; + +use cubecl_common::config::logger::LoggerSinks; +pub(crate) use cubecl_common::config::logger::{LogLevel, LoggerConfig}; + +/// Central logging utility for `CubeCL`, managing multiple log outputs. +#[derive(Debug)] +pub struct Logger { + sinks: LoggerSinks, + compilation_index: Vec, + profiling_index: Vec, + autotune_index: Vec, + streaming_index: Vec, + memory_index: Vec, + /// Global configuration for logging settings. + pub config: Arc, +} + +impl Default for Logger { + fn default() -> Self { + Self::new() + } +} + +impl Logger { + /// Creates a new `Logger` instance based on the global configuration. + /// + /// Note that creating a logger is quite expensive. + pub fn new() -> Self { + let config = CubeClRuntimeConfig::get(); + let mut sinks = LoggerSinks::new(); + + let compilation_index = register_enabled( + &mut sinks, + &config.compilation.logger, + !matches!( + config.compilation.logger.level, + CompilationLogLevel::Disabled + ), + ); + let profiling_index = register_enabled( + &mut sinks, + &config.profiling.logger, + !matches!(config.profiling.logger.level, ProfilingLogLevel::Disabled), + ); + let autotune_index = register_enabled( + &mut sinks, + &config.autotune.logger, + !matches!(config.autotune.logger.level, AutotuneLogLevel::Disabled), + ); + let streaming_index = register_enabled( + &mut sinks, + &config.streaming.logger, + !matches!(config.streaming.logger.level, StreamingLogLevel::Disabled), + ); + let memory_index = register_enabled( + &mut sinks, + &config.memory.logger, + !matches!(config.memory.logger.level, MemoryLogLevel::Disabled), + ); + + Self { + sinks, + compilation_index, + profiling_index, + autotune_index, + streaming_index, + memory_index, + config, + } + } + + /// Logs a message for streaming, directing it to all configured streaming loggers. + pub fn log_streaming(&mut self, msg: &S) { + self.sinks.log(&self.streaming_index, msg); + } + + /// Logs a message for memory, directing it to all configured memory loggers. + pub fn log_memory(&mut self, msg: &S) { + self.sinks.log(&self.memory_index, msg); + } + + /// Logs a message for compilation, directing it to all configured compilation loggers. + pub fn log_compilation(&mut self, msg: &S) { + self.sinks.log(&self.compilation_index, msg); + } + + /// Logs a message for profiling, directing it to all configured profiling loggers. + pub fn log_profiling(&mut self, msg: &S) { + self.sinks.log(&self.profiling_index, msg); + } + + /// Logs a message for autotuning, directing it to all configured autotuning loggers. + pub fn log_autotune(&mut self, msg: &S) { + self.sinks.log(&self.autotune_index, msg); + } + + /// Returns the current streaming log level from the global configuration. + pub fn log_level_streaming(&self) -> StreamingLogLevel { + self.config.streaming.logger.level + } + + /// Returns the current autotune log level from the global configuration. + pub fn log_level_autotune(&self) -> AutotuneLogLevel { + self.config.autotune.logger.level + } + + /// Returns the current compilation log level from the global configuration. + pub fn log_level_compilation(&self) -> CompilationLogLevel { + self.config.compilation.logger.level + } + + /// Returns the current profiling log level from the global configuration. + pub fn log_level_profiling(&self) -> ProfilingLogLevel { + self.config.profiling.logger.level + } +} + +fn register_enabled( + sinks: &mut LoggerSinks, + config: &LoggerConfig, + enabled: bool, +) -> Vec { + if enabled { + sinks.register(config) + } else { + Vec::new() + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/config/memory.rs b/third_party/cubecl-runtime-0.10.0-patched/src/config/memory.rs new file mode 100644 index 00000000..40395635 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/config/memory.rs @@ -0,0 +1,48 @@ +use super::logger::{LogLevel, LoggerConfig}; + +/// Configuration for memory settings in `CubeCL`. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, Default)] +pub struct MemoryConfig { + /// Logger configuration for memory-related logs, using specific log levels. + #[serde(default)] + pub logger: LoggerConfig, + /// Configuration for persistent memory pools. + #[serde(default)] + pub persistent_memory: PersistentMemory, +} + +/// Configuration options for persistent memory pools in `CubeCL` runtimes. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, Default)] +pub enum PersistentMemory { + /// Persistent memory is enabled but used only when explicitly specified. + #[default] + #[serde(rename = "enabled")] + Enabled, + /// Persistent memory is disabled, allowing dynamic allocations. + #[serde(rename = "disabled")] + Disabled, + /// Persistent memory is enforced, preventing dynamic allocations. + /// + /// # Warning + /// + /// Enforcing persistent memory may cause out-of-memory errors if tensors of varying sizes are used. + #[serde(rename = "enforced")] + Enforced, +} + +/// Log levels for memory-related events in `CubeCL`. +#[derive(Default, Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] +pub enum MemoryLogLevel { + /// No memory-related logging. + #[default] + #[serde(rename = "disabled")] + Disabled, + /// Logs basic memory events, such as creating memory pages and manually cleaning memory. + #[serde(rename = "basic")] + Basic, + /// Logs detailed memory information. + #[serde(rename = "full")] + Full, +} + +impl LogLevel for MemoryLogLevel {} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/config/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/src/config/mod.rs new file mode 100644 index 00000000..e3421dfd --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/config/mod.rs @@ -0,0 +1,20 @@ +/// Autotune config module. +pub mod autotune; +/// Cache config module. +#[cfg(std_io)] +pub mod cache; +/// Compilation config module. +pub mod compilation; +/// Memory config module. +pub mod memory; +/// Profiling config module. +pub mod profiling; +/// Streaming config module. +pub mod streaming; + +mod base; +mod logger; + +pub use base::*; +pub use cubecl_common::config::RuntimeConfig; +pub use logger::Logger; diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/config/profiling.rs b/third_party/cubecl-runtime-0.10.0-patched/src/config/profiling.rs new file mode 100644 index 00000000..33b1dce7 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/config/profiling.rs @@ -0,0 +1,36 @@ +use super::logger::{LogLevel, LoggerConfig}; + +/// Configuration for profiling settings in `CubeCL`. +#[derive(Default, Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct ProfilingConfig { + /// Logger configuration for profiling logs, using profiling-specific log levels. + #[serde(default)] + pub logger: LoggerConfig, +} + +/// Log levels for profiling in `CubeCL`. +#[derive(Default, Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] +pub enum ProfilingLogLevel { + /// Profiling logging is disabled. + #[default] + #[serde(rename = "disabled")] + Disabled, + + /// Only the kernels that run are logged without timing. + #[serde(rename = "minimal")] + Minimal, + + /// Basic profiling information is logged. + #[serde(rename = "basic")] + Basic, + + /// Medium level of profiling details is logged. + #[serde(rename = "medium")] + Medium, + + /// Full profiling details are logged. + #[serde(rename = "full")] + Full, +} + +impl LogLevel for ProfilingLogLevel {} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/config/streaming.rs b/third_party/cubecl-runtime-0.10.0-patched/src/config/streaming.rs new file mode 100644 index 00000000..a10fc726 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/config/streaming.rs @@ -0,0 +1,44 @@ +use super::logger::{LogLevel, LoggerConfig}; + +/// Configuration for streaming settings in `CubeCL`. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct StreamingConfig { + /// Logger configuration for streaming logs, using binary log levels. + #[serde(default)] + pub logger: LoggerConfig, + /// The maximum number of streams to be used. + #[serde(default = "default_max_streams")] + pub max_streams: u8, +} + +impl Default for StreamingConfig { + fn default() -> Self { + Self { + logger: Default::default(), + max_streams: default_max_streams(), + } + } +} + +fn default_max_streams() -> u8 { + 128 +} + +/// Log levels for streaming in `CubeCL`. +#[derive(Default, Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] +pub enum StreamingLogLevel { + /// Compilation logging is disabled. + #[default] + #[serde(rename = "disabled")] + Disabled, + + /// Basic streaming information is logged such as when streams are merged. + #[serde(rename = "basic")] + Basic, + + /// Full streaming details are logged. + #[serde(rename = "full")] + Full, +} + +impl LogLevel for StreamingLogLevel {} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/id.rs b/third_party/cubecl-runtime-0.10.0-patched/src/id.rs new file mode 100644 index 00000000..f2c055a3 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/id.rs @@ -0,0 +1,266 @@ +use alloc::format; +use alloc::string::String; +use alloc::sync::Arc; +use core::{ + any::{Any, TypeId}, + fmt::Display, + hash::{Hash, Hasher}, +}; +use cubecl_common::{ + format::{DebugRaw, format_str}, + hash::{StableHash, StableHasher}, +}; +use cubecl_ir::AddressType; +use derive_more::{Eq, PartialEq}; + +use crate::server::{CubeDim, ExecutionMode}; + +#[macro_export(local_inner_macros)] +/// Create a new storage ID type. +macro_rules! storage_id_type { + ($name:ident) => { + /// Storage ID. + #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug, PartialOrd, Ord)] + pub struct $name { + value: usize, + } + + impl $name { + /// Create a new ID. + pub fn new() -> Self { + use core::sync::atomic::{AtomicUsize, Ordering}; + + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + let value = COUNTER.fetch_add(1, Ordering::Relaxed); + if value == usize::MAX { + core::panic!("Memory ID overflowed"); + } + Self { value } + } + } + + impl Default for $name { + fn default() -> Self { + Self::new() + } + } + }; +} + +/// Kernel unique identifier. +#[derive(Clone, PartialEq, Eq)] +pub struct KernelId { + #[eq(skip)] + type_name: &'static str, + pub(crate) type_id: core::any::TypeId, + pub(crate) address_type: AddressType, + /// The [`CubeDim`] for this kernel + pub cube_dim: CubeDim, + pub(crate) mode: ExecutionMode, + pub(crate) info: Option, +} + +impl Hash for KernelId { + fn hash(&self, state: &mut H) { + self.type_id.hash(state); + self.address_type.hash(state); + self.cube_dim.hash(state); + self.mode.hash(state); + self.info.hash(state); + } +} + +impl core::fmt::Debug for KernelId { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut debug_str = f.debug_struct("KernelId"); + debug_str + .field("type", &DebugRaw(self.type_name)) + .field("address_type", &self.address_type); + debug_str.field("cube_dim", &self.cube_dim); + debug_str.field("mode", &self.mode); + match &self.info { + Some(info) => debug_str.field("info", info), + None => debug_str.field("info", &self.info), + }; + debug_str.finish() + } +} + +impl Display for KernelId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match &self.info { + Some(info) => f.write_str( + format_str( + format!("{info:?}").as_str(), + &[('(', ')'), ('[', ']'), ('{', '}')], + true, + ) + .as_str(), + ), + None => f.write_str("No info"), + } + } +} + +impl KernelId { + /// Create a new [kernel id](KernelId) for a type. + pub fn new() -> Self { + Self { + type_id: core::any::TypeId::of::(), + type_name: core::any::type_name::(), + info: None, + cube_dim: CubeDim::new_single(), + mode: ExecutionMode::Checked, + address_type: Default::default(), + } + } + + /// Render the key in a standard format that can be used between runs. + /// + /// Can be used as a persistent kernel cache key. + pub fn stable_format(&self) -> String { + format!( + "{}-{}-{:?}-{:?}-{:?}", + self.type_name, self.address_type, self.cube_dim, self.mode, self.info + ) + } + + /// Hash the key in a stable way that can be used between runs. + /// + /// Can be used as a persistent kernel cache key. + pub fn stable_hash(&self) -> StableHash { + let mut hasher = StableHasher::new(); + self.type_name.hash(&mut hasher); + self.address_type.hash(&mut hasher); + self.cube_dim.hash(&mut hasher); + self.mode.hash(&mut hasher); + self.info.hash(&mut hasher); + + hasher.finalize() + } + + /// Add information to the [kernel id](KernelId). + /// + /// The information is used to differentiate kernels of the same kind but with different + /// configurations, which affect the generated code. + pub fn info( + mut self, + info: I, + ) -> Self { + self.info = Some(Info::new(info)); + self + } + + /// Set the [execution mode](ExecutionMode). + pub fn mode(&mut self, mode: ExecutionMode) { + self.mode = mode; + } + + /// Set the [cube dim](CubeDim). + pub fn cube_dim(mut self, cube_dim: CubeDim) -> Self { + self.cube_dim = cube_dim; + self + } + + /// Set the [`AddressType`]. + pub fn address_type(mut self, addr_ty: AddressType) -> Self { + self.address_type = addr_ty; + self + } +} + +impl core::fmt::Debug for Info { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.value.fmt(f) + } +} + +impl Info { + fn new(id: T) -> Self { + Self { + value: Arc::new(id), + } + } +} + +/// This trait allows various types to be used as keys within a single data structure. +/// +/// The downside is that the hashing method is hardcoded and cannot be configured using the +/// [`core::hash::Hash`] function. The provided [Hasher] will be modified, but only based on the +/// result of the hash from the [`DefaultHasher`]. +trait DynKey: core::fmt::Debug + Send + Sync { + fn dyn_type_id(&self) -> TypeId; + fn dyn_eq(&self, other: &dyn DynKey) -> bool; + fn dyn_hash(&self, state: &mut dyn Hasher); + fn dyn_hash_one(&self) -> StableHash; + fn as_any(&self) -> &dyn Any; +} + +impl PartialEq for Info { + fn eq(&self, other: &Self) -> bool { + self.value.dyn_eq(other.value.as_ref()) + } +} + +/// Extra information +#[derive(Clone)] +pub(crate) struct Info { + value: Arc, +} +impl Eq for Info {} + +impl Hash for Info { + fn hash(&self, state: &mut H) { + self.value.dyn_type_id().hash(state); + self.value.dyn_hash(state) + } +} + +impl DynKey for T { + fn dyn_eq(&self, other: &dyn DynKey) -> bool { + if let Some(other) = other.as_any().downcast_ref::() { + self == other + } else { + false + } + } + + fn dyn_type_id(&self) -> TypeId { + TypeId::of::() + } + + fn dyn_hash(&self, state: &mut dyn Hasher) { + let hash = self.dyn_hash_one(); + state.write_u128(hash); + } + + fn dyn_hash_one(&self) -> StableHash { + let mut hasher = StableHasher::new(); + self.hash(&mut hasher); + hasher.finalize() + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + #[test_log::test] + pub fn kernel_id_hash() { + let value_1 = KernelId::new::<()>().info("1"); + let value_2 = KernelId::new::<()>().info("2"); + + let mut set = HashSet::new(); + + set.insert(value_1.clone()); + + assert!(set.contains(&value_1)); + assert!(!set.contains(&value_2)); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/kernel.rs b/third_party/cubecl-runtime-0.10.0-patched/src/kernel.rs new file mode 100644 index 00000000..c08f10b9 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/kernel.rs @@ -0,0 +1,296 @@ +use alloc::{ + boxed::Box, + string::{String, ToString}, + vec::Vec, +}; +use core::{ + fmt::Display, + marker::PhantomData, + sync::atomic::{AtomicI8, Ordering}, +}; + +use cubecl_common::format::format_str; +use cubecl_ir::{Id, Scope, StorageType, Type}; +use serde::{Deserialize, Serialize}; + +use crate::{ + compiler::{CompilationError, Compiler, CubeTask}, + config::{CubeClRuntimeConfig, RuntimeConfig, compilation::CompilationLogLevel}, + id::KernelId, + server::{CubeDim, ExecutionMode}, +}; + +/// Implement this trait to create a [kernel definition](KernelDefinition). +pub trait KernelMetadata: Send + Sync + 'static { + /// Name of the kernel for debugging. + fn name(&self) -> &'static str { + core::any::type_name::() + } + + /// Identifier for the kernel, used for caching kernel compilation. + fn id(&self) -> KernelId; + + /// Type of addresses in this kernel + fn address_type(&self) -> StorageType; +} + +#[derive(Debug, Clone)] +#[allow(missing_docs)] +pub struct KernelDefinition { + pub buffers: Vec, + pub tensor_maps: Vec, + pub scalars: Vec, + pub cube_dim: CubeDim, + pub body: Scope, + pub options: KernelOptions, +} + +#[derive(Default, Clone, Debug, Hash, PartialEq, Eq)] +/// Options for a specific kernel compilation +pub struct KernelOptions { + /// The name of the kernel + pub kernel_name: String, + /// Whether to include debug symbols + pub debug_symbols: bool, + /// CUDA Cluster dim, if any + pub cluster_dim: Option, +} + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +/// Global argument of a kernel. +pub struct KernelArg { + /// The kernel id. + pub id: Id, + /// Whether the global argument can only be accessed for reading, or if it can also be accessed + /// for write. + pub visibility: Visibility, + /// The type of the argument. + pub ty: Type, + /// The size of the argument. + pub size: Option, + /// Whether the argument has metadata. + pub has_extended_meta: bool, +} + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct ScalarKernelArg { + pub ty: StorageType, + pub count: usize, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize)] +#[allow(missing_docs)] +pub enum Visibility { + Read, + ReadWrite, +} + +/// A kernel, compiled in the target language +pub struct CompiledKernel { + /// The name of the kernel entrypoint. + /// For example + /// + /// ```text + /// #[cube(launch)] + /// fn gelu_array() {} + /// ``` + /// + /// would have the entrypoint name "`gelu_array`". + pub entrypoint_name: String, + + /// A fully qualified debug name of the kernel. + /// + /// For example + /// + /// ```text + /// #[cube(launch)] + /// fn gelu_array() {} + /// ``` + /// + /// would have a debug name such as + /// + /// ```text + /// gelu::gelu_array::GeluArray< + /// cubecl_core::frontend::element::float::F32, + /// cubecl_cuda::runtime::CudaRuntime, + /// > + /// ``` + pub debug_name: Option<&'static str>, + + /// Source code of the kernel + pub source: String, + /// In-memory representation of the kernel + pub repr: Option, + /// Size of a cube for the compiled kernel + pub cube_dim: CubeDim, + /// Extra debugging information about the compiled kernel. + pub debug_info: Option, +} + +/// Extra debugging information about the compiled kernel. +#[derive(new)] +pub struct DebugInformation { + /// The language tag of the source.. + pub lang_tag: &'static str, + /// The compilation id. + pub id: KernelId, +} + +/// Kernel that can be defined +pub trait CubeKernel: KernelMetadata { + /// Define the kernel for compilation + fn define(&self) -> KernelDefinition; +} + +/// Wraps a [`CubeKernel`] to allow it be compiled. +pub struct KernelTask { + kernel_definition: K, + _compiler: PhantomData, +} + +/// Generic [`CubeTask`] for compiling kernels +pub struct CubeTaskKernel { + /// The inner compilation task being wrapped + pub task: Box>, +} + +impl KernelTask { + /// Create a new kernel task + pub fn new(kernel_definition: K) -> Self { + Self { + kernel_definition, + _compiler: PhantomData, + } + } +} + +impl CubeTask for KernelTask { + fn compile( + &self, + compiler: &mut C, + compilation_options: &C::CompilationOptions, + mode: ExecutionMode, + addr_type: StorageType, + ) -> Result, CompilationError> { + let gpu_ir = self.kernel_definition.define(); + let entrypoint_name = gpu_ir.options.kernel_name.clone(); + let cube_dim = gpu_ir.cube_dim; + let lower_level_ir = compiler.compile(gpu_ir, compilation_options, mode, addr_type)?; + + Ok(CompiledKernel { + entrypoint_name, + debug_name: Some(core::any::type_name::()), + source: lower_level_ir.to_string(), + repr: Some(lower_level_ir), + cube_dim, + debug_info: None, + }) + } +} + +impl KernelMetadata for KernelTask { + // Forward ID to underlying kernel definition. + fn id(&self) -> KernelId { + self.kernel_definition.id() + } + + // Forward name to underlying kernel definition. + fn name(&self) -> &'static str { + self.kernel_definition.name() + } + + fn address_type(&self) -> StorageType { + self.kernel_definition.address_type() + } +} + +impl KernelMetadata for Box> { + // Deref and use existing ID. + fn id(&self) -> KernelId { + self.as_ref().id() + } + + // Deref and use existing name. + fn name(&self) -> &'static str { + self.as_ref().name() + } + + fn address_type(&self) -> StorageType { + self.as_ref().address_type() + } +} + +static COMPILATION_LEVEL: AtomicI8 = AtomicI8::new(-1); + +fn compilation_level() -> u8 { + let compilation_level = COMPILATION_LEVEL.load(Ordering::Relaxed); + if compilation_level == -1 { + let val = match CubeClRuntimeConfig::get().compilation.logger.level { + CompilationLogLevel::Full => 2, + CompilationLogLevel::Disabled => 0, + CompilationLogLevel::Basic => 1, + }; + + COMPILATION_LEVEL.store(val, Ordering::Relaxed); + val as u8 + } else { + compilation_level as u8 + } +} + +impl Display for CompiledKernel { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match compilation_level() { + 2 => self.format_full(f), + _ => self.format_basic(f), + } + } +} + +impl CompiledKernel { + fn format_basic(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("[Compiling kernel]")?; + if let Some(name) = self.debug_name { + if name.len() <= 32 { + f.write_fmt(format_args!(" {name}"))?; + } else { + f.write_fmt(format_args!(" {}", name.split('<').next().unwrap_or("")))?; + } + } + + Ok(()) + } + + fn format_full(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("[START_KERNEL_COMPILATION]")?; + + if let Some(name) = self.debug_name { + if name.len() <= 32 { + f.write_fmt(format_args!("\nname: {name}"))?; + } else { + let name = format_str(name, &[('<', '>')], false); + f.write_fmt(format_args!("\nname: {name}"))?; + } + } + + if let Some(info) = &self.debug_info { + f.write_fmt(format_args!("\nid: {:#?}", info.id))?; + } + + f.write_fmt(format_args!( + " +source: +```{} +{} +``` +[END_KERNEL_COMPILATION] +", + self.debug_info + .as_ref() + .map(|info| info.lang_tag) + .unwrap_or(""), + self.source + )) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/lib.rs b/third_party/cubecl-runtime-0.10.0-patched/src/lib.rs new file mode 100644 index 00000000..a262b97a --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/lib.rs @@ -0,0 +1,58 @@ +#![no_std] +#![warn(missing_docs)] + +//! `CubeCL` runtime crate that helps creating high performance async runtimes. + +#[cfg(feature = "std")] +extern crate std; + +extern crate alloc; + +#[macro_use] +extern crate derive_new; + +/// Various identifier types used in `CubeCL`. +pub mod id; + +/// Kernel related traits. +pub mod kernel; + +/// Stream related utilities. +pub mod stream; + +/// Compute client module. +pub mod client; + +/// Autotune module +pub mod tune; + +/// Memory management module. +pub mod memory_management; +/// Compute server module. +pub mod server; +/// Compute Storage module. +pub mod storage; + +/// `CubeCL` config module. +pub mod config; + +pub use cubecl_common::benchmark; + +/// Logging utilities to be used by a compute server. +pub mod logging; + +/// TMA-related runtime types +pub mod tma; + +/// Compiler trait and related types +pub mod compiler; +/// Runtime trait and related types +pub mod runtime; +/// Simple system profiling using timestamps. +pub mod timestamp_profiler; + +/// Validation utils for shared properties +pub mod validation; + +/// Allocators moddule. +pub mod allocator; diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/logging/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/src/logging/mod.rs new file mode 100644 index 00000000..85646a02 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/logging/mod.rs @@ -0,0 +1,6 @@ +mod profiling; +pub use profiling::*; + +mod server; + +pub use server::*; diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/logging/profiling.rs b/third_party/cubecl-runtime-0.10.0-patched/src/logging/profiling.rs new file mode 100644 index 00000000..542f90ad --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/logging/profiling.rs @@ -0,0 +1,159 @@ +use alloc::{ + format, + string::{String, ToString}, + vec::Vec, +}; +use core::fmt::Display; +use hashbrown::HashMap; + +#[derive(Debug, Default)] +pub(crate) struct Profiled { + durations: HashMap, +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct ProfileItem { + total_duration: core::time::Duration, + num_computed: usize, +} + +#[derive(Debug, Copy, Clone)] +/// Control the amount of info being display when profiling. +pub enum ProfileLevel { + /// Provide only the summary information about kernels being run. + Basic, + /// Provide the summary information about kernels being run with their trace. + Medium, + /// Provide more information about kernels being run. + Full, + /// Only the execution are logged. + ExecutionOnly, +} + +impl Profiled { + /// If some computation was profiled. + pub fn is_empty(&self) -> bool { + self.durations.is_empty() + } + pub fn update(&mut self, name: &String, duration: core::time::Duration) { + let name = if name.contains("\n") { + name.split("\n").next().unwrap() + } else { + name + }; + if let Some(item) = self.durations.get_mut(name) { + item.update(duration); + } else { + self.durations.insert( + name.to_string(), + ProfileItem { + total_duration: duration, + num_computed: 1, + }, + ); + } + } +} + +impl Display for Profiled { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let header_name = "Name"; + let header_num_computed = "Num Computed"; + let header_duration = "Duration"; + let header_ratio = "Ratio"; + + let mut ratio_len = header_ratio.len(); + let mut name_len = header_name.len(); + let mut num_computed_len = header_num_computed.len(); + let mut duration_len = header_duration.len(); + + let mut total_duration = core::time::Duration::from_secs(0); + let mut total_computed = 0; + + let mut items: Vec<(String, String, String, core::time::Duration)> = self + .durations + .iter() + .map(|(key, item)| { + let name = key.clone(); + let num_computed = format!("{}", item.num_computed); + let duration = format!("{:?}", item.total_duration); + + name_len = usize::max(name_len, name.len()); + num_computed_len = usize::max(num_computed_len, num_computed.len()); + duration_len = usize::max(duration_len, duration.len()); + + total_duration += item.total_duration; + total_computed += item.num_computed; + + (name, num_computed, duration, item.total_duration) + }) + .collect(); + + let total_duration_fmt = format!("{total_duration:?}"); + let total_compute_fmt = format!("{total_computed:?}"); + let total_ratio_fmt = "100 %"; + + duration_len = usize::max(duration_len, total_duration_fmt.len()); + num_computed_len = usize::max(num_computed_len, total_compute_fmt.len()); + ratio_len = usize::max(ratio_len, total_ratio_fmt.len()); + + let line_length = name_len + duration_len + num_computed_len + ratio_len + 11; + + let write_line = |char: &str, f: &mut core::fmt::Formatter<'_>| { + writeln!(f, "|{}| ", char.repeat(line_length)) + }; + items.sort_by(|(_, _, _, a), (_, _, _, b)| b.cmp(a)); + + write_line("⎺", f)?; + + writeln!( + f, + "| {header_name:, + log_compile_info: bool, + log_streaming: StreamingLogLevel, + log_channel: Option>, + log_memory: MemoryLogLevel, +} + +impl Default for ServerLogger { + fn default() -> Self { + let logger = Logger::new(); + + let disabled = matches!( + logger.config.compilation.logger.level, + CompilationLogLevel::Disabled + ) && matches!( + logger.config.profiling.logger.level, + ProfilingLogLevel::Disabled + ) && matches!(logger.config.memory.logger.level, MemoryLogLevel::Disabled) + && matches!( + logger.config.streaming.logger.level, + StreamingLogLevel::Disabled + ); + + if disabled { + return Self { + profile_level: None, + log_compile_info: false, + log_streaming: StreamingLogLevel::Disabled, + log_channel: None, + log_memory: MemoryLogLevel::Disabled, + }; + } + let profile_level = match logger.config.profiling.logger.level { + ProfilingLogLevel::Disabled => None, + ProfilingLogLevel::Minimal => Some(ProfileLevel::ExecutionOnly), + ProfilingLogLevel::Basic => Some(ProfileLevel::Basic), + ProfilingLogLevel::Medium => Some(ProfileLevel::Medium), + ProfilingLogLevel::Full => Some(ProfileLevel::Full), + }; + + let log_compile_info = match logger.config.compilation.logger.level { + CompilationLogLevel::Disabled => false, + CompilationLogLevel::Basic => true, + CompilationLogLevel::Full => true, + }; + let log_streaming = logger.config.streaming.logger.level; + let log_memory = logger.config.memory.logger.level; + + let (send, rec) = async_channel::unbounded(); + + // Spawn the logger as a detached task. + let async_logger = AsyncLogger { + message: rec, + logger, + profiled: Default::default(), + }; + // Spawn the future in the background to logs messages / durations. + spawn_detached_fut(async_logger.process()); + + Self { + profile_level, + log_compile_info, + log_streaming, + log_memory, + log_channel: Some(send), + } + } +} + +impl ServerLogger { + /// Returns the profile level, none if profiling is deactivated. + pub fn profile_level(&self) -> Option { + self.profile_level + } + + /// Returns true if compilation info should be logged. + pub fn compilation_activated(&self) -> bool { + self.log_compile_info + } + + /// Log the argument to a file when the compilation logger is activated. + pub fn log_compilation(&self, arg: &I) + where + I: Display, + { + if let Some(channel) = &self.log_channel + && self.log_compile_info + { + // Channel will never be full, don't care if it's closed. + let _ = channel.try_send(LogMessage::Compilation(arg.to_string())); + } + } + + /// Log the argument to the logger when the streaming logger is activated. + pub fn log_streaming String, C: FnOnce(StreamingLogLevel) -> bool>( + &self, + cond: C, + format: I, + ) { + if let Some(channel) = &self.log_channel + && cond(self.log_streaming) + { + // Channel will never be full, don't care if it's closed. + let _ = channel.try_send(LogMessage::Streaming(format())); + } + } + + /// Log the argument to the logger when the memory logger is activated. + pub fn log_memory String, C: FnOnce(MemoryLogLevel) -> bool>( + &self, + cond: C, + format: I, + ) { + if let Some(channel) = &self.log_channel + && cond(self.log_memory) + { + // Channel will never be full, don't care if it's closed. + let _ = channel.try_send(LogMessage::Memory(format())); + } + } + + /// Register a profiled task without timing. + pub fn register_execution(&self, name: impl Display) { + if let Some(channel) = &self.log_channel + && matches!(self.profile_level, Some(ProfileLevel::ExecutionOnly)) + { + // Channel will never be full, don't care if it's closed. + let _ = channel.try_send(LogMessage::Execution(name.to_string())); + } + } + + /// Register a profiled task. + pub fn register_profiled(&self, name: impl Display, duration: ProfileDuration) { + if let Some(channel) = &self.log_channel + && self.profile_level.is_some() + { + // Channel will never be full, don't care if it's closed. + let _ = channel.try_send(LogMessage::Profile(name.to_string(), duration)); + } + } + + /// Show the profiling summary if activated and reset its state. + pub fn profile_summary(&self) { + if let Some(channel) = &self.log_channel + && self.profile_level.is_some() + { + // Channel will never be full, don't care if it's closed. + let _ = channel.try_send(LogMessage::ProfileSummary); + } + } +} + +struct AsyncLogger { + message: Receiver, + logger: Logger, + profiled: Profiled, +} + +impl AsyncLogger { + async fn process(mut self) { + while let Ok(msg) = self.message.recv().await { + match msg { + LogMessage::Compilation(msg) => { + self.logger.log_compilation(&msg); + } + LogMessage::Streaming(msg) => { + self.logger.log_streaming(&msg); + } + LogMessage::Memory(msg) => { + self.logger.log_memory(&msg); + } + LogMessage::Profile(name, profile) => { + let duration = profile.resolve().await.duration(); + self.profiled.update(&name, duration); + self.logger + .log_profiling(&format!("| {duration:<10?} | {name}")); + } + LogMessage::Execution(name) => { + self.logger.log_profiling(&format!("Executing {name}")); + } + LogMessage::ProfileSummary => { + if !self.profiled.is_empty() { + self.logger.log_profiling(&self.profiled); + self.profiled = Profiled::default(); + } + } + } + } + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/base.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/base.rs new file mode 100644 index 00000000..84f15291 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/base.rs @@ -0,0 +1,103 @@ +use alloc::string::{String, ToString}; + +/// Amount of memory in use by this allocator +/// and statistics on how much memory is reserved and +/// wasted in total. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MemoryUsage { + /// The number of allocations currently active. + /// + /// This is not the number of times an actual allocation happens to create a new memory page, + /// but really the number of active slices. + pub number_allocs: u64, + /// The number of bytes that are currently actually in use. + /// + /// This doesn't include any padding or other memory that needs to be + /// reserved, and is the minimum amount of memory that could possible + /// be allocated. + pub bytes_in_use: u64, + /// The amount of bytes used for padding memory in currently active allocations. + pub bytes_padding: u64, + /// The total amount of memory reserved on the device. + /// + /// This will be at least as much as `bytes_in_use` but in practice will + /// be higher, as allocations reserve memory for future allocations + /// and for padding. + pub bytes_reserved: u64, +} + +impl MemoryUsage { + /// Calculate the combined memory usage of two reports (summing them). + pub fn combine(&self, other: MemoryUsage) -> MemoryUsage { + MemoryUsage { + number_allocs: self.number_allocs + other.number_allocs, + bytes_in_use: self.bytes_in_use + other.bytes_in_use, + bytes_padding: self.bytes_padding + other.bytes_padding, + bytes_reserved: self.bytes_reserved + other.bytes_reserved, + } + } +} + +#[derive(new)] +pub(crate) struct BytesFormat { + bytes: u64, +} + +impl core::fmt::Display for BytesFormat { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let unit = 1000; + + if self.bytes < unit { + f.write_fmt(format_args!("{} B", self.bytes)) + } else { + let size = self.bytes as f64; + let exp = match size.log(1000.0).floor() as usize { + 0 => 1, + e => e, + }; + let unit_prefix = "KMGTPEZY".as_bytes(); + f.write_fmt(format_args!( + "{:.2} {}B", + (size / unit.pow(exp as u32) as f64), + unit_prefix[exp - 1] as char, + )) + } + } +} + +fn bytes_format(bytes: u64) -> String { + BytesFormat::new(bytes).to_string() +} + +impl core::fmt::Display for MemoryUsage { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // In the future it'd be nice if MemoryUsage also held some stats about say, + // the 5 biggest allocations, to show when you an OOM. + let usage_percentage = (self.bytes_in_use as f32 / self.bytes_reserved as f32) * 100.0; + let padding_percentage = (self.bytes_padding as f32 / self.bytes_in_use as f32) * 100.0; + writeln!(f, "Memory Usage Report:")?; + writeln!(f, " Number of allocations: {}", self.number_allocs)?; + writeln!(f, " Bytes in use: {}", bytes_format(self.bytes_in_use))?; + writeln!( + f, + " Bytes used for padding: {}", + bytes_format(self.bytes_padding) + )?; + writeln!( + f, + " Total bytes reserved: {}", + bytes_format(self.bytes_reserved) + )?; + writeln!(f, " Usage efficiency: {usage_percentage:.2}%")?; + writeln!(f, " Padding overhead: {padding_percentage:.2}%") + } +} + +/// The managed tensor buffer handle that points to some memory segment. +/// It should not contain actual data. +pub trait MemoryHandle: Clone + core::fmt::Debug { + /// Checks if the underlying memory can be safely mutated. + fn can_mut(&self) -> bool; + /// Get the binding associated to the current handle. + fn binding(self) -> Binding; +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/mod.rs new file mode 100644 index 00000000..bff0a141 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/mod.rs @@ -0,0 +1,5 @@ +mod policy; +mod queue; + +pub use policy::FlushingPolicy; +pub use queue::{Fence, PendingDropQueue}; diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/policy.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/policy.rs new file mode 100644 index 00000000..9129e73a --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/policy.rs @@ -0,0 +1,112 @@ +use cubecl_common::bytes::Bytes; + +/// Defines the thresholds that determine when a [`PendingDropQueue`] should be +/// flushed. +/// +/// A flush is triggered when **either** limit is exceeded — whichever comes +/// first. Set a field to `u32::MAX` / `usize::MAX` to effectively disable it. +#[derive(Debug)] +pub struct FlushingPolicy { + /// Flush when this many allocations have been staged. + pub max_bytes_count: u32, + /// Flush when the total staged size reaches this many bytes. + pub max_bytes_size: u32, +} + +impl Default for FlushingPolicy { + fn default() -> Self { + Self { + max_bytes_count: 64, + max_bytes_size: 64 * 1024 * 1024, // 64 MiB + } + } +} + +/// Tracks staged allocations and evaluates them against a [`FlushingPolicy`]. +#[derive(Default, Debug)] +pub(crate) struct FlushingPolicyState { + bytes_count: u32, + bytes_size: u32, +} + +impl FlushingPolicyState { + /// Record a newly staged [`Bytes`] allocation. + pub(crate) fn register(&mut self, bytes: &Bytes) { + self.bytes_count += 1; + self.bytes_size += bytes.len() as u32; + } + + /// Reset all counters, typically called after a flush. + pub(crate) fn reset(&mut self) { + self.bytes_count = 0; + self.bytes_size = 0; + } + + /// Returns `true` if either threshold in `policy` has been reached. + pub(crate) fn should_flush(&self, policy: &FlushingPolicy) -> bool { + self.bytes_count >= policy.max_bytes_count || self.bytes_size >= policy.max_bytes_size + } +} + +#[cfg(test)] +mod policy_tests { + use std::vec; + + use super::*; + + fn policy() -> FlushingPolicy { + FlushingPolicy { + max_bytes_count: 4, + max_bytes_size: 100, + } + } + + fn state() -> FlushingPolicyState { + FlushingPolicyState { + bytes_count: 0, + bytes_size: 0, + } + } + + #[test] + fn no_flush_when_below_both_thresholds() { + let s = state(); + assert!(!s.should_flush(&policy())); + } + + #[test] + fn flush_when_count_threshold_reached() { + let mut s = state(); + for _ in 0..4 { + s.register(&Bytes::from_elems(vec![0u8])); + } + assert!(s.should_flush(&policy())); + } + + #[test] + fn flush_when_size_threshold_reached() { + let mut s = state(); + s.register(&Bytes::from_elems(vec![0u8; 101])); + assert!(s.should_flush(&policy())); + } + + #[test] + fn flush_triggered_by_whichever_limit_comes_first() { + let mut s = state(); + // Only 2 allocations but already over the size limit. + s.register(&Bytes::from_elems(vec![0u8; 60])); + s.register(&Bytes::from_elems(vec![0u8; 60])); + assert!(s.should_flush(&policy())); + } + + #[test] + fn reset_clears_state() { + let mut s = state(); + for _ in 0..4 { + s.register(&Bytes::from_elems(vec![0u8])); + } + assert!(s.should_flush(&policy())); + s.reset(); + assert!(!s.should_flush(&policy())); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/queue.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/queue.rs new file mode 100644 index 00000000..11ba8f40 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/drop_queue/queue.rs @@ -0,0 +1,331 @@ +use alloc::vec::Vec; +use cubecl_common::bytes::Bytes; + +use crate::memory_management::{ + drop_queue::FlushingPolicy, drop_queue::policy::FlushingPolicyState, +}; + +/// A synchronization primitive that blocks until the device has finished +/// processing all commands submitted before the fence was created. +pub trait Fence { + /// Block the current thread until the signals this fence. + fn sync(self); +} + +/// Defers the drop of CPU-side [`Bytes`] allocations until the device has +/// finished reading them. +/// +/// # How it works +/// +/// The device uploads are asynchronous: after you copy bytes into a staging buffer +/// and enqueue an upload command, the CPU memory must remain valid until the +/// device is done. `PendingDropQueue` manages this lifetime with a two-phase +/// approach: +/// +/// 1. **Stage** – call [`push`](Self::push) to hand over bytes that are +/// in-flight. They land in the `staged` list. +/// 2. **Flush** – call [`flush`](Self::flush) to rotate the lists. The +/// previously staged bytes move to `pending`, a new [`Fence`] is created +/// to mark the end of the current upload batch, and any bytes that were +/// *already* pending (i.e. the batch before that) are freed after syncing +/// the previous fence. +/// +/// This double-buffer scheme means CPU memory is held for at most two flush +/// cycles, while avoiding any unnecessary stalls on the hot path. +/// +/// # Flushing policy +/// +/// Call [`should_flush`](Self::should_flush) to check whether enough bytes +/// have accumulated to warrant a flush. You may also flush unconditionally +/// (e.g. at the end of a frame). +pub struct PendingDropQueue { + /// Fence signalling that the device has consumed everything in `pending`. + fence: Option, + /// Bytes from the *previous* flush cycle, kept alive until `event` fires. + pending: Vec, + /// Bytes queued in the *current* cycle, not yet associated with a fence. + staged: Vec, + /// The configuration of the queue. + policy: FlushingPolicy, + /// The current state of the policy. + policy_state: FlushingPolicyState, +} + +impl core::fmt::Debug for PendingDropQueue { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PendingDropQueue") + .field("pending", &self.pending) + .field("staged", &self.staged) + .field("policy", &self.policy) + .field("policy_state", &self.policy_state) + .finish() + } +} + +impl Default for PendingDropQueue { + fn default() -> Self { + Self::new(Default::default()) + } +} + +impl PendingDropQueue { + /// Creates a new `PendingDropQueue`. + pub fn new(policy: FlushingPolicy) -> Self { + Self { + fence: None, + pending: Vec::new(), + staged: Vec::new(), + policy, + policy_state: Default::default(), + } + } + /// Enqueue `bytes` to be dropped once the device has finished reading them. + /// + /// The bytes are added to the current staged batch and will be freed on + /// the flush cycle *after* the next call to [`flush`](Self::flush). + pub fn push(&mut self, bytes: Bytes) { + self.policy_state.register(&bytes); + self.staged.push(bytes); + } + + /// Returns `true` when the staged batch is large enough to justify a + /// flush. + pub fn should_flush(&self) -> bool { + self.policy_state.should_flush(&self.policy) + } + + /// Rotate the double-buffer and free any memory the device is done with. + /// + /// `factory` is called to produce a [`Fence`]. It should submit (or + /// record) a device signal command so that syncing the fence guarantees all + /// preceding device work is complete. + pub fn flush F>(&mut self, factory: Factory) { + // Sync the fence from the previous flush and free the bytes it was + // protecting. + if let Some(event) = self.fence.take() { + event.sync(); + self.pending.clear(); + } + + // Safety net: if pending is somehow still populated (no prior fence), + // stall immediately rather than freeing memory the GPU might still + // be reading. + if !self.pending.is_empty() { + let event = factory(); + event.sync(); + self.pending.clear(); + } + + // The current staged batch becomes the new pending batch. + core::mem::swap(&mut self.pending, &mut self.staged); + + // Record a fence so the *next* flush knows when this batch is safe to + // free. + self.fence = Some(factory()); + self.policy_state.reset(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use core::cell::Cell; + + // --------------------------------------------------------------------------- + // Test helpers + // --------------------------------------------------------------------------- + + #[derive(Clone)] + struct MockFence<'a> { + sync_count: &'a Cell, + } + + impl Fence for MockFence<'_> { + fn sync(self) { + self.sync_count.set(self.sync_count.get() + 1); + } + } + + fn make_queue<'a>( + sync_count: &'a Cell, + ) -> ( + PendingDropQueue>, + impl Fn() -> MockFence<'a> + 'a, + ) { + let queue = PendingDropQueue::new(test_policy()); + let factory = move || MockFence { sync_count }; + (queue, factory) + } + + fn sample_bytes() -> Bytes { + Bytes::from_elems(vec![1u8, 2, 3]) + } + + fn test_policy() -> FlushingPolicy { + FlushingPolicy { + max_bytes_count: 2048, + max_bytes_size: 8, + } + } + + // --------------------------------------------------------------------------- + // push / should_flush + // --------------------------------------------------------------------------- + + #[test] + fn push_at_count_threshold_triggers_flush_hint() { + let sync_count = Cell::new(0u32); + let (mut queue, _factory) = make_queue(&sync_count); + + for _ in 0..test_policy().max_bytes_count { + queue.push(sample_bytes()); + } + + assert!(queue.should_flush()); + } + + #[test] + fn push_large_allocation_triggers_flush_via_size_threshold() { + let sync_count = Cell::new(0u32); + let (mut queue, _factory) = make_queue(&sync_count); + let big = Bytes::from_elems(vec![0u8; test_policy().max_bytes_size as usize + 1]); + + queue.push(big); + + assert!(queue.should_flush()); + } + + // --------------------------------------------------------------------------- + // flush – fence / sync behaviour + // --------------------------------------------------------------------------- + + #[test] + fn first_flush_creates_fence_without_syncing() { + let sync_count = Cell::new(0u32); + let (mut queue, factory) = make_queue(&sync_count); + + queue.push(sample_bytes()); + queue.flush(&factory); + + // The fence is created but must not be synced yet — that happens on + // the next flush. + assert_eq!( + sync_count.get(), + 0, + "fence should not be synced on first flush" + ); + } + + #[test] + fn second_flush_syncs_fence_from_first_flush() { + let sync_count = Cell::new(0u32); + let (mut queue, factory) = make_queue(&sync_count); + + queue.push(sample_bytes()); + queue.flush(&factory); // flush 1 – creates fence A + + queue.push(sample_bytes()); + queue.flush(&factory); // flush 2 – syncs fence A, creates fence B + + assert_eq!(sync_count.get(), 1, "exactly one sync after two flushes"); + } + + #[test] + fn each_subsequent_flush_syncs_the_previous_fence() { + let sync_count = Cell::new(0u32); + let (mut queue, factory) = make_queue(&sync_count); + + for _ in 0..10 { + queue.push(sample_bytes()); + queue.flush(&factory); + } + + // Each flush except the first syncs the fence from the previous one. + assert_eq!(sync_count.get(), 9); + } + + // --------------------------------------------------------------------------- + // flush – buffer rotation + // --------------------------------------------------------------------------- + + #[test] + fn staged_is_empty_after_flush() { + let sync_count = Cell::new(0u32); + let (mut queue, factory) = make_queue(&sync_count); + + for _ in 0..5 { + queue.push(sample_bytes()); + } + queue.flush(&factory); + + assert!(queue.staged.is_empty()); + } + + #[test] + fn pending_holds_previously_staged_bytes_after_flush() { + let sync_count = Cell::new(0u32); + let (mut queue, factory) = make_queue(&sync_count); + + for _ in 0..5 { + queue.push(sample_bytes()); + } + queue.flush(&factory); + + assert_eq!(queue.pending.len(), 5); + } + + #[test] + fn pending_is_replaced_on_second_flush() { + let sync_count = Cell::new(0u32); + let (mut queue, factory) = make_queue(&sync_count); + + for _ in 0..5 { + queue.push(sample_bytes()); + } + queue.flush(&factory); // pending = 5 items + + queue.push(sample_bytes()); + queue.flush(&factory); // syncs fence → pending cleared, rotated + + // Only the one item staged between the two flushes should be pending. + assert_eq!(queue.pending.len(), 1); + } + + // --------------------------------------------------------------------------- + // flush – policy state reset + // --------------------------------------------------------------------------- + + #[test] + fn should_flush_resets_after_flush() { + let sync_count = Cell::new(0u32); + let (mut queue, factory) = make_queue(&sync_count); + + for _ in 0..test_policy().max_bytes_count { + queue.push(sample_bytes()); + } + assert!(queue.should_flush()); + + queue.flush(&factory); + + assert!( + !queue.should_flush(), + "policy state should be reset after flush" + ); + } + + // --------------------------------------------------------------------------- + // Edge cases + // --------------------------------------------------------------------------- + + #[test] + fn flush_on_empty_queue_is_safe() { + let sync_count = Cell::new(0u32); + let (mut queue, factory) = make_queue(&sync_count); + + // Should not panic regardless of how many times it is called. + queue.flush(&factory); + queue.flush(&factory); + queue.flush(&factory); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_manage.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_manage.rs new file mode 100644 index 00000000..cc4e9e5e --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_manage.rs @@ -0,0 +1,1101 @@ +use super::{ + MemoryConfiguration, MemoryPoolOptions, MemoryUsage, PoolType, + memory_pool::{ExclusiveMemoryPool, MemoryPool, PersistentPool, SlicedPool}, +}; +use crate::{ + config::{ + CubeClRuntimeConfig, RuntimeConfig, + memory::{MemoryLogLevel, PersistentMemory}, + }, + logging::ServerLogger, + memory_management::{BytesFormat, memory_pool::Slice}, + server::IoError, + storage::{ComputeStorage, StorageHandle}, +}; + +use alloc::format; +use alloc::string::{String, ToString}; +#[cfg(not(exclusive_memory_only))] +use alloc::vec; +use alloc::vec::Vec; +use cubecl_common::{backtrace::BackTrace, stub::Arc}; +use cubecl_ir::MemoryDeviceProperties; + +pub use super::memory_pool::{ManagedMemoryBinding, handle::*}; + +// These are 288 bytes vs 64 bytes. Adding boxing isn't really worth +// saving the 200 bytes. +#[allow(clippy::large_enum_variant)] +enum DynamicPool { + Sliced(SlicedPool), + Exclusive(ExclusiveMemoryPool), +} + +impl MemoryPool for DynamicPool { + fn accept(&self, size: u64) -> bool { + match self { + DynamicPool::Sliced(pool) => pool.accept(size), + DynamicPool::Exclusive(pool) => pool.accept(size), + } + } + + fn find(&self, binding: &ManagedMemoryBinding) -> Result<&Slice, IoError> { + match self { + DynamicPool::Sliced(m) => m.find(binding), + DynamicPool::Exclusive(m) => m.find(binding), + } + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + fn try_reserve(&mut self, size: u64) -> Option { + match self { + DynamicPool::Sliced(m) => m.try_reserve(size), + DynamicPool::Exclusive(m) => m.try_reserve(size), + } + } + + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, storage)) + )] + fn alloc( + &mut self, + storage: &mut Storage, + size: u64, + ) -> Result { + match self { + DynamicPool::Sliced(m) => m.alloc(storage, size), + DynamicPool::Exclusive(m) => m.alloc(storage, size), + } + } + + fn get_memory_usage(&self) -> MemoryUsage { + match self { + DynamicPool::Sliced(m) => m.get_memory_usage(), + DynamicPool::Exclusive(m) => m.get_memory_usage(), + } + } + + fn cleanup( + &mut self, + storage: &mut Storage, + alloc_nr: u64, + explicit: bool, + ) { + match self { + DynamicPool::Sliced(m) => m.cleanup(storage, alloc_nr, explicit), + DynamicPool::Exclusive(m) => m.cleanup(storage, alloc_nr, explicit), + }; + storage.flush(); + } + + fn bind( + &mut self, + reserved: ManagedMemoryHandle, + assigned: ManagedMemoryHandle, + cursor: u64, + ) -> Result<(), IoError> { + match self { + DynamicPool::Sliced(m) => m.bind(reserved, assigned, cursor), + DynamicPool::Exclusive(m) => m.bind(reserved, assigned, cursor), + } + } +} + +#[derive(Default, Clone, Copy, Debug)] +/// The mode of allocation used. +pub enum MemoryAllocationMode { + /// Use the automatic memory management strategy for allocation. + #[default] + Auto, + /// Use a persistent memory management strategy, meaning that all allocations are for data that is + /// likely never going to be freed. + Persistent, +} + +/// Reserves and keeps track of chunks of memory in the storage, and slices upon these chunks. +pub struct MemoryManagement { + name: String, + persistent: PersistentPool, + pools: Vec, + storage: Storage, + alloc_reserve_count: u64, + mode: MemoryAllocationMode, + config: PersistentMemory, + logger: Arc, +} + +fn generate_bucket_sizes( + start_size: u64, + end_size: u64, + max_buckets: usize, + alignment: u64, +) -> Vec { + let mut buckets = Vec::with_capacity(max_buckets); + let log_min = (start_size as f64).ln(); + let log_max = (end_size as f64).ln(); + let log_range = log_max - log_min; + + // Pure exponential performed best, but let's try slightly denser in lower-mid range + for i in 0..max_buckets { + let p = i as f64 / (max_buckets - 1) as f64; + // Slight bias toward lower-mid range with less aggressive curve than sigmoid + let log_size = log_min + log_range * p; + let size = log_size.exp() as u64; + let aligned_size = size.next_multiple_of(alignment); + buckets.push(aligned_size); + } + + buckets.dedup(); + buckets +} + +const DEALLOC_SCALE_MB: u64 = 1024 * 1024 * 1024; +const BASE_DEALLOC_PERIOD: u64 = 5000; + +/// The options for creating a new [`MemoryManagement`] instance. +#[derive(Debug)] +pub struct MemoryManagementOptions { + /// The name of the memory management. + name: String, + /// The [`MemoryAllocationOption`] used by this instance. + memory: MemoryAllocationOption, +} + +impl MemoryManagementOptions { + /// Creates a new [`MemoryManagementOptions`]. + pub fn new>(name: S) -> Self { + Self { + name: name.into(), + memory: MemoryAllocationOption::FromConfig, + } + } + + /// Forces the [`MemoryAllocationMode`] during execution to always be the provided one. + pub fn mode(mut self, mode: MemoryAllocationMode) -> Self { + self.memory = MemoryAllocationOption::Provided(mode); + self + } +} + +#[derive(Default, Debug)] +/// Determines which [`MemoryAllocationMode`] is used during allocations. +enum MemoryAllocationOption { + #[default] + /// Uses the [`GlobalConfig`] to determine the mode of allocation. + FromConfig, + /// Use the provided [`MemoryAllocationMode`]. + Provided(MemoryAllocationMode), +} + +impl MemoryManagement { + /// Creates the options from device limits. + pub fn from_configuration( + storage: Storage, + properties: &MemoryDeviceProperties, + config: MemoryConfiguration, + logger: Arc, + options: MemoryManagementOptions, + ) -> Self { + let pool_options = match config { + #[cfg(not(exclusive_memory_only))] + MemoryConfiguration::SubSlices => { + // Round chunk size to be aligned. + let memory_alignment = properties.alignment; + let max_page = properties.max_page_size; + let mut pools = Vec::new(); + + const MB: u64 = 1024 * 1024; + + // Add in a pool for allocations that are smaller than the min alignment, + // as they can't use offsets at all (on wgpu at least). + pools.push(MemoryPoolOptions { + pool_type: PoolType::ExclusivePages { max_alloc_size: 0 }, + dealloc_period: None, + }); + + let mut current = max_page; + let mut max_sizes = vec![]; + let mut page_sizes = vec![]; + let mut base = pools.len() as u32; + + while current >= 32 * MB { + current /= 4; + + // Make sure every pool has an aligned size. + current = current.next_multiple_of(memory_alignment); + + max_sizes.push(current / 2u64.pow(base)); + page_sizes.push(current); + base += 1; + } + + max_sizes.reverse(); + page_sizes.reverse(); + + for i in 0..max_sizes.len() { + let max = max_sizes[i]; + let page_size = page_sizes[i]; + + pools.push(MemoryPoolOptions { + // Creating max slices lower than the chunk size reduces fragmentation. + pool_type: PoolType::SlicedPages { + page_size, + max_slice_size: max, + }, + dealloc_period: None, + }); + } + + // Add pools from big to small. + pools.push(MemoryPoolOptions { + pool_type: PoolType::SlicedPages { + page_size: max_page / memory_alignment * memory_alignment, + max_slice_size: max_page / memory_alignment * memory_alignment, + }, + dealloc_period: None, + }); + pools + } + MemoryConfiguration::ExclusivePages => { + // Add all bin sizes. Nb: because of alignment some buckets + // end up as the same size, so only want unique ones, + // but also keep the order, so a BTree will do. + const MIN_BUCKET_SIZE: u64 = 1024 * 32; + const NUM_POOLS: usize = 24; + + let sizes = generate_bucket_sizes( + MIN_BUCKET_SIZE, + properties.max_page_size, + NUM_POOLS, + properties.alignment, + ); + + sizes + .iter() + .map(|&size| { + let dealloc_period = (BASE_DEALLOC_PERIOD as f64 + * (1.0 + size as f64 / (DEALLOC_SCALE_MB as f64)).round()) + as u64; + + MemoryPoolOptions { + pool_type: PoolType::ExclusivePages { + max_alloc_size: size, + }, + dealloc_period: Some(dealloc_period), + } + }) + .collect() + } + MemoryConfiguration::Custom { pool_options } => pool_options, + }; + + logger.log_memory( + |level| !matches!(level, MemoryLogLevel::Disabled), + || { + let mut msg = String::new(); + for pool in pool_options.iter() { + msg += &format!("[{}] Using memory pool: \n {pool:?}\n", options.name); + } + msg + }, + ); + + let pools: Vec<_> = pool_options + .iter() + .enumerate() + .map(|(pool_pos, options)| { + let pool_pos = pool_pos as u8; + + match options.pool_type { + PoolType::SlicedPages { + page_size, + max_slice_size, + } => DynamicPool::Sliced(SlicedPool::new( + page_size, + max_slice_size, + properties.alignment, + pool_pos, + )), + PoolType::ExclusivePages { max_alloc_size } => { + DynamicPool::Exclusive(ExclusiveMemoryPool::new( + max_alloc_size, + properties.alignment, + options.dealloc_period.unwrap_or(u64::MAX), + pool_pos, + )) + } + } + }) + .collect(); + + let config = CubeClRuntimeConfig::get().memory.persistent_memory.clone(); + + let mode = match options.memory { + MemoryAllocationOption::Provided(mode) => mode, + MemoryAllocationOption::FromConfig => match config { + PersistentMemory::Enabled => MemoryAllocationMode::Auto, + PersistentMemory::Disabled => MemoryAllocationMode::Auto, + PersistentMemory::Enforced => MemoryAllocationMode::Persistent, + }, + }; + + Self { + name: options.name, + persistent: PersistentPool::new( + properties.max_page_size, + properties.alignment, + pools.len() as u8, + ), + pools, + storage, + alloc_reserve_count: 0, + mode, + config, + logger, + } + } + + /// Change the mode of allocation. + pub fn mode(&mut self, mode: MemoryAllocationMode) { + // We override the mode based on the cubecl config. + let mode = match self.config { + PersistentMemory::Enabled => mode, + PersistentMemory::Disabled | PersistentMemory::Enforced => return, + }; + + self.logger.log_memory( + |level| !matches!(level, MemoryLogLevel::Disabled), + || { + format!( + "[{}] Setting memory allocation mode: from {:?} => {mode:?}", + self.name, self.mode + ) + }, + ); + self.mode = mode; + } + + /// Cleanup allocations in pools that are deemed unnecessary. + pub fn cleanup(&mut self, explicit: bool) { + self.logger.log_memory( + |level| !matches!(level, MemoryLogLevel::Disabled) && explicit, + || "Manual memory cleanup ...".to_string(), + ); + + self.persistent + .cleanup(&mut self.storage, self.alloc_reserve_count, explicit); + + for pool in self.pools.iter_mut() { + pool.cleanup(&mut self.storage, self.alloc_reserve_count, explicit); + } + } + + /// Returns the storage from the specified binding + pub fn get_cursor(&self, binding: ManagedMemoryBinding) -> Result { + let slice = self.find(binding)?; + Ok(slice.cursor) + } + + /// Returns the storage from the specified binding + fn find(&self, binding: ManagedMemoryBinding) -> Result<&Slice, IoError> { + let id = binding.descriptor(); + + if id.location().pool >= self.pools.len() as u8 { + return self.persistent.find(&binding); + } + + let pool = + self.pools + .get(id.location().pool as usize) + .ok_or_else(|| IoError::NotFound { + backtrace: BackTrace::capture(), + reason: format!("Pool {} doesn't exist", id.location().pool).into(), + })?; + + let slice = pool.find(&binding)?; + + assert_eq!(slice.handle.descriptor(), binding.descriptor()); + + Ok(slice) + } + + /// Returns the storage from the specified binding + pub fn get_storage(&mut self, binding: ManagedMemoryBinding) -> Result { + let slice = self.find(binding)?; + Ok(slice.storage.clone()) + } + + /// Returns the resource from the storage at the specified handle + pub fn get_resource( + &mut self, + binding: ManagedMemoryBinding, + offset_start: Option, + offset_end: Option, + ) -> Result { + let handle = self.get_storage(binding)?; + + let handle = match offset_start { + Some(offset) => handle.offset_start(offset), + None => handle, + }; + let handle = match offset_end { + Some(offset) => handle.offset_end(offset), + None => handle, + }; + Ok(self.storage().get(&handle)) + } + + /// Finds a spot in memory for a resource with the given size in bytes, and returns a handle to it + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + pub fn reserve(&mut self, size: u64) -> Result { + // If this happens every nanosecond, counts overflows after 585 years, so not worth thinking too + // hard about overflow here. + self.alloc_reserve_count += 1; + + if let Some(val) = self.persistent.try_reserve(size) { + self.logger.log_memory( + |level| matches!(level, MemoryLogLevel::Full), + || { + format!( + "[{}] Reserved memory {size} using persistent memory", + self.name + ) + }, + ); + return Ok(val); + } + + if matches!(self.mode, MemoryAllocationMode::Persistent) || self.persistent.has_size(size) { + let allocated = self.persistent.alloc(&mut self.storage, size); + + self.logger.log_memory( + |level| !matches!(level, MemoryLogLevel::Disabled), + || { + format!( + "[{}] Allocated a new memory page using persistent memory, \n{}", + self.name, self, + ) + }, + ); + return allocated; + } + + self.logger.log_memory( + |level| matches!(level, MemoryLogLevel::Full), + || { + format!( + "[{}] Reserved memory {} using dynamic pool", + self.name, + BytesFormat::new(size) + ) + }, + ); + + // Find first pool that fits this allocation + let pool = self + .pools + .iter_mut() + .find(|p| p.accept(size)) + .ok_or(IoError::BufferTooBig { + size, + backtrace: BackTrace::capture(), + })?; + + if let Some(slice) = pool.try_reserve(size) { + return Ok(slice); + } + + let allocated = pool.alloc(&mut self.storage, size); + + self.logger.log_memory( + |level| matches!(level, MemoryLogLevel::Full), + || { + format!( + "[{}], Allocated a new memory page, current usage: \n{}", + self.name, self + ) + }, + ); + + allocated + } + + /// Fetch the storage used by the memory manager. + /// + /// # Notes + /// + /// The storage should probably not be used for allocations since the handles won't be + /// compatible with the ones provided by the current trait. Prefer using the + /// [alloc](ComputeStorage::alloc) and [dealloc](ComputeStorage::dealloc) functions. + /// + /// This is useful if you need to time the deallocations based on async computation, or to + /// change the mode of storage for different reasons. + pub fn storage(&mut self) -> &mut Storage { + &mut self.storage + } + + /// Get the current memory usage. + pub fn memory_usage(&self) -> MemoryUsage { + let memory_usage = self.pools.iter().map(|x| x.get_memory_usage()).fold( + MemoryUsage { + number_allocs: 0, + bytes_in_use: 0, + bytes_padding: 0, + bytes_reserved: 0, + }, + |m1, m2| m1.combine(m2), + ); + memory_usage.combine(self.persistent.get_memory_usage()) + } + + /// Print out a report of the current memory usage. + pub fn print_memory_usage(&self) { + #[cfg(feature = "std")] + log::info!("{}", self.memory_usage()); + } + + /// Binds the given [handle](HandleId) to a [`MemorySlot`]. + pub fn bind( + &mut self, + reserved: ManagedMemoryHandle, + assigned: ManagedMemoryHandle, + cursor: u64, + ) -> Result<(), IoError> { + let descriptor = reserved.descriptor(); + + if descriptor.location().init == 0 { + return Err(IoError::NotFound { + backtrace: BackTrace::capture(), + reason: "Reserved memory isn't initialized".into(), + }); + } + + let pool_index = descriptor.location().pool as usize; + if pool_index >= self.pools.len() { + return self.persistent.bind(reserved, assigned, cursor); + } + + self.pools + .get_mut(pool_index) + .map(|p| p.bind(reserved, assigned, cursor)) + .ok_or_else(|| IoError::NotFound { + backtrace: BackTrace::capture(), + reason: format!("Memory pool {} doesn't exist", pool_index).into(), + })? + } +} + +impl core::fmt::Display for MemoryManagement { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("\n# MemoryManagement\n\n")?; + f.write_fmt(format_args!(" - name: {:?}\n", self.name))?; + f.write_fmt(format_args!("\n## Persistent\n\n{}", self.persistent))?; + f.write_str("\n## Dynamic\n\n")?; + + for pool in self.pools.iter() { + match pool { + DynamicPool::Sliced(pool) => f.write_fmt(format_args!("{pool}\n"))?, + DynamicPool::Exclusive(pool) => f.write_fmt(format_args!("{pool}\n"))?, + } + } + let memory_usage = self.memory_usage(); + f.write_fmt(format_args!("\n## Summary\n\n{memory_usage}"))?; + + Ok(()) + } +} + +impl core::fmt::Debug for MemoryManagement { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str( + alloc::format!( + "DynamicMemoryManagement {:?}", + core::any::type_name::(), + ) + .as_str(), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{memory_management::MemoryManagement, storage::BytesStorage}; + use alloc::vec; + + const DUMMY_MEM_PROPS: MemoryDeviceProperties = MemoryDeviceProperties { + max_page_size: 128 * 1024 * 1024, + alignment: 32, + }; + + fn options() -> MemoryManagementOptions { + MemoryManagementOptions { + name: "test".into(), + memory: MemoryAllocationOption::FromConfig, + } + } + + // Test pools with slices. + #[test_log::test] + #[cfg(not(exclusive_memory_only))] + fn test_handle_mutability() { + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &DUMMY_MEM_PROPS, + MemoryConfiguration::SubSlices, + Arc::new(ServerLogger::default()), + options(), + ); + let handle = memory_management.reserve(10).unwrap(); + let other_ref = handle.clone(); + assert!(!handle.can_mut(), "Handle can't be mut when multiple ref."); + drop(other_ref); + assert!(handle.can_mut(), "Handle should be mut when only one ref."); + } + + // Test pools with slices. + #[test_log::test] + #[cfg(not(exclusive_memory_only))] + fn test_memory_usage() { + let max_page_size = 512; + + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &DUMMY_MEM_PROPS, + MemoryConfiguration::Custom { + pool_options: vec![MemoryPoolOptions { + pool_type: PoolType::ExclusivePages { + max_alloc_size: max_page_size, + }, + dealloc_period: None, + }], + }, + Arc::new(ServerLogger::default()), + options(), + ); + let handle = memory_management.reserve(100); + let usage = memory_management.memory_usage(); + + assert_eq!(usage.bytes_in_use, 100); + assert!(usage.bytes_reserved >= 100 && usage.bytes_reserved <= max_page_size); + + // Drop and re-alloc. + drop(handle); + let _handle = memory_management.reserve(100); + let usage_new = memory_management.memory_usage(); + assert_eq!(usage, usage_new); + } + + #[test_log::test] + fn alloc_two_chunks_on_one_page() { + let page_size = 2048; + + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &DUMMY_MEM_PROPS, + MemoryConfiguration::Custom { + pool_options: vec![MemoryPoolOptions { + pool_type: PoolType::SlicedPages { + page_size, + max_slice_size: page_size, + }, + dealloc_period: None, + }], + }, + Arc::new(ServerLogger::default()), + options(), + ); + + let alloc_size = 512; + let _handle = memory_management.reserve(alloc_size); + let _new_handle = memory_management.reserve(alloc_size); + + let usage = memory_management.memory_usage(); + assert_eq!(usage.number_allocs, 2); + assert_eq!(usage.bytes_in_use, alloc_size * 2); + assert_eq!(usage.bytes_reserved, page_size); + } + + #[test_log::test] + fn alloc_reuses_storage() { + // If no storage is re-used, this will allocate two pages. + let page_size = 512; + + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &DUMMY_MEM_PROPS, + MemoryConfiguration::Custom { + pool_options: vec![MemoryPoolOptions { + pool_type: PoolType::SlicedPages { + page_size, + max_slice_size: page_size, + }, + dealloc_period: None, + }], + }, + Arc::new(ServerLogger::default()), + options(), + ); + + let alloc_size = 512; + let _handle = memory_management.reserve(alloc_size); + drop(_handle); + let _new_handle = memory_management.reserve(alloc_size); + + let usage = memory_management.memory_usage(); + assert_eq!(usage.number_allocs, 1); + assert_eq!(usage.bytes_in_use, alloc_size); + assert_eq!(usage.bytes_reserved, page_size); + } + + #[test_log::test] + fn alloc_allocs_new_storage() { + let page_size = 1024; + + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &DUMMY_MEM_PROPS, + MemoryConfiguration::Custom { + pool_options: vec![MemoryPoolOptions { + pool_type: PoolType::SlicedPages { + page_size, + max_slice_size: page_size, + }, + dealloc_period: None, + }], + }, + Arc::new(ServerLogger::default()), + options(), + ); + + let alloc_size = 768; + let _handle = memory_management.reserve(alloc_size); + let _new_handle = memory_management.reserve(alloc_size); + + let usage = memory_management.memory_usage(); + assert_eq!(usage.number_allocs, 2); + assert_eq!(usage.bytes_in_use, alloc_size * 2); + assert_eq!(usage.bytes_reserved, page_size * 2); + } + + #[test_log::test] + fn alloc_respects_alignment_size() { + let page_size = 500; + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &MemoryDeviceProperties { + max_page_size: page_size, + alignment: 50, + }, + MemoryConfiguration::Custom { + pool_options: vec![MemoryPoolOptions { + pool_type: PoolType::SlicedPages { + page_size, + max_slice_size: page_size, + }, + dealloc_period: None, + }], + }, + Arc::new(ServerLogger::default()), + options(), + ); + let alloc_size = 40; + let _handle = memory_management.reserve(alloc_size); + let _new_handle = memory_management.reserve(alloc_size); + let usage = memory_management.memory_usage(); + // Each slice should be aligned to 50 bytes, so 20 padding bytes. + assert_eq!(usage.bytes_padding, 10 * 2); + } + + #[test_log::test] + fn allocs_on_correct_page() { + let sizes = [100, 200, 300, 400]; + + let pools = sizes + .iter() + .map(|size| MemoryPoolOptions { + pool_type: PoolType::SlicedPages { + page_size: *size, + max_slice_size: *size, + }, + dealloc_period: None, + }) + .collect(); + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &MemoryDeviceProperties { + max_page_size: 128 * 1024 * 1024, + alignment: 10, + }, + MemoryConfiguration::Custom { + pool_options: pools, + }, + Arc::new(ServerLogger::default()), + options(), + ); + // Allocate one thing on each page. + let alloc_sizes = [50, 150, 250, 350]; + let _handles = alloc_sizes.map(|s| memory_management.reserve(s)); + + let usage = memory_management.memory_usage(); + + // Total memory should be size of all pages, and no more. + assert_eq!(usage.bytes_in_use, alloc_sizes.iter().sum::()); + assert!(usage.bytes_reserved >= sizes.iter().sum::()); + } + + #[test_log::test] + #[cfg(not(exclusive_memory_only))] + fn allocate_deallocate_reallocate() { + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &MemoryDeviceProperties { + max_page_size: 128 * 1024 * 1024, + alignment: 32, + }, + MemoryConfiguration::SubSlices, + Arc::new(ServerLogger::default()), + options(), + ); + // Allocate a bunch + let handles: Vec<_> = (0..5) + .map(|i| memory_management.reserve(1000 * (i + 1))) + .collect(); + let usage_before = memory_management.memory_usage(); + // Deallocate + drop(handles); + // Reallocate + let _new_handles: Vec<_> = (0..5) + .map(|i| memory_management.reserve(1000 * (i + 1))) + .collect(); + let usage_after = memory_management.memory_usage(); + assert_eq!(usage_before.number_allocs, usage_after.number_allocs); + assert_eq!(usage_before.bytes_in_use, usage_after.bytes_in_use); + // Usage after can actually be _less_ because of defragging. + assert!(usage_before.bytes_reserved >= usage_after.bytes_reserved); + } + + #[test_log::test] + #[cfg(not(exclusive_memory_only))] + fn test_fragmentation_resistance() { + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &MemoryDeviceProperties { + max_page_size: 128 * 1024 * 1024, + alignment: 32, + }, + MemoryConfiguration::SubSlices, + Arc::new(ServerLogger::default()), + options(), + ); + // Allocate a mix of small and large chunks + let sizes = [50, 1000, 100, 5000, 200, 10000, 300]; + let handles: Vec<_> = sizes + .iter() + .map(|&size| memory_management.reserve(size).unwrap()) + .collect(); + let usage_before = memory_management.memory_usage(); + // Deallocate every other allocation + for i in (0..handles.len()).step_by(2) { + drop(handles[i].clone()); + } + // Reallocate similar sizes + for &size in &sizes[0..sizes.len() / 2] { + memory_management.reserve(size).unwrap(); + } + let usage_after = memory_management.memory_usage(); + // Check that we haven't increased our memory usage significantly + assert!(usage_after.bytes_reserved <= (usage_before.bytes_reserved as f64 * 1.1) as u64); + } + + // Test pools without slices. More or less same as tests above. + #[test_log::test] + fn noslice_test_handle_mutability() { + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &(MemoryDeviceProperties { + max_page_size: 128 * 1024 * 1024, + alignment: 32, + }), + MemoryConfiguration::ExclusivePages, + Arc::new(ServerLogger::default()), + options(), + ); + let handle = memory_management.reserve(10).unwrap(); + let other_ref = handle.clone(); + assert!(!handle.can_mut(), "Handle can't be mut when multiple ref."); + drop(other_ref); + assert!(handle.can_mut(), "Handle should be mut when only one ref."); + } + + #[test_log::test] + fn noslice_alloc_two_chunk() { + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &DUMMY_MEM_PROPS, + MemoryConfiguration::Custom { + pool_options: vec![MemoryPoolOptions { + pool_type: PoolType::ExclusivePages { + max_alloc_size: 1024, + }, + dealloc_period: None, + }], + }, + Arc::new(ServerLogger::default()), + options(), + ); + + let alloc_size = 512; + let _handle = memory_management.reserve(alloc_size); + let _new_handle = memory_management.reserve(alloc_size); + + let usage = memory_management.memory_usage(); + assert_eq!(usage.number_allocs, 2); + assert_eq!(usage.bytes_in_use, alloc_size * 2); + assert!(usage.bytes_reserved >= alloc_size * 2); + } + + #[test_log::test] + fn noslice_alloc_reuses_storage() { + // If no storage is re-used, this will allocate two pages. + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &DUMMY_MEM_PROPS, + MemoryConfiguration::Custom { + pool_options: vec![MemoryPoolOptions { + pool_type: PoolType::ExclusivePages { + max_alloc_size: 1024, + }, + dealloc_period: None, + }], + }, + Arc::new(ServerLogger::default()), + options(), + ); + + let alloc_size = 512; + let _handle = memory_management.reserve(alloc_size); + drop(_handle); + let _new_handle = memory_management.reserve(alloc_size); + + let usage = memory_management.memory_usage(); + assert_eq!(usage.number_allocs, 1); + assert_eq!(usage.bytes_in_use, alloc_size); + assert!(usage.bytes_reserved >= alloc_size); + } + + #[test_log::test] + fn noslice_alloc_allocs_new_storage() { + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &DUMMY_MEM_PROPS, + MemoryConfiguration::Custom { + pool_options: vec![MemoryPoolOptions { + pool_type: PoolType::ExclusivePages { + max_alloc_size: 1024, + }, + dealloc_period: None, + }], + }, + Arc::new(ServerLogger::default()), + options(), + ); + + let alloc_size = 768; + let _handle = memory_management.reserve(alloc_size); + let _new_handle = memory_management.reserve(alloc_size); + let usage = memory_management.memory_usage(); + assert_eq!(usage.number_allocs, 2); + assert_eq!(usage.bytes_in_use, alloc_size * 2); + assert!(usage.bytes_reserved >= alloc_size * 2); + } + + #[test_log::test] + fn noslice_alloc_respects_alignment_size() { + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &MemoryDeviceProperties { + max_page_size: DUMMY_MEM_PROPS.max_page_size, + alignment: 50, + }, + MemoryConfiguration::Custom { + pool_options: vec![MemoryPoolOptions { + pool_type: PoolType::ExclusivePages { + max_alloc_size: 50 * 20, + }, + dealloc_period: None, + }], + }, + Arc::new(ServerLogger::default()), + options(), + ); + let alloc_size = 40; + let _handle = memory_management.reserve(alloc_size); + let _new_handle = memory_management.reserve(alloc_size); + let usage = memory_management.memory_usage(); + // Each slice should be aligned to 60 bytes, so 20 padding bytes. + assert_eq!(usage.bytes_padding, 10 * 2); + } + + #[test_log::test] + fn noslice_allocs_on_correct_page() { + let pools = [100, 200, 300, 400] + .iter() + .map(|&size| MemoryPoolOptions { + pool_type: PoolType::SlicedPages { + page_size: size, + max_slice_size: size, + }, + dealloc_period: None, + }) + .collect(); + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &MemoryDeviceProperties { + max_page_size: DUMMY_MEM_PROPS.max_page_size, + alignment: 10, + }, + MemoryConfiguration::Custom { + pool_options: pools, + }, + Arc::new(ServerLogger::default()), + options(), + ); + // Allocate one thing on each page. + let alloc_sizes = [50, 150, 250, 350]; + let _handles = alloc_sizes.map(|s| memory_management.reserve(s)); + let usage = memory_management.memory_usage(); + // Total memory should be size of all pages, and no more. + assert_eq!(usage.bytes_in_use, alloc_sizes.iter().sum::()); + } + + #[test_log::test] + fn noslice_allocate_deallocate_reallocate() { + let mut memory_management = MemoryManagement::from_configuration( + BytesStorage::default(), + &MemoryDeviceProperties { + max_page_size: 128 * 1024 * 1024, + alignment: 32, + }, + MemoryConfiguration::ExclusivePages, + Arc::new(ServerLogger::default()), + options(), + ); + // Allocate a bunch + let handles: Vec<_> = (0..5) + .map(|i| memory_management.reserve(1000 * (i + 1))) + .collect(); + let usage_before = memory_management.memory_usage(); + // Deallocate + drop(handles); + // Reallocate + let _new_handles: Vec<_> = (0..5) + .map(|i| memory_management.reserve(1000 * (i + 1))) + .collect(); + let usage_after = memory_management.memory_usage(); + assert_eq!(usage_before.number_allocs, usage_after.number_allocs); + assert_eq!(usage_before.bytes_in_use, usage_after.bytes_in_use); + assert_eq!(usage_before.bytes_reserved, usage_after.bytes_reserved); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/base.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/base.rs new file mode 100644 index 00000000..450e9065 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/base.rs @@ -0,0 +1,117 @@ +use super::{ManagedMemoryBinding, ManagedMemoryDescriptor, ManagedMemoryHandle}; +use crate::{ + memory_management::MemoryUsage, + server::IoError, + storage::{ComputeStorage, StorageHandle}, +}; + +/// Declares how memory is allocated in a reusable pool. +pub trait MemoryPool { + /// Whether the memory pool accepts the given size. + fn accept(&self, size: u64) -> bool; + + /// Binds an uninitialized handle to a previously reserved memory slice. + /// + /// # Arguments + /// + /// * `reserved` - An existing, initialized handle representing the underlying memory allocation. + /// * `assigned` - A new handle that will be initialized to point into the `reserved` memory region. + /// * `cursor` - A sequence point or timestamp determining when this binding becomes valid for access. + /// + /// # Errors + /// + /// Returns [`IoError`] if the reservation is invalid or if the cursor position + /// is outside the bounds of the memory pool. + fn bind( + &mut self, + reserved: ManagedMemoryHandle, + assigned: ManagedMemoryHandle, + cursor: u64, + ) -> Result<(), IoError>; + + /// Retrieves the slice for the binding. + fn find(&self, binding: &ManagedMemoryBinding) -> Result<&Slice, IoError>; + + /// Try to reserve a memory slice of the given size. + /// + /// # Notes + /// + /// It is not guaranteed the `try_reserve` function will reapply the accept function. + /// Therefore it is a good idea to call [`MemoryUsage::accept()`] before using `try_reserve`. + /// + /// # Returns + /// + /// A [slice handle](StorageHandle) if the current memory pool has enough memory, otherwise it + /// will returns [None]. You can then call [`MemoryPool::alloc()`] to increase the amount of + /// memory the pool has. + fn try_reserve(&mut self, size: u64) -> Option; + + /// Increases the amount of memory the pool has and returns a [slice handle](StorageHandle) + /// corresponding to the requested size. + /// + /// # Notes + /// + /// The function uses a [`ComputeStorage`] to perform the allocation. It might return an error + /// if the allocation fails or if the requested size is bigger than the memory pool is + /// configured to handle. + fn alloc( + &mut self, + storage: &mut Storage, + size: u64, + ) -> Result; + + /// Computes the [`MemoryUsage`] for this pool. + fn get_memory_usage(&self) -> MemoryUsage; + + /// Cleanup the memory pool, maybe freeing some memory using the [`ComputeStorage`]. + fn cleanup( + &mut self, + storage: &mut Storage, + alloc_nr: u64, + explicit: bool, + ); +} + +#[derive(Debug)] +/// Slice of data with its associated storage. +pub(crate) struct Slice { + pub storage: StorageHandle, + pub handle: ManagedMemoryHandle, + pub padding: u64, + pub cursor: u64, +} + +impl Slice { + pub fn new(storage: StorageHandle, padding: u64) -> Self { + Self { + storage, + handle: ManagedMemoryHandle::new(), + padding, + cursor: 0, + } + } + /// If the slice is free to be reused. + pub(crate) fn is_free(&self) -> bool { + self.handle.is_free() + } + + /// The total size of the slice including padding. + pub(crate) fn effective_size(&self) -> u64 { + self.storage.size() + self.padding + } + + /// The description of the slice. + pub(crate) fn descriptor(&self) -> &ManagedMemoryDescriptor { + self.handle.descriptor() + } +} + +/// Calculates the padding required to store the given size in a buffer given the memory alignment. +pub(crate) fn calculate_padding(size: u64, memory_alignment: u64) -> u64 { + let remainder = size % memory_alignment; + if remainder != 0 { + memory_alignment - remainder + } else { + 0 + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/exclusive_pool.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/exclusive_pool.rs new file mode 100644 index 00000000..44b0ee0b --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/exclusive_pool.rs @@ -0,0 +1,259 @@ +use crate::{ + memory_management::{BytesFormat, MemoryLocation, MemoryUsage}, + server::IoError, + storage::{ComputeStorage, StorageUtilization}, +}; + +use alloc::vec::Vec; +use cubecl_common::backtrace::BackTrace; + +use super::{ManagedMemoryBinding, ManagedMemoryHandle, MemoryPool, Slice, calculate_padding}; + +/// A memory pool that allocates buffers in a range of sizes and reuses them to minimize allocations. +/// +/// - Only one slice is supported per page, due to the limitations in WGPU where each buffer should only bound with +/// either read only or `read_write` slices but not a mix of both. +/// - The pool uses a ring buffer to efficiently manage and reuse pages. +pub struct ExclusiveMemoryPool { + pages: Vec, + pages_tmp: Vec, + alignment: u64, + dealloc_period: u64, + last_dealloc_check: u64, + max_alloc_size: u64, + cur_avg_size: f64, + location_base: MemoryLocation, +} + +impl core::fmt::Display for ExclusiveMemoryPool { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!( + " - Exclusive Pool max_alloc_size={}\n", + BytesFormat::new(self.max_alloc_size) + ))?; + + for page in self.pages.iter() { + let is_free = page.slice.is_free(); + let size = BytesFormat::new(page.slice.effective_size()); + + f.write_fmt(format_args!(" - Page {size} is_free={is_free}\n"))?; + } + + if !self.pages.is_empty() { + f.write_fmt(format_args!("\n{}\n", self.get_memory_usage()))?; + } + + Ok(()) + } +} + +const SIZE_AVG_DECAY: f64 = 0.01; + +// How many times to find the allocation 'free' before deallocating it. +const ALLOC_AFTER_FREE: u32 = 5; + +struct MemoryPage { + slice: Slice, + alloc_size: u64, + free_count: u32, +} + +impl ExclusiveMemoryPool { + pub(crate) fn new( + max_alloc_size: u64, + alignment: u64, + dealloc_period: u64, + pool_pos: u8, + ) -> Self { + // Pages should be allocated to be aligned. + assert_eq!(max_alloc_size % alignment, 0); + + Self { + pages: Vec::new(), + pages_tmp: Vec::new(), + alignment, + dealloc_period, + last_dealloc_check: 0, + max_alloc_size, + cur_avg_size: max_alloc_size as f64 / 2.0, + location_base: MemoryLocation::new(pool_pos, 0, 0), + } + } + + /// Finds a free page that can contain the given size + /// Returns a slice on that page if successful. + fn get_free_page(&mut self, size: u64) -> Option<&mut MemoryPage> { + // Return the smallest free page that fits. + self.pages + .iter_mut() + .filter(|page| page.alloc_size >= size && page.slice.is_free()) + .min_by_key(|page| page.free_count) + } + + fn alloc_page( + &mut self, + storage: &mut Storage, + size: u64, + ) -> Result<(usize, &mut MemoryPage), IoError> { + let alloc_size = (self.cur_avg_size as u64) + .max(size) + .next_multiple_of(self.alignment); + + let storage = storage.alloc(alloc_size)?; + + let padding = calculate_padding(size, self.alignment); + let mut slice = Slice::new(storage, padding); + + // Return a smaller part of the slice. By construction, we only ever + // get a page with a big enough size, so this is ok to do. + slice.storage.utilization = StorageUtilization { offset: 0, size }; + slice.padding = padding; + + self.pages.push(MemoryPage { + slice, + alloc_size, + // Start the allocation at 'almost ready to free'. Every use will decrement this. + // This means allocations start as "suspected as unused" and over time will be kept for longer. + free_count: ALLOC_AFTER_FREE - 1, + }); + + let idx = self.pages.len() - 1; + Ok((idx, &mut self.pages[idx])) + } +} + +impl MemoryPool for ExclusiveMemoryPool { + fn accept(&self, size: u64) -> bool { + self.max_alloc_size >= size + } + + /// Reserves memory of specified size using the reserve algorithm, and return + /// a handle to the reserved memory. + /// + /// Also clean ups, merging free slices together if permitted by the merging strategy + fn try_reserve(&mut self, size: u64) -> Option { + self.cur_avg_size = + self.cur_avg_size * (1.0 - SIZE_AVG_DECAY) + size as f64 * SIZE_AVG_DECAY; + + let padding = calculate_padding(size, self.alignment); + + self.get_free_page(size).map(|page| { + // Return a smaller part of the slice. By construction, we only ever + // get a page with a big enough size, so this is ok to do. + page.slice.storage.utilization = StorageUtilization { offset: 0, size }; + page.slice.padding = padding; + page.free_count = page.free_count.saturating_sub(1); + page.slice.handle.clone() + }) + } + + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, storage)) + )] + fn alloc( + &mut self, + storage: &mut Storage, + size: u64, + ) -> Result { + if size > self.max_alloc_size { + return Err(IoError::BufferTooBig { + size, + backtrace: BackTrace::capture(), + }); + } + + let (idx, page) = self.alloc_page(storage, size)?; + let handle = page.slice.handle.clone(); + let mut location = self.location_base; + location.page = idx as u16; + handle.descriptor().update_location(location); + + Ok(handle) + } + + fn get_memory_usage(&self) -> MemoryUsage { + let used_slices: Vec<_> = self + .pages + .iter() + .filter(|page| !page.slice.is_free()) + .collect(); + + MemoryUsage { + number_allocs: used_slices.len() as u64, + bytes_in_use: used_slices + .iter() + .map(|page| page.slice.storage.size()) + .sum(), + bytes_padding: used_slices.iter().map(|page| page.slice.padding).sum(), + bytes_reserved: self.pages.iter().map(|page| page.alloc_size).sum(), + } + } + + fn cleanup( + &mut self, + storage: &mut Storage, + alloc_nr: u64, + explicit: bool, + ) { + // Check such that an alloc is free after at most dealloc_period. + let check_period = self.dealloc_period / (ALLOC_AFTER_FREE as u64); + + if explicit || alloc_nr - self.last_dealloc_check >= check_period { + self.last_dealloc_check = alloc_nr; + + for mut page in self.pages.drain(..) { + if page.slice.is_free() { + page.free_count += 1; + + // If free found is sufficiently high (ie. we've seen this alloc as free multiple times, + // without it being used in the meantime), deallocate it. + if page.free_count >= ALLOC_AFTER_FREE || explicit { + storage.dealloc(page.slice.storage.id); + continue; + } + } + + let page_index = self.pages_tmp.len(); + page.slice + .handle + .descriptor() + .update_page(page_index as u16); + self.pages_tmp.push(page); + } + + core::mem::swap(&mut self.pages, &mut self.pages_tmp); + } + } + + fn bind( + &mut self, + old: ManagedMemoryHandle, + new: ManagedMemoryHandle, + cursor: u64, + ) -> Result<(), IoError> { + let id_old = old.descriptor(); + let page = &mut self.pages[id_old.page()]; + new.descriptor().update_location(id_old.location()); + + page.slice.handle = new; + page.slice.cursor = cursor; + + Ok(()) + } + + fn find(&self, binding: &ManagedMemoryBinding) -> Result<&Slice, IoError> { + let binding_descriptor = binding.descriptor(); + let page_index = binding_descriptor.page(); + + let page = self + .pages + .get(page_index) + .ok_or_else(|| IoError::NotFound { + backtrace: BackTrace::capture(), + reason: alloc::format!("Memory page {} doesn't exist", page_index).into(), + })?; + + Ok(&page.slice) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/handle.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/handle.rs new file mode 100644 index 00000000..0bb85f09 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/handle.rs @@ -0,0 +1,269 @@ +use crate::memory_management::MemoryHandle; +use alloc::sync::Arc; +use core::cell::Cell; + +/// Managed Memory handle +#[derive(Debug)] +pub struct ManagedMemoryHandle { + descriptor: Arc, + // Holds only the reference counts of the handle. + handle_count: Arc<()>, +} + +/// Binding of a memory handle +#[derive(Debug)] +pub struct ManagedMemoryBinding { + descriptor: Arc, +} + +impl Clone for ManagedMemoryHandle { + fn clone(&self) -> Self { + Self { + descriptor: self.descriptor.clone(), + handle_count: self.handle_count.clone(), + } + } +} + +/// Managed memory descriptor. +/// +/// The location is wrapped in `Cell` for interior mutability: multiple +/// handles share the same descriptor via `Arc`, yet the memory management +/// system needs to update the location after creation (e.g. during +/// `reserve` / `bind`). All mutation happens on a single device thread, +/// so `Cell` is safe — we just need `unsafe impl Sync` because `Cell` +/// is `!Sync`. +/// +/// An alternative would be `spin::Mutex` which avoids the +/// `unsafe impl Sync` at the cost of a lock on every access. +pub(crate) struct ManagedMemoryDescriptor { + pub(crate) id: ManagedMemoryId, + location: Cell, +} + +// SAFETY: The channel requires ManagedMemoryHandle to be Send + Sync. +// Cell is _not_ Sync, but, we know that we only access this from the device thread, +// so we lie to the compiler and claim it is Sync. Other code must NOT rely on +// ManagedMemoryDescriptor being Send + Sync. +unsafe impl Send for ManagedMemoryDescriptor {} +unsafe impl Sync for ManagedMemoryDescriptor {} + +impl core::fmt::Debug for ManagedMemoryDescriptor { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("ManagedMemoryDescriptor") + .field("id", &self.id) + .field("location", &self.location()) + .finish() + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +/// Managed memory unique identifier. +pub struct ManagedMemoryId { + pub(crate) value: usize, +} + +impl PartialEq for ManagedMemoryDescriptor { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Eq for ManagedMemoryDescriptor {} + +#[derive(Clone, Copy, Debug)] +/// Defines where the [`ManagedMemoryId`] is located. +pub(crate) struct MemoryLocation { + /// The memory pool index in the global memory management. + pub pool: u8, + /// The memory page index in a memory pool. + pub page: u16, + /// The memory slice index in a memory page. + pub slice: u32, + /// Whether the memory location is known/initialized. + pub init: u8, +} + +impl ManagedMemoryDescriptor { + /// Update the memory location for the given [`ManagedMemoryId`]. + pub(crate) fn update_location(&self, location: MemoryLocation) { + self.location.set(location); + } + + /// Update only the slice position for the given [`ManagedMemoryId`]. + pub(crate) fn update_slice(&self, slice: u32) { + self.location.update(|mut loc| { + loc.slice = slice; + loc + }); + } + + /// Update only the memory page position for the given [`ManagedMemoryId`]. + pub fn update_page(&self, page: u16) { + self.location.update(|mut loc| { + loc.page = page; + loc + }); + } + + /// Retrieves the current location. + pub(crate) fn location(&self) -> MemoryLocation { + self.location.get() + } + + pub(crate) fn slice(&self) -> usize { + self.location.get().slice as usize + } + + pub(crate) fn page(&self) -> usize { + self.location.get().page as usize + } +} + +impl MemoryLocation { + /// Creates a new memory location. + pub(crate) fn new(pool: u8, page: u16, slice: u32) -> Self { + Self { + pool, + page, + slice, + init: 1, + } + } + + /// Creates a new uninitialized memory location. + pub(crate) fn uninit() -> Self { + Self { + pool: 0, + page: 0, + slice: 0, + init: 0, + } + } +} + +impl ManagedMemoryHandle { + /// Creates a new managed memory handle. + pub fn new() -> Self { + let value = Self::gen_id(); + + Self { + descriptor: Arc::new(ManagedMemoryDescriptor { + id: ManagedMemoryId { value }, + location: Cell::new(MemoryLocation::uninit()), + }), + handle_count: Arc::new(()), + } + } + + /// Retrieves the descriptor for the current handle. + pub(crate) fn descriptor(&self) -> &ManagedMemoryDescriptor { + &self.descriptor + } + + /// Return whether the current handle can be modified in-place. + pub fn can_mut(&self) -> bool { + Arc::strong_count(&self.handle_count) <= 2 + } + + /// Return whether the current handle is free. + pub fn is_free(&self) -> bool { + Arc::strong_count(&self.descriptor) <= 1 + } + + /// Returns the binding for the current handle. + pub fn binding(self) -> ManagedMemoryBinding { + ManagedMemoryBinding { + descriptor: self.descriptor.clone(), + } + } + + fn gen_id() -> usize { + static COUNTER: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); + let value = COUNTER.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + if value == usize::MAX { + core::panic!("Memory ID overflowed"); + } + value + } +} + +impl ManagedMemoryBinding { + /// Retrieves the descriptor for the current binding. + pub(crate) fn descriptor(&self) -> &ManagedMemoryDescriptor { + &self.descriptor + } +} + +impl Default for ManagedMemoryHandle { + fn default() -> Self { + Self::new() + } +} + +impl Clone for ManagedMemoryBinding { + fn clone(&self) -> Self { + Self { + descriptor: self.descriptor.clone(), + } + } +} + +impl MemoryHandle for ManagedMemoryHandle { + fn can_mut(&self) -> bool { + self.can_mut() + } + + fn binding(self) -> ManagedMemoryBinding { + self.binding() + } +} + +/// Calculates a best-effort heuristic for the alignment of row-aligned tensors. +/// Prefers contiguous alignments for unit dimensions, 16-byte minimum alignment for non-unit, +/// scaling with input size up to `buffer_align`. +pub fn optimal_align(shape: usize, elem_size: usize, buffer_align: usize) -> usize { + if shape == 1 { + elem_size + } else { + (shape * elem_size) + .next_power_of_two() + .clamp(16, buffer_align) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_memory_id_mutability() { + let handle1 = ManagedMemoryHandle::new(); + handle1.descriptor().update_slice(4); + assert_eq!(handle1.descriptor().slice(), 4); + + let handle2 = ManagedMemoryHandle::new(); + handle2 + .clone() + .descriptor() + .update_location(handle1.descriptor().location()); + assert_eq!(handle2.descriptor().slice(), 4); + } + + #[test] + fn test_location_visible_through_shared_arc() { + let handle = ManagedMemoryHandle::new(); + let handle2 = handle.clone(); + + let location = MemoryLocation::new(1, 2, 3); + handle.descriptor().update_location(location); + + assert_eq!(handle2.descriptor().location().pool, 1); + assert_eq!(handle2.descriptor().location().page, 2); + assert_eq!(handle2.descriptor().location().slice, 3); + assert_eq!(handle2.descriptor().location().init, 1); + + handle.descriptor().update_slice(42); + assert_eq!(handle2.descriptor().slice(), 42); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/memory_page.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/memory_page.rs new file mode 100644 index 00000000..0c39fa40 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/memory_page.rs @@ -0,0 +1,717 @@ +use crate::{ + memory_management::{ + BytesFormat, ManagedMemoryBinding, ManagedMemoryHandle, MemoryLocation, MemoryUsage, + memory_pool::{Slice, calculate_padding}, + }, + server::IoError, + storage::{StorageHandle, StorageUtilization}, +}; +use alloc::format; +use alloc::string::String; +use alloc::vec::Vec; +use core::fmt::{Debug, Display}; +use cubecl_common::backtrace::BackTrace; + +/// A memory page is responsible to reserve [slices](Slice) of data based on a fixed [storage buffer](StorageHandle). +pub struct MemoryPage { + storage: StorageHandle, + slices: Vec, + /// This is a vector to be used temporary to store the updated slices. + /// + /// It avoids allocating a new vector all the time. + slices_tmp: Vec, + /// Memory alignment. + alignment: u64, + location_base: MemoryLocation, +} + +impl MemoryPage { + /// Creates a new memory page with the given storage and memory alignment. + pub fn new(storage: StorageHandle, alignment: u64, location_base: MemoryLocation) -> Self { + let mut this = MemoryPage { + storage: storage.clone(), + slices: Vec::new(), + slices_tmp: Vec::new(), + alignment, + location_base, + }; + + let slice = Slice::new(storage, 0); + let slice_pos = this.slices.len() as u32; + let mut location = this.location_base; + location.slice = slice_pos; + slice.handle.descriptor().update_location(location); + this.slices.push(slice); + + this + } + + /// Binds a user defined [`ManagedMemoryHandle`] to a slice in this memory pool. + pub fn bind( + &mut self, + reserved: ManagedMemoryHandle, + new: ManagedMemoryHandle, + cursor: u64, + ) -> Result<(), IoError> { + let slice = &mut self.slices[reserved.descriptor().slice()]; + new.descriptor() + .update_location(reserved.descriptor().location()); + slice.cursor = cursor; + slice.handle = new; + + Ok(()) + } + + /// Gets the [memory usage](MemoryUsage) of the current memory page. + pub fn memory_usage(&self) -> MemoryUsage { + let mut usage = MemoryUsage { + number_allocs: 0, + bytes_in_use: 0, + bytes_padding: 0, + bytes_reserved: 0, + }; + + for slice in self.slices.iter() { + usage.bytes_reserved += slice.effective_size(); + + if !slice.handle.is_free() { + usage.number_allocs += 1; + usage.bytes_in_use += slice.storage.size(); + usage.bytes_padding += slice.padding; + } + } + + usage + } + + /// Gets the [summary](MemoryPageSummary) of the current memory page. + /// + /// # Arguments + /// + /// - `memory_blocks`: whether the memory block details are included in the summary. + pub fn summary(&self, memory_blocks: bool) -> MemoryPageSummary { + let mut summary = MemoryPageSummary::default(); + + for slice in self.slices.iter() { + let is_free = slice.handle.is_free(); + if is_free { + summary.amount_free += slice.effective_size(); + summary.num_free += 1; + } else { + summary.amount_full += slice.effective_size(); + summary.num_full += 1; + } + if memory_blocks { + summary.blocks.push(MemoryBlock { + is_free, + size: slice.effective_size(), + }); + } + } + summary.amount_total = self.storage.size(); + summary.num_total = self.slices.len(); + + summary + } + + /// Reserves a slice of the given size if there is enough place in the page. + /// + /// # Notes + /// + /// If the current memory page is fragmented, meaning multiple contiguous slices of data exist, + /// you can call the [`Self::coalesce()`] function to merge those. + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + pub fn try_reserve(&mut self, size: u64) -> Option { + let padding = calculate_padding(size, self.alignment); + let effective_size = size + padding; + + for (index, slice) in self.slices.iter_mut().enumerate() { + let can_use_slice = + slice.storage.utilization.size >= effective_size && slice.handle.is_free(); + + if !can_use_slice { + continue; + } + + let can_be_split = slice.storage.utilization.size > effective_size; + let handle = slice.handle.clone(); + let storage_old = slice.storage.clone(); + + // Updates the current storage utilization. + slice.storage.utilization.size = size; + slice.padding = padding; + + if can_be_split { + let new_slice = Slice::new(storage_old.offset_start(effective_size), 0); + self.add_new_slice(index, size, new_slice); + } + + return Some(handle); + } + + None + } + + /// Gets the [storage handle](SliceHandle) with the correct offset and size using the slice + /// binding. + /// + /// If the handle isn't returned, it means the binding isn't present in the given page. + pub fn find(&self, binding: &ManagedMemoryBinding) -> Result<&Slice, IoError> { + let slice_index = binding.descriptor().slice(); + + self.slices + .get(slice_index) + .ok_or_else(|| IoError::NotFound { + backtrace: BackTrace::capture(), + reason: alloc::format!("Memory slice {} doesn't exist", slice_index).into(), + }) + } + + pub fn update_page(&mut self, page: u16) { + self.location_base.page = page; + + for slice in self.slices.iter() { + slice.descriptor().update_page(page); + } + } + + /// Recompute the memory page metadata to make sure adjacent slices are merged together into a + /// single slice. + /// + /// This is necessary to allow bigger slices to be reserved on the current page. + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + pub fn coalesce(&mut self) { + self.slices_tmp.clear(); + let mut job = self.memory_job(); + let mut tasks = job.tasks.drain(..); + + let mut task = match tasks.next() { + Some(task) => Some(task), + None => return, + }; + + let mut offset = 0; + let mut size = 0; + + for (index, slice) in self.slices.drain(..).enumerate() { + let status = match &mut task { + Some(task) => task.on_coalesce(index), + None => MemoryTaskStatus::Ignoring, + }; + + match status { + MemoryTaskStatus::StartMerging => { + offset = slice.storage.utilization.offset; + size = slice.effective_size(); + } + MemoryTaskStatus::Merging => { + size += slice.effective_size(); + } + MemoryTaskStatus::Ignoring => { + let slice_pos_updated = self.slices_tmp.len(); + slice + .handle + .descriptor() + .update_slice(slice_pos_updated as u32); + self.slices_tmp.push(slice); + } + MemoryTaskStatus::Completed => { + let slice_pos_updated = self.slices_tmp.len(); + size += slice.effective_size(); + + let mut storage = self.storage.clone(); + storage.utilization = StorageUtilization { offset, size }; + let page = Slice::new(storage, 0); + let mut location = self.location_base; + location.slice = slice_pos_updated as u32; + page.descriptor().update_location(location); + self.slices_tmp.push(page); + task = tasks.next(); + } + }; + } + + core::mem::swap(&mut self.slices, &mut self.slices_tmp); + } + + fn add_new_slice( + &mut self, + index_previous: usize, + reserved_size_previous: u64, + new_slice: Slice, + ) { + self.slices_tmp.clear(); + + let mut new_slice = Some(new_slice); + + let mut index_current = 0; + for mut slice in self.slices.drain(..) { + if index_current == index_previous { + let slice_pos_updated = self.slices_tmp.len() as u32; + slice.storage.utilization.size = reserved_size_previous; + slice.handle.descriptor().update_slice(slice_pos_updated); + self.slices_tmp.push(slice); + index_current += 1; + + // New slice + let slice_pos_updated = self.slices_tmp.len() as u32; + let new_slice = new_slice.take().unwrap(); + let mut location = self.location_base; + location.slice = slice_pos_updated; + new_slice.descriptor().update_location(location); + + self.slices_tmp.push(new_slice); + index_current += 1; + } else { + let slice_pos_updated = self.slices_tmp.len() as u32; + slice.handle.descriptor().update_slice(slice_pos_updated); + self.slices_tmp.push(slice); + index_current += 1; + } + } + + core::mem::swap(&mut self.slices, &mut self.slices_tmp); + } + + fn memory_job(&self) -> MemoryJob { + let mut job = MemoryJob::default(); + let mut task = MemoryTask::default(); + + for (index, slice) in self.slices.iter().enumerate() { + if slice.handle.is_free() { + task.size += slice.effective_size(); + task.tag_coalesce(index); + } else { + task = job.add(task); + } + } + job.add(task); + + job + } +} + +#[derive(Debug, PartialEq, Eq)] +struct MemoryBlock { + is_free: bool, + size: u64, +} + +#[derive(Default, PartialEq, Eq)] +pub struct MemoryPageSummary { + blocks: Vec, + pub amount_free: u64, + pub amount_full: u64, + pub amount_total: u64, + pub num_free: usize, + pub num_full: usize, + pub num_total: usize, +} + +impl Display for MemoryBlock { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.is_free { + true => f.write_fmt(format_args!("Free ({})", BytesFormat::new(self.size))), + false => f.write_fmt(format_args!("Reserved ({})", BytesFormat::new(self.size))), + } + } +} +impl Display for MemoryPageSummary { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("{self:?}")) + } +} + +impl Debug for MemoryPageSummary { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("\n==== Memory Page Summary ====\n")?; + f.write_str("[Info]\n")?; + + for (tag, num, amount) in [ + ("Free ", self.num_free, self.amount_free), + ("Full ", self.num_full, self.amount_full), + ("Total", self.num_total, self.amount_total), + ] { + f.write_fmt(format_args!( + " - {tag}: {} slices ({})\n", + num, + BytesFormat::new(amount), + ))?; + } + + f.write_str("\n[Blocks]\n")?; + let mut blocks = String::new(); + for (i, b) in self.blocks.iter().enumerate() { + if i == 0 { + blocks += "|"; + } + blocks += format!(" {b} |").as_str(); + } + let size = blocks.len(); + for _ in 0..size { + f.write_str("-")?; + } + f.write_str("\n")?; + f.write_str(&blocks)?; + f.write_str("\n")?; + for _ in 0..size { + f.write_str("-")?; + } + + f.write_str("\n=============================")?; + f.write_str("\n") + } +} + +impl Display for MemoryPage { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("{}", self.summary(true))) + } +} + +#[derive(Default, Debug, PartialEq, Eq)] +struct MemoryJob { + tasks: Vec, +} + +#[derive(Default, Debug, PartialEq, Eq)] +/// The goal of the memory task is to gather contiguous slice indices that can be merged into a single slice. +struct MemoryTask { + /// The first slice index to be merged. + start_index: usize, + /// The number of slices to be merged. + count: usize, + /// Which slice index is being merge right now. + cursor: usize, + /// The total size in bytes in the resulting merged slice. + size: u64, +} + +impl MemoryTask { + /// Tells the task that the given slice index will be coalesced. + fn tag_coalesce(&mut self, index: usize) { + if self.count == 0 { + self.start_index = index; + } + + debug_assert!( + self.start_index + self.count == index, + "Only contiguous index can be coalesced in a single task" + ); + + self.count += 1; + } + /// Tells the task that the given slice index is being coalesce. + fn on_coalesce(&mut self, index: usize) -> MemoryTaskStatus { + let index_current = self.start_index + self.cursor; + + if index_current == index { + self.cursor += 1; + if self.cursor == 1 { + return MemoryTaskStatus::StartMerging; + } + + if self.cursor == self.count { + return MemoryTaskStatus::Completed; + } else { + return MemoryTaskStatus::Merging; + } + } + + MemoryTaskStatus::Ignoring + } +} + +impl MemoryJob { + fn add(&mut self, mut task: MemoryTask) -> MemoryTask { + // A single index can't be merge with anything. + if task.count < 2 { + return MemoryTask::default(); + } + + let mut returned = MemoryTask::default(); + core::mem::swap(&mut task, &mut returned); + self.tasks.push(returned); + task + } +} + +#[derive(Debug)] +enum MemoryTaskStatus { + Merging, + StartMerging, + Ignoring, + Completed, +} + +#[cfg(test)] +#[allow(clippy::bool_assert_comparison, clippy::identity_op)] +mod tests { + use crate::storage::{StorageId, StorageUtilization}; + use alloc::vec; + + use super::*; + + const MB: u64 = 1024 * 1024; + + #[test_log::test] + fn test_memory_page() { + let mut page = new_memory_page(32 * MB); + let slice = page + .try_reserve(16 * MB) + .expect("Enough space to allocate a new slice"); + + assert_eq!(slice.is_free(), false); + assert_eq!(slice.can_mut(), true); + + let storage = &page + .find(&slice.binding()) + .expect("To find the correct storage") + .storage; + + assert_eq!( + storage.utilization, + StorageUtilization { + offset: 0, + size: 16 * MB + }, + "Utilization to be correct" + ); + + let summary = page.summary(true); + + assert_eq!( + summary, + MemoryPageSummary { + blocks: vec![ + MemoryBlock { + is_free: true, + size: 16 * MB + }, + MemoryBlock { + is_free: true, + size: 16 * MB + } + ], + amount_free: 32 * MB, + amount_full: 0, + amount_total: 32 * MB, + num_free: 2, + num_full: 0, + num_total: 2 + }, + "Summary is correct before coalesce", + ); + page.coalesce(); + let summary = page.summary(true); + + assert_eq!( + summary, + MemoryPageSummary { + blocks: vec![MemoryBlock { + is_free: true, + size: 32 * MB + },], + amount_free: 32 * MB, + amount_full: 0, + amount_total: 32 * MB, + num_free: 1, + num_full: 0, + num_total: 1 + }, + "Summary is correct after coalesce", + ); + } + + #[test_log::test] + fn test_memory_job() { + let mut page = new_memory_page(32 * MB); + let slice = page + .try_reserve(16 * MB) + .expect("Enough space to allocate a new slice"); + + core::mem::drop(slice); + let job = page.memory_job(); + + assert_eq!( + job, + MemoryJob { + tasks: vec![MemoryTask { + start_index: 0, + count: 2, + cursor: 0, + size: 32 * MB, + }] + } + ); + } + + #[test_log::test] + fn test_scenario() { + let mut page = new_memory_page(32 * MB); + + let slice_1 = page + .try_reserve(4 * MB) + .expect("Enough space to allocate a new slice"); + let slice_2 = page + .try_reserve(15 * MB) + .expect("Enough space to allocate a new slice"); + let slice_3 = page + .try_reserve(8 * MB) + .expect("Enough space to allocate a new slice"); + let slice_4 = page + .try_reserve(4 * MB) + .expect("Enough space to allocate a new slice"); + + assert_eq!( + page.summary(true), + MemoryPageSummary { + blocks: vec![ + MemoryBlock { + is_free: false, + size: 4 * MB + }, + MemoryBlock { + is_free: false, + size: 15 * MB + }, + MemoryBlock { + is_free: false, + size: 8 * MB + }, + MemoryBlock { + is_free: false, + size: 4 * MB + }, + MemoryBlock { + is_free: true, + size: 1 * MB + } + ], + amount_free: 1 * MB, + amount_full: 31 * MB, + amount_total: 32 * MB, + num_free: 1, + num_full: 4, + num_total: 5 + }, + ); + + let slice_5 = page.try_reserve(8 * MB); + assert!(slice_5.is_none(), "No more place"); + + core::mem::drop(slice_2); + let slice_5 = page.try_reserve(9 * MB); + assert!(slice_5.is_some(), "Now we have more place"); + + let slice_6 = page.try_reserve(9 * MB); + assert!(slice_6.is_none(), "No more place"); + + core::mem::drop(slice_3); + let slice_6 = page.try_reserve(9 * MB); + assert!(slice_6.is_none(), "No more place"); + + page.coalesce(); + + assert_eq!( + page.summary(true), + MemoryPageSummary { + blocks: vec![ + MemoryBlock { + is_free: false, + size: 4 * MB + }, + MemoryBlock { + is_free: false, + size: 9 * MB + }, + MemoryBlock { + is_free: true, + size: 14 * MB + }, + MemoryBlock { + is_free: false, + size: 4 * MB + }, + MemoryBlock { + is_free: true, + size: 1 * MB + } + ], + amount_free: 15 * MB, + amount_full: 17 * MB, + amount_total: 32 * MB, + num_free: 2, + num_full: 3, + num_total: 5 + }, + ); + + assert_eq!( + page.find(&slice_4.clone().binding()) + .unwrap() + .storage + .utilization, + StorageUtilization { + offset: 27 * MB, + size: 4 * MB + }, + "Utilization to be correct" + ); + + let slice_6 = page.try_reserve(9 * MB); + assert!(slice_6.is_some(), "Now we have more place"); + core::mem::drop(slice_1); + core::mem::drop(slice_4); + + page.coalesce(); + + assert_eq!( + page.find(&slice_6.clone().unwrap().binding()) + .unwrap() + .storage + .utilization, + StorageUtilization { + offset: 13 * MB, + size: 9 * MB + }, + "Utilization to be correct" + ); + + assert_eq!( + page.summary(true), + MemoryPageSummary { + blocks: vec![ + MemoryBlock { + is_free: true, + size: 4 * MB + }, + MemoryBlock { + is_free: false, + size: 9 * MB + }, + MemoryBlock { + is_free: false, + size: 9 * MB + }, + MemoryBlock { + is_free: true, + size: 10 * MB + } + ], + amount_free: 14 * MB, + amount_full: 18 * MB, + amount_total: 32 * MB, + num_free: 2, + num_full: 2, + num_total: 4 + }, + ); + } + + fn new_memory_page(size: u64) -> MemoryPage { + let storage = StorageHandle::new(StorageId::new(), StorageUtilization { offset: 0, size }); + + MemoryPage::new(storage, 4, MemoryLocation::new(0, 0, 0)) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/mod.rs new file mode 100644 index 00000000..b9bf1dcb --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/mod.rs @@ -0,0 +1,14 @@ +mod base; +mod exclusive_pool; +pub(crate) mod handle; +mod memory_page; +mod persistent_pool; +mod sliced_pool; + +pub(crate) use base::*; +pub(crate) use exclusive_pool::*; +pub(crate) use memory_page::*; +pub(crate) use persistent_pool::*; +pub(crate) use sliced_pool::*; + +pub use handle::*; diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/persistent_pool.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/persistent_pool.rs new file mode 100644 index 00000000..9be976cb --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/persistent_pool.rs @@ -0,0 +1,275 @@ +use super::{ManagedMemoryHandle, MemoryPool, Slice, calculate_padding}; +use crate::memory_management::{BytesFormat, MemoryLocation}; +use crate::storage::StorageUtilization; +use crate::{memory_management::MemoryUsage, server::IoError}; +use alloc::vec; +use alloc::vec::Vec; +use cubecl_common::backtrace::BackTrace; +use hashbrown::HashMap; + +pub struct PersistentPool { + slices: Vec, + sizes: HashMap>, + alignment: u64, + max_alloc_size: u64, + location_base: MemoryLocation, +} + +impl core::fmt::Display for PersistentPool { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + for (size, positions) in self.sizes.iter() { + let mut num_free = 0; + let mut num_full = 0; + let total = positions.len(); + + for pos in positions { + let slice = &self.slices[*pos]; + let is_free = slice.is_free(); + if is_free { + num_free += 1; + } else { + num_full += 1; + } + } + + f.write_fmt(format_args!( + " - Slices {} => {num_free} free - {num_full} full - {total} total\n", + BytesFormat::new(*size) + ))?; + } + + if !self.sizes.is_empty() { + f.write_fmt(format_args!("\n{}\n", self.get_memory_usage()))?; + } + + Ok(()) + } +} + +impl PersistentPool { + pub fn new(max_alloc_size: u64, alignment: u64, pool_pos: u8) -> Self { + Self { + slices: Vec::new(), + sizes: HashMap::new(), + max_alloc_size, + alignment, + location_base: MemoryLocation::new(pool_pos, 0, 0), + } + } + + pub fn has_size(&mut self, size: u64) -> bool { + let padding = calculate_padding(size, self.alignment); + let effective_size = size + padding; + self.sizes.contains_key(&effective_size) + } +} + +impl MemoryPool for PersistentPool { + fn accept(&self, size: u64) -> bool { + self.max_alloc_size >= size + } + + fn find(&self, binding: &super::ManagedMemoryBinding) -> Result<&Slice, IoError> { + let slice_index = binding.descriptor().slice(); + + self.slices + .get(slice_index) + .ok_or_else(|| IoError::NotFound { + backtrace: BackTrace::capture(), + reason: alloc::format!("Memory slice {} doesn't exist", slice_index).into(), + }) + } + + fn try_reserve(&mut self, size: u64) -> Option { + let padding = calculate_padding(size, self.alignment); + let effective_size = size + padding; + + if let Some(positions) = self.sizes.get_mut(&effective_size) { + for pos in positions { + let slice = &mut self.slices[*pos]; + + if slice.is_free() { + slice.storage.utilization.size = size; + slice.storage.utilization.offset = 0; + return Some(slice.handle.clone()); + } + } + } + + None + } + + fn alloc( + &mut self, + storage: &mut Storage, + size: u64, + ) -> Result { + let padding = calculate_padding(size, self.alignment); + let effective_size = size + padding; + + let storage_handle = storage.alloc(effective_size)?; + let mut slice = Slice::new(storage_handle, padding); + slice.storage.utilization = StorageUtilization { offset: 0, size }; + let slice_id = slice.descriptor(); + let slice_pos = self.slices.len(); + let mut location = self.location_base; + location.slice = slice_pos as u32; + slice_id.update_location(location); + + match self.sizes.get_mut(&effective_size) { + Some(vals) => { + vals.push(slice_pos); + } + None => { + self.sizes.insert(effective_size, vec![slice_pos]); + } + } + + let handle = slice.handle.clone(); + self.slices.push(slice); + + Ok(handle) + } + + fn get_memory_usage(&self) -> MemoryUsage { + let used_slices: Vec<_> = self + .slices + .iter() + .filter(|slice| !slice.is_free()) + .collect(); + + MemoryUsage { + number_allocs: used_slices.len() as u64, + bytes_in_use: used_slices.iter().map(|slice| slice.storage.size()).sum(), + bytes_padding: used_slices.iter().map(|slice| slice.padding).sum(), + bytes_reserved: self.slices.iter().map(|slice| slice.effective_size()).sum(), + } + } + + fn cleanup( + &mut self, + storage: &mut Storage, + _alloc_nr: u64, + explicit: bool, + ) { + if explicit { + // We have to recompute all locations, so it's just safer to rebuild everything. + let mut slices = Vec::new(); + let mut sizes = HashMap::>::new(); + + for slice in self.slices.drain(..) { + if slice.is_free() { + storage.dealloc(slice.storage.id); + } else { + let slice_pos = slices.len(); + let effective_size = slice.effective_size(); + slice.descriptor().update_slice(slice_pos as u32); + slices.push(slice); + + match sizes.get_mut(&effective_size) { + Some(vals) => { + vals.push(slice_pos); + } + None => { + sizes.insert(effective_size, vec![slice_pos]); + } + } + } + } + + self.sizes = sizes; + self.slices = slices; + storage.flush(); + } + } + + fn bind( + &mut self, + old: ManagedMemoryHandle, + new: ManagedMemoryHandle, + cursor: u64, + ) -> Result<(), IoError> { + let slice = &mut self.slices[old.descriptor().slice()]; + new.descriptor() + .update_location(old.descriptor().location()); + slice.cursor = cursor; + slice.handle = new; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use crate::memory_management::memory_pool::calculate_padding; + use crate::storage::BytesStorage; + + use super::*; + + /// `alloc` and `try_reserve` must use the same `sizes` key (`size + padding`). See #1224. + #[test_log::test] + fn persistent_pool_try_reserve_reuses_slice_with_padding() { + let mut storage = BytesStorage::default(); + let alignment = 4u64; + let mut pool = PersistentPool::new(1024 * 1024, alignment, 0); + + let size = 1025u64; + assert_ne!( + calculate_padding(size, alignment), + 0, + "test needs non-zero padding so alloc vs try_reserve keys differed pre-fix" + ); + + let handle = pool.alloc(&mut storage, size).expect("alloc"); + assert!( + pool.try_reserve(size).is_none(), + "slice must stay reserved while the handle is alive" + ); + + core::mem::drop(handle); + + assert!( + pool.try_reserve(size).is_some(), + "freed slice should be reusable" + ); + } + + #[test_log::test] + fn persistent_pool() { + let mut storage = BytesStorage::default(); + let mut pool = PersistentPool::new(1024 * 1024, 4, 0); + + let result = pool.try_reserve(1024); + assert!(result.is_none(), "No alloc yet"); + + let alloc1 = pool.alloc(&mut storage, 1024); + let result = pool.try_reserve(1024); + assert!(result.is_none(), "No free slice yet, handle1 is alive"); + + core::mem::drop(alloc1); + let result = pool.try_reserve(1024); + assert!(result.is_some(), "Handle1 is free to be reused."); + core::mem::drop(result); + + let result = pool.try_reserve(1025); + assert!(result.is_none(), "Not the same size."); + + let alloc2 = pool.alloc(&mut storage, 1024); + let usage = pool.get_memory_usage(); + assert_eq!(usage.bytes_in_use, 1024); + assert_eq!(usage.bytes_reserved, 2048); + + let result = pool.try_reserve(1024); + let usage = pool.get_memory_usage(); + assert!(result.is_some(), "Handle1 is free to be reused."); + assert_eq!(usage.bytes_in_use, 2048); + assert_eq!(usage.bytes_reserved, 2048); + + core::mem::drop(alloc2); + core::mem::drop(result); + + let usage = pool.get_memory_usage(); + assert_eq!(usage.bytes_in_use, 0); + assert_eq!(usage.bytes_reserved, 2048); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/sliced_pool.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/sliced_pool.rs new file mode 100644 index 00000000..6fe32b36 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/memory_pool/sliced_pool.rs @@ -0,0 +1,177 @@ +use crate::{ + memory_management::{ + BytesFormat, ManagedMemoryHandle, MemoryLocation, MemoryUsage, + memory_pool::{MemoryPage, MemoryPool, Slice}, + }, + server::IoError, + storage::StorageId, +}; +use alloc::vec::Vec; +use core::fmt::Display; + +pub struct SlicedPool { + pages: Vec<(MemoryPage, StorageId)>, + pages_tmp: Vec<(MemoryPage, StorageId)>, + page_size: u64, + alignment: u64, + max_alloc_size: u64, + location_base: MemoryLocation, +} + +impl SlicedPool { + pub fn new(page_size: u64, max_slice_size: u64, alignment: u64, pool_pos: u8) -> Self { + Self { + pages: Vec::new(), + pages_tmp: Vec::new(), + page_size, + alignment, + max_alloc_size: max_slice_size, + location_base: MemoryLocation::new(pool_pos, 0, 0), + } + } +} + +impl MemoryPool for SlicedPool { + fn accept(&self, size: u64) -> bool { + self.max_alloc_size >= size + || + // If the size is close to the page size so it doesn't create much fragmentation with + // unused space. + match self.page_size.checked_sub(size) { + Some(diff) => diff * 5 < self.page_size, // 20 % unused space is the max allowed. + None => false, + } + } + + fn find(&self, binding: &super::ManagedMemoryBinding) -> Result<&Slice, IoError> { + let (page, _) = &self.pages[binding.descriptor().page()]; + page.find(binding) + } + + fn try_reserve(&mut self, size: u64) -> Option { + for (page, _) in self.pages.iter_mut() { + page.coalesce(); + if let Some(handle) = page.try_reserve(size) { + return Some(handle); + } + } + + None + } + + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, storage)) + )] + fn alloc( + &mut self, + storage: &mut Storage, + size: u64, + ) -> Result { + let storage = storage.alloc(self.page_size)?; + + let storage_id = storage.id; + let mut location_base = self.location_base; + location_base.page = self.pages.len() as u16; + + let mut page = MemoryPage::new(storage, self.alignment, location_base); + let returned = page.try_reserve(size); + self.pages.push((page, storage_id)); + + Ok(returned.expect("effective_size to be smaller than page_size")) + } + + fn get_memory_usage(&self) -> MemoryUsage { + let mut usage = MemoryUsage { + number_allocs: 0, + bytes_in_use: 0, + bytes_padding: 0, + bytes_reserved: 0, + }; + + for (page, _) in self.pages.iter() { + let current = page.memory_usage(); + usage = usage.combine(current); + } + + usage + } + + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, storage)) + )] + fn cleanup( + &mut self, + storage: &mut Storage, + _alloc_nr: u64, + explicit: bool, + ) { + if !explicit { + return; + } + + for (mut page, id) in self.pages.drain(..) { + page.coalesce(); + let summary = page.summary(false); + + if summary.amount_free == summary.amount_total { + storage.dealloc(id); + } else { + let page_pos = self.pages_tmp.len() as u16; + page.update_page(page_pos); + self.pages_tmp.push((page, id)); + } + } + + core::mem::swap(&mut self.pages, &mut self.pages_tmp); + } + + /// Binds a user defined [`ManagedMemoryHandle`] to a slice in this memory pool. + fn bind( + &mut self, + reserved: ManagedMemoryHandle, + assigned: ManagedMemoryHandle, + cursor: u64, + ) -> Result<(), IoError> { + let (page, _) = &mut self.pages[reserved.descriptor().page()]; + + page.bind(reserved, assigned, cursor)?; + + Ok(()) + } +} + +impl Display for SlicedPool { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + if self.pages.is_empty() { + return Ok(()); + } + + f.write_fmt(format_args!( + " - Sliced Pool page_size={} max_alloc_size={}\n", + BytesFormat::new(self.page_size), + BytesFormat::new(self.max_alloc_size) + ))?; + + for (page, id) in self.pages.iter() { + let summary = page.summary(false); + f.write_fmt(format_args!( + " - Page {id} num_slices={} =>", + summary.num_total + ))?; + + let size_free = BytesFormat::new(summary.amount_free); + let size_full = BytesFormat::new(summary.amount_full); + let size_total = BytesFormat::new(summary.amount_total); + + f.write_fmt(format_args!( + " {size_free} free - {size_full} full - {size_total} total\n" + ))?; + } + + f.write_fmt(format_args!("\n{}\n", self.get_memory_usage()))?; + + Ok(()) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/mod.rs new file mode 100644 index 00000000..a143b2be --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/memory_management/mod.rs @@ -0,0 +1,75 @@ +pub(crate) mod memory_pool; + +mod base; + +/// Export utilities to keep track of CPU buffers when performing async data copies. +pub mod drop_queue; + +pub use base::*; + +/// Dynamic memory management strategy. +mod memory_manage; +pub use memory_manage::*; + +use alloc::vec::Vec; + +/// The type of memory pool to use. +#[derive(Debug, Clone)] +pub enum PoolType { + /// Use a memory where every allocation is a separate page. + ExclusivePages { + /// The minimum number of bytes to allocate in this pool. + max_alloc_size: u64, + }, + /// Use a memory where each allocation is a slice of a bigger allocation. + SlicedPages { + /// The page size to allocate. + page_size: u64, + /// The maximum size of a slice to allocate in the pool. + max_slice_size: u64, + }, +} + +/// Options to create a memory pool. +#[derive(Debug, Clone)] +pub struct MemoryPoolOptions { + /// What kind of pool to use. + pub pool_type: PoolType, + /// Period after which allocations are deemed unused and deallocated. + /// + /// This period is measured in the number of allocations in the parent allocator. If a page + /// in the pool was unused for the entire period, it will be deallocated. This period is + /// approximmate, as checks are only done occasionally. + pub dealloc_period: Option, +} + +/// High level configuration of memory management. +#[derive(Clone, Debug)] +pub enum MemoryConfiguration { + /// The default preset, which uses pools that allocate sub slices. + #[cfg(not(exclusive_memory_only))] + SubSlices, + /// Default preset for using exclusive pages. + /// This can be necessary for backends don't support sub-slices. + ExclusivePages, + /// Custom settings. + Custom { + /// Options for each pool to construct. When allocating, the first + /// possible pool will be picked for an allocation. + pool_options: Vec, + }, +} + +#[allow(clippy::derivable_impls)] +impl Default for MemoryConfiguration { + fn default() -> Self { + #[cfg(exclusive_memory_only)] + { + MemoryConfiguration::ExclusivePages + } + #[cfg(not(exclusive_memory_only))] + { + MemoryConfiguration::SubSlices + } + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/runtime.rs b/third_party/cubecl-runtime-0.10.0-patched/src/runtime.rs new file mode 100644 index 00000000..2f480488 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/runtime.rs @@ -0,0 +1,52 @@ +use alloc::boxed::Box; +use alloc::vec::Vec; +use cubecl_common::device::{Device, DeviceId}; +use cubecl_ir::TargetProperties; +use cubecl_zspace::{Shape, Strides}; + +use crate::{ + client::ComputeClient, + compiler::{Compiler, CubeTask}, + server::ComputeServer, +}; + +/// Runtime for the `CubeCL`. +pub trait Runtime: Sized + Send + Sync + 'static + core::fmt::Debug + Clone { + /// The compiler used to compile the inner representation into tokens. + type Compiler: Compiler; + /// The compute server used to run kernels and perform autotuning. + type Server: ComputeServer>>; + /// The device used to retrieve the compute client. + type Device: Device; + + /// Retrieve the compute client from the runtime device. + fn client(device: &Self::Device) -> ComputeClient; + + /// The runtime name on the given device. + fn name(client: &ComputeClient) -> &'static str; + + /// Return true if global input array lengths should be added to kernel info. + fn require_array_lengths() -> bool { + false + } + + /// Returns the maximum cube count on each dimension that can be launched. + fn max_cube_count() -> (u32, u32, u32); + + /// Whether a tensor with `shape` and `strides` can be read as is. If the result is false, the + /// tensor should be made contiguous before reading. + fn can_read_tensor(shape: &Shape, strides: &Strides) -> bool; + + /// Returns the properties of the target hardware architecture. + fn target_properties() -> TargetProperties; + + /// Returns all devices available under the provided type id. + fn enumerate_devices( + type_id: u16, + info: &::Info, + ) -> Vec; + /// Returns all devices that can be handled by the runtime. + fn enumerate_all_devices(info: &::Info) -> Vec { + Self::enumerate_devices(0, info) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/server/base.rs b/third_party/cubecl-runtime-0.10.0-patched/src/server/base.rs new file mode 100644 index 00000000..601da7b9 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/server/base.rs @@ -0,0 +1,1159 @@ +use super::Handle; +use crate::{ + client::ComputeClient, + compiler::CompilationError, + config::{CubeClRuntimeConfig, RuntimeConfig, compilation::BoundsCheckMode}, + kernel::KernelMetadata, + logging::ServerLogger, + memory_management::{ManagedMemoryHandle, MemoryAllocationMode, MemoryUsage}, + runtime::Runtime, + server::Binding, + storage::{ComputeStorage, ManagedResource}, + tma::{OobFill, TensorMapFormat, TensorMapInterleave, TensorMapPrefetch, TensorMapSwizzle}, +}; +use ahash::AHasher; +use alloc::boxed::Box; +#[cfg(feature = "profile-tracy")] +use alloc::format; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::{ + fmt::Debug, + hash::{Hash, Hasher}, +}; +use cubecl_common::{ + backtrace::BackTrace, + bytes::Bytes, + device::{self, DeviceId}, + future::DynFut, + profile::ProfileDuration, + stream_id::StreamId, + stub::RwLock, +}; +use cubecl_ir::{DeviceProperties, ElemType, StorageType}; +use cubecl_zspace::{Shape, Strides, metadata::Metadata}; +use hashbrown::HashSet; +use thiserror::Error; + +#[derive(Error, Clone)] +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +/// An error during profiling. +pub enum ProfileError { + /// An unknown error happened during profiling + #[error( + "An unknown error happened during profiling\nCaused by:\n {reason}\nBacktrace:\n{backtrace}" + )] + Unknown { + /// The caused of the error + reason: String, + /// The captured backtrace. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// No profiling was registered + #[error("No profiling registered\nBacktrace:\n{backtrace}")] + NotRegistered { + /// The captured backtrace. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// A launch error happened during profiling + #[error("A launch error happened during profiling\nCaused by:\n {0}")] + Launch(#[from] LaunchError), + + /// An execution error happened during profiling + #[error("An execution error happened during profiling\nCaused by:\n {0}")] + Server(#[from] Box), +} + +impl core::fmt::Debug for ProfileError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("{self}")) + } +} + +/// Contains many different types that are useful for server implementations and compute clients. +pub struct ServerUtilities { + /// The time when `profile-tracy` is activated. + #[cfg(feature = "profile-tracy")] + pub epoch_time: web_time::Instant, + /// The GPU client when `profile-tracy` is activated. + #[cfg(feature = "profile-tracy")] + pub gpu_client: tracy_client::GpuContext, + /// Information shared between all servers. + pub properties: DeviceProperties, + /// Stable hash of the device properties + pub properties_hash: u64, + /// Information specific to the current server. + pub info: Server::Info, + /// The logger based on global cubecl configs. + pub logger: Arc, + /// How to create the allocation. + pub layout_policy: Server::MemoryLayoutPolicy, + /// How to enforce bounds checking on kernels. + pub check_mode: BoundsCheckMode, + /// A set containing the ids for which the inter-device communication has already been initialized. + pub initialized_comms: RwLock>, +} + +/// Defines how the memory layout is determined. +pub trait MemoryLayoutPolicy: Send + Sync + 'static { + /// Applies the memory layout policy to a list of descriptors. + /// + /// Returns a vector of `MemoryLayout`, one per descriptor, with layouts that share a + /// single `Binding`. + fn apply( + &self, + stream_id: StreamId, + descriptors: &[MemoryLayoutDescriptor], + ) -> (Handle, Vec); +} + +impl core::fmt::Debug for ServerUtilities +where + Server: ComputeServer, + Server::Info: core::fmt::Debug, +{ + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_struct("ServerUtilities") + .field("properties", &self.properties) + .field("info", &self.info) + .field("logger", &self.logger) + .finish() + } +} + +impl ServerUtilities { + /// Creates a new server utilities. + pub fn new( + properties: DeviceProperties, + logger: Arc, + info: S::Info, + allocator: S::MemoryLayoutPolicy, + ) -> Self { + // Start a tracy client if needed. + #[cfg(feature = "profile-tracy")] + let client = tracy_client::Client::start(); + + Self { + properties_hash: properties.checksum(), + properties, + logger, + // Create the GPU client if needed. + #[cfg(feature = "profile-tracy")] + gpu_client: client + .clone() + .new_gpu_context( + Some(&format!("{info:?}")), + // In the future should ask the server what makes sense here. 'Invalid' atm is a generic stand-in (Tracy doesn't have CUDA/RocM atm anyway). + tracy_client::GpuContextType::Invalid, + 0, // Timestamps are manually aligned to this epoch so start at 0. + 1.0, // Timestamps are manually converted to be nanoseconds so period is 1. + ) + .unwrap(), + #[cfg(feature = "profile-tracy")] + epoch_time: web_time::Instant::now(), + info, + layout_policy: allocator, + check_mode: CubeClRuntimeConfig::get().compilation.check_mode, + initialized_comms: RwLock::new(HashSet::default()), + } + } +} + +/// Kernel Launch Errors. +#[derive(Error, Clone)] +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +pub enum LaunchError { + /// The given kernel can't be compiled. + #[error("A compilation error happened during launch\nCaused by:\n {0}")] + CompilationError(#[from] CompilationError), + + /// The server is out of memory. + #[error( + "An out-of-memory error happened during launch\nCaused by:\n {reason}\nBacktrace\n{backtrace}" + )] + OutOfMemory { + /// The caused of the memory error. + reason: String, + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// Too many resources were requested + #[error("Too many resources were requested during launch\n{0}")] + TooManyResources(#[from] ResourceLimitError), + + /// Unknown launch error. + #[error( + "An unknown error happened during launch\nCaused by:\n {reason}\nBacktrace\n{backtrace}" + )] + Unknown { + /// The caused of the unknown error. + reason: String, + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// Can't launch because of an IO Error. + #[error("An io error happened during launch\nCaused by:\n {0}")] + IoError(#[from] IoError), +} + +/// Resource limit errors. +#[derive(Error, Clone)] +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +pub enum ResourceLimitError { + /// Shared memory exceeds maximum + #[error( + "Too much shared memory requested.\nRequested {requested} bytes, maximum {max} bytes available.\nBacktrace\n{backtrace}" + )] + SharedMemory { + /// Value requested + requested: usize, + /// Maximum value + max: usize, + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + /// Total units exceeds maximum + #[error( + "Total unit count exceeds maximum.\nRequested {requested} units, max units is {max}.\nBacktrace\n{backtrace}" + )] + Units { + /// Requested value + requested: u32, + /// Maximum value + max: u32, + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + /// `CubeDim` exceeds maximum + #[error( + "Cube dim exceeds maximum bounds.\nRequested {requested:?}, max is {max:?}.\nBacktrace\n{backtrace}" + )] + CubeDim { + /// Requested value + requested: (u32, u32, u32), + /// Maximum value + max: (u32, u32, u32), + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, +} + +impl core::fmt::Debug for LaunchError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("{self}")) + } +} + +impl core::fmt::Debug for ResourceLimitError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("{self}")) + } +} + +/// Error that can happen asynchronously while executing registered kernels. +#[derive(Error, Debug, Clone)] +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +pub enum ServerError { + /// A generic runtime error. + #[error("An error happened during execution\nCaused by:\n {reason}\nBacktrace:\n{backtrace}")] + Generic { + /// The details of the generic error. + reason: String, + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// A launch error happened during profiling + #[error("A launch error happened during profiling\nCaused by:\n {0}")] + Launch(#[from] LaunchError), + + /// An execution error happened during profiling + #[error("An execution error happened during profiling\nCaused by:\n {0}")] + Profile(#[from] ProfileError), + + /// An execution error happened during profiling + #[error("An execution error happened during profiling\nCaused by:\n {0}")] + Io(#[from] IoError), + + /// The server is an invalid state. + #[error("The server is in an invalid state\nCaused by:\n {errors:?}")] + ServerUnhealthy { + /// The details of the generic error. + errors: Vec, + /// The backtrace for this error. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, +} + +/// How errors are handled in a stream when executing a task. +#[derive(Clone, Copy)] +pub struct StreamErrorMode { + /// Whether the task still executes even if the stream is in error. + pub ignore: bool, + /// Whether the errors are flushed by the current task. + pub flush: bool, +} + +/// Narrow backend hook used to order an audited external write on the +/// server's native stream without synchronizing the CPU. +/// +/// The numeric handle is intentionally consumed only by a runtime-specific, +/// unsafe adapter. General CubeCL users should not use this trait. +#[doc(hidden)] +pub trait ExternalWriteServer: ComputeServer { + /// Return the native stream associated with `binding` after resolving the + /// server's normal cross-stream dependencies. + fn external_write_stream( + &mut self, + binding: Binding, + stream_id: StreamId, + ) -> Result; +} + +/// The compute server is responsible for handling resources and computations over resources. +/// +/// Everything in the server is mutable, therefore it should be solely accessed through the +/// [`ComputeClient`] for thread safety. +pub trait ComputeServer: + Send + core::fmt::Debug + ServerCommunication + device::DeviceService + 'static +where + Self: Sized, +{ + /// The kernel type defines the computation algorithms. + type Kernel: KernelMetadata; + /// Information that can be retrieved for the runtime. + type Info: Debug + Send + Sync; + /// Manages how allocations are performed for a server. + type MemoryLayoutPolicy: MemoryLayoutPolicy; + /// The [storage](ComputeStorage) type defines how data is stored and accessed. + type Storage: ComputeStorage; + + /// Initializes [memory](ManagedMemoryHandle) on the given [stream](StreamId) with the given size. + fn initialize_memory(&mut self, memory: ManagedMemoryHandle, size: u64, stream_id: StreamId); + + /// Reserves N [Bytes] of the provided sizes to be used as staging to load data. + fn staging( + &mut self, + _sizes: &[usize], + _stream_id: StreamId, + ) -> Result, ServerError> { + Err(IoError::UnsupportedIoOperation { + backtrace: BackTrace::capture(), + } + .into()) + } + + /// Retrieve the server logger. + fn logger(&self) -> Arc; + + /// Retrieve the server utilities. + fn utilities(&self) -> Arc>; + + /// Given bindings, returns the owned resources as bytes. + fn read( + &mut self, + descriptors: Vec, + stream_id: StreamId, + ) -> DynFut, ServerError>>; + + /// Writes the specified bytes into the buffers given + fn write(&mut self, descriptors: Vec<(CopyDescriptor, Bytes)>, stream_id: StreamId); + + /// Wait for the completion of every task in the server. + fn sync(&mut self, stream_id: StreamId) -> DynFut>; + + /// Given a resource handle, returns the storage resource. + fn get_resource( + &mut self, + binding: Binding, + stream_id: StreamId, + ) -> Result::Resource>, ServerError>; + + /// Executes the `kernel` over the given memory `handles`. + /// + /// Kernels have mutable access to every resource they are given + /// and are responsible of determining which should be read or written. + /// + /// # Safety + /// + /// When executing with mode [`ExecutionMode::Unchecked`], out-of-bound reads and writes can happen. + unsafe fn launch( + &mut self, + kernel: Self::Kernel, + count: CubeCount, + bindings: KernelArguments, + kind: ExecutionMode, + stream_id: StreamId, + ); + + /// Flush all outstanding tasks in the server. + fn flush(&mut self, stream_id: StreamId) -> Result<(), ServerError>; + + /// The current memory usage of the server. + fn memory_usage(&mut self, stream_id: StreamId) -> Result; + + /// Ask the server to release memory that it can release. + fn memory_cleanup(&mut self, stream_id: StreamId); + + /// Enable collecting timestamps. + fn start_profile(&mut self, stream_id: StreamId) -> Result; + + /// Disable collecting timestamps. + fn end_profile( + &mut self, + stream_id: StreamId, + token: ProfilingToken, + ) -> Result; + + /// Update the memory mode of allocation in the server. + fn allocation_mode(&mut self, mode: MemoryAllocationMode, stream_id: StreamId); +} + +/// An ID unique to any unordered combination of devices. +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +pub struct CommunicationId { + /// The ID as a `String`. + pub id: u64, +} + +impl From> for CommunicationId { + fn from(mut value: Vec) -> Self { + // Make sure that device ids are sorted so that any combination of the same devices uses the same communicator. + value.sort(); + let mut hasher = AHasher::default(); + value.hash(&mut hasher); + CommunicationId { + id: hasher.finish(), + } + } +} + +/// Different reduce operations. +pub enum ReduceOperation { + /// Sum. + Sum, + /// Mean. + Mean, +} + +/// Defines functions for optimized data transfer between servers, supporting custom communication +/// mechanisms such as peer-to-peer communication or specialized implementations. +pub trait ServerCommunication { + /// Indicates whether server-to-server communication is enabled for this implementation. + const SERVER_COMM_ENABLED: bool; + + /// Ensure that all queued collective operations have been executed. + /// + /// # Arguments + /// + /// * `stream_id` - The [`StreamId`] of the stream waiting for the sync. + /// + /// # Returns + /// + /// Returns a `Result` containing an `ServerError` if the operation fails. + #[allow(unused_variables)] + fn sync_collective(&mut self, stream_id: StreamId) -> Result<(), ServerError> { + todo!() // For backends other than cuda. + } + + /// Initialize the communication between the devices in `device_ids`. + /// + /// # Arguments + /// + /// * `device_ids` - The IDs of the devices that need communication. + /// + /// # Returns + /// + /// Returns a `Result` containing an `ServerError` if the operation fails. + #[allow(unused_variables)] + fn comm_init(&mut self, device_ids: Vec) -> Result<(), ServerError> { + unimplemented!() + } + + /// Performs an `all_reduce` operation on the input data and writes it to the output buffer. + /// see + /// + /// # Arguments + /// + /// * `src` - The data to be reduced. + /// * `dst` - Where to write the result. + /// * `dtype` - The element type of the data being reduced + /// * `stream_id` - The data's stream id. + /// * `op` - The reduce's aggregation operation e.g. mean, sum, etc. + /// * `device_ids` - The list of device ids from which to `all_reduce`. + /// + /// # Returns + /// + /// Returns a `Result` containing an `ServerError` if the operation fails. + #[allow(unused_variables)] + fn all_reduce( + &mut self, + src: Binding, + dst: Binding, + dtype: ElemType, + stream_id: StreamId, + op: ReduceOperation, + device_ids: Vec, + ) -> Result<(), ServerError> { + unimplemented!() + } + + /// Sends data from this server to a destination server. + /// + /// # Arguments + /// + /// * `desc` - A descriptor specifying the data to be sent, including shape, strides, and binding. + /// * `dtype` - The element type of the data being sent. + /// * `stream_id` - The stream ID associated with the server's operation. + /// * `device_id_dst` - ID of the device receiving the data. + /// + /// # Returns + /// + /// Returns a `Result` containing an `ServerError` if the operation fails. + #[allow(unused_variables)] + fn send( + &mut self, + desc: CopyDescriptor, + dtype: ElemType, + stream_id: StreamId, + device_id_dst: DeviceId, + ) -> Result<(), ServerError> { + unimplemented!() + } + + /// Receive data from another server. + /// + /// # Arguments + /// + /// * `handle` - The handle in which the received data is written. + /// * `dtype` - The element type of the data being sent. + /// * `stream_id` - The stream ID associated with the server's operation. + /// * `device_id_src` - ID of the device sending the data. + /// + /// # Returns + /// + /// Returns a `Result` containing an `ServerError` if the operation fails. + #[allow(unused_variables)] + fn recv( + &mut self, + handle: Handle, + dtype: ElemType, + stream_id: StreamId, + device_id_src: DeviceId, + ) -> Result<(), ServerError> { + unimplemented!() + } +} + +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +/// Profiling identification so that the server can support recursive and overlapping profilings. +pub struct ProfilingToken { + /// The token value. + pub id: u64, +} + +/// Type of allocation, either contiguous or optimized (row-aligned when possible) +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub enum MemoryLayoutStrategy { + /// Contiguous layout, with no padding + Contiguous, + /// Optimized for access speed. In practice this means row-aligned with padding for runtimes + /// that support it. + Optimized, +} + +/// Descriptor for a new tensor allocation +#[derive(new, Debug, Clone)] +pub struct MemoryLayoutDescriptor { + /// Strategy used to create the memory layout. + pub strategy: MemoryLayoutStrategy, + /// Shape of the tensor + pub shape: Shape, + /// Size of each element in the tensor (used for conversion of shape to bytes) + pub elem_size: usize, +} + +impl MemoryLayoutDescriptor { + /// Create an optimized allocation descriptor + pub fn optimized(shape: Shape, elem_size: usize) -> Self { + MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Optimized, shape, elem_size) + } + + /// Create a contiguous allocation descriptor + pub fn contiguous(shape: Shape, elem_size: usize) -> Self { + MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Contiguous, shape, elem_size) + } +} + +/// An allocation with associated strides. Strides depend on tensor layout. +#[derive(Debug, Clone)] +pub struct MemoryLayout { + /// The handle for the memory resource + pub memory: Handle, + /// TODO: `Strides` should become `Layout`. + /// + /// The strides of the tensor + pub strides: Strides, +} + +impl MemoryLayout { + /// Create a new memory layout. + pub fn new(handle: Handle, strides: impl Into) -> Self { + MemoryLayout { + memory: handle, + strides: strides.into(), + } + } +} + +/// A reason for an error. +#[derive(Default, Clone)] +pub struct Reason { + inner: ReasonInner, +} + +#[cfg(std_io)] +mod _reason_serde { + use super::*; + + use alloc::string::ToString; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + impl Serialize for Reason { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + // Use the Display implementation (via to_string) to flatten the enum + serializer.serialize_str(&self.to_string()) + } + } + + impl<'de> Deserialize<'de> for Reason { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // Deserialize into a standard String first + let s = String::deserialize(deserializer)?; + + // Wrap it in the Dynamic variant since we can't safely + // reconstruct a 'static str from a runtime string. + Ok(Reason { + inner: ReasonInner::Dynamic(Arc::new(s)), + }) + } + } +} + +#[derive(Default, Clone)] +enum ReasonInner { + Static(&'static str), + Dynamic(Arc), + #[default] + NotProvided, +} + +impl core::fmt::Display for Reason { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match &self.inner { + ReasonInner::Static(content) => f.write_str(content), + ReasonInner::Dynamic(content) => f.write_str(content), + ReasonInner::NotProvided => f.write_str("No reason provided for the error"), + } + } +} + +impl core::fmt::Debug for Reason { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Display::fmt(&self, f) + } +} + +impl From<&'static str> for Reason { + fn from(value: &'static str) -> Self { + Self { + inner: ReasonInner::Static(value), + } + } +} + +impl From for Reason { + fn from(value: String) -> Self { + Self { + inner: ReasonInner::Dynamic(Arc::new(value)), + } + } +} + +/// Error returned from `create`/`read`/`write` functions. Due to async execution not all errors +/// are able to be caught, so some IO errors will still panic. +#[derive(Error, Clone)] +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +pub enum IoError { + /// Buffer size exceeds the max available + #[error("can't allocate buffer of size: {size}\n{backtrace}")] + BufferTooBig { + /// The size of the buffer in bytes. + size: u64, + /// The captured backtrace. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// Strides aren't supported for this copy operation on this runtime + #[error("the provided strides are not supported for this operation\n{backtrace}")] + UnsupportedStrides { + /// The backtrace. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// Memory wasn't found in the memory pool + #[error("couldn't find resource for that handle: {reason}\n{backtrace}")] + NotFound { + /// The backtrace. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + /// The reason the handle is invalid. + reason: Reason, + }, + + /// Handle wasn't found in the memory pool + #[error("couldn't free the handle, since it is currently in used. \n{backtrace}")] + FreeError { + /// The backtrace. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// Unknown error happened during execution + #[error("Unknown error happened during execution\n{backtrace}")] + Unknown { + /// Details of the error + description: String, + /// The backtrace. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// The current IO operation is not supported + #[error("The current IO operation is not supported\n{backtrace}")] + UnsupportedIoOperation { + /// The backtrace. + #[cfg_attr(std_io, serde(skip))] + backtrace: BackTrace, + }, + + /// Can't perform the IO operation because of a runtime error. + #[error("Can't perform the IO operation because of a runtime error: {0}")] + Execution(#[from] Box), +} + +impl core::fmt::Debug for IoError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("{self}")) + } +} + +/// Arguments to execute a kernel. +#[derive(Debug, Default)] +pub struct KernelArguments { + /// Buffer bindings + pub buffers: Vec, + /// Packed scalars and metadata. First scalars sorted by type, then static metadata, + /// then dynamic metadata. + pub info: MetadataBindingInfo, + /// Tensor map bindings + pub tensor_maps: Vec, +} + +impl core::fmt::Display for KernelArguments { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("KernelArguments")?; + for b in self.buffers.iter() { + f.write_fmt(format_args!("\n - buffer: {b:?}\n"))?; + } + + Ok(()) + } +} + +impl KernelArguments { + /// Create a new bindings struct + pub fn new() -> Self { + Self::default() + } + + /// Add a buffer binding + pub fn with_buffer(mut self, binding: Binding) -> Self { + self.buffers.push(binding); + self + } + + /// Extend the buffers with `bindings` + pub fn with_buffers(mut self, bindings: Vec) -> Self { + self.buffers.extend(bindings); + self + } + + /// Set the info to `info` + pub fn with_info(mut self, info: MetadataBindingInfo) -> Self { + self.info = info; + self + } + + /// Extend the tensor maps with `bindings` + pub fn with_tensor_maps(mut self, bindings: Vec) -> Self { + self.tensor_maps.extend(bindings); + self + } +} + +/// Binding of a set of scalars of the same type to execute a kernel. +/// +/// The [`ComputeServer`] is responsible to convert those info into actual [`Binding`] when launching +/// kernels. +#[derive(new, Debug, Default)] +pub struct MetadataBindingInfo { + /// Scalar and metadata values + pub data: Vec, + /// Start of the dynamically sized portion of the metadata, relative to the entire info buffer + pub dynamic_metadata_offset: usize, +} + +impl MetadataBindingInfo { + /// Create a new binding info for custom data, for externally compiled kernels. + pub fn custom(data: Vec) -> Self { + Self::new(data, 0) + } +} + +/// A binding with shape and stride info for non-contiguous reading +#[derive(new, Debug)] +pub struct CopyDescriptor { + /// Binding for the memory resource + pub handle: Binding, + /// Shape of the resource + pub shape: Shape, + /// Strides of the resource + pub strides: Strides, + /// Size of each element in the resource + pub elem_size: usize, +} + +/// A tensor map used with TMA ops +#[derive(new, Debug)] +pub struct TensorMapBinding { + /// The binding for the backing tensor + pub binding: Binding, + /// The tensormap metadata + pub map: TensorMapMeta, +} + +/// `TensorMap` metadata for the opaque proxy used in TMA copies +#[derive(Debug, Clone)] +pub struct TensorMapMeta { + /// Tensormap format (tiled or im2col) + pub format: TensorMapFormat, + /// Metadata of the backing tensor + pub metadata: Metadata, + /// Element stride, usually 1 but may be 2 for complex tensors + /// For im2col, this is equivalent to the kernel stride + pub elem_stride: Strides, + /// Interleave mode + pub interleave: TensorMapInterleave, + /// Swizzle mode + pub swizzle: TensorMapSwizzle, + /// Prefetch settings + pub prefetch: TensorMapPrefetch, + /// OOB fill value + pub oob_fill: OobFill, + /// Storage type + pub storage_ty: StorageType, +} + +/// Specifieds the number of cubes to be dispatched for a kernel. +/// +/// This translates to eg. a grid for CUDA, or to `num_workgroups` for wgsl. +#[allow(clippy::large_enum_variant)] +pub enum CubeCount { + /// Dispatch a known count of x, y, z cubes. + Static(u32, u32, u32), + /// Dispatch an amount based on the values in this buffer. The buffer should contain a u32 array [x, y, z]. + Dynamic(Binding), +} + +/// Defines how to select cube count based on the number of cubes required. +pub enum CubeCountSelection { + /// If the number of cubes is the same as required. + Exact(CubeCount), + /// If the number of cubes isn't the same as required. + /// + /// This can happen based on the hardware limit, requiring the kernel to perform OOB checks. + Approx(CubeCount, u32), +} + +impl CubeCountSelection { + /// Creates a [`CubeCount`] while respecting the hardware limits. + pub fn new(client: &ComputeClient, num_cubes: u32) -> Self { + let cube_count = cube_count_spread(&client.properties().hardware.max_cube_count, num_cubes); + + let num_cubes_actual = cube_count[0] * cube_count[1] * cube_count[2]; + let cube_count = CubeCount::Static(cube_count[0], cube_count[1], cube_count[2]); + + match num_cubes_actual == num_cubes { + true => CubeCountSelection::Exact(cube_count), + false => CubeCountSelection::Approx(cube_count, num_cubes_actual), + } + } + + /// If some cubes will be idle. + pub fn has_idle(&self) -> bool { + matches!(self, Self::Approx(..)) + } + + /// Converts into [`CubeCount`]. + pub fn cube_count(self) -> CubeCount { + match self { + CubeCountSelection::Exact(cube_count) => cube_count, + CubeCountSelection::Approx(cube_count, _) => cube_count, + } + } +} + +impl From for CubeCount { + fn from(value: CubeCountSelection) -> Self { + value.cube_count() + } +} + +impl CubeCount { + /// Create a new static cube count with the given x = y = z = 1. + pub fn new_single() -> Self { + CubeCount::Static(1, 1, 1) + } + + /// Create a new static cube count with the given x, and y = z = 1. + pub fn new_1d(x: u32) -> Self { + CubeCount::Static(x, 1, 1) + } + + /// Create a new static cube count with the given x and y, and z = 1. + pub fn new_2d(x: u32, y: u32) -> Self { + CubeCount::Static(x, y, 1) + } + + /// Create a new static cube count with the given x, y and z. + pub fn new_3d(x: u32, y: u32, z: u32) -> Self { + CubeCount::Static(x, y, z) + } + + /// Checks whether the cube count is definitely empty, i.e. has 0 dispatches. + pub fn is_empty(&self) -> bool { + match self { + Self::Static(x, y, z) => *x == 0 || *y == 0 || *z == 0, + Self::Dynamic(_) => false, + } + } +} + +impl Debug for CubeCount { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + CubeCount::Static(x, y, z) => f.write_fmt(format_args!("({x}, {y}, {z})")), + CubeCount::Dynamic(_) => f.write_str("binding"), + } + } +} + +impl Clone for CubeCount { + fn clone(&self) -> Self { + match self { + Self::Static(x, y, z) => Self::Static(*x, *y, *z), + Self::Dynamic(binding) => Self::Dynamic(binding.clone()), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +#[allow(missing_docs)] +/// The number of units across all 3 axis totalling to the number of working units in a cube. +pub struct CubeDim { + /// The number of units in the x axis. + pub x: u32, + /// The number of units in the y axis. + pub y: u32, + /// The number of units in the z axis. + pub z: u32, +} + +impl CubeDim { + /// Creates a new [`CubeDim`] based on the maximum number of tasks that can be parellalized by units, in other words, + /// by the maximum number of working units. + /// + /// # Notes + /// + /// For complex problems, you probably want to have your own logic function to create the + /// [`CubeDim`], but for simpler problems such as elemwise-operation, this is a great default. + pub fn new(client: &ComputeClient, working_units: usize) -> Self { + let properties = client.properties(); + let plane_size = properties.hardware.plane_size_max; + let plane_count = Self::calculate_plane_count_per_cube( + working_units as u32, + plane_size, + properties.hardware.num_cpu_cores, + ); + + // Make sure it respects the max units per cube (especially on wasm) + let limit = properties.hardware.max_units_per_cube / plane_size; + + // Ensure at least 1 plane so CubeDim is always valid (num_elems() > 0). + Self::new_2d(plane_size, u32::min(limit, plane_count).max(1)) + } + + fn calculate_plane_count_per_cube( + working_units: u32, + plane_dim: u32, + num_cpu_cores: Option, + ) -> u32 { + match num_cpu_cores { + Some(num_cores) => core::cmp::min(num_cores, working_units), + None => { + let plane_count_max = core::cmp::max(1, working_units / plane_dim); + + // Ensures `plane_count` is a power of 2. + const NUM_PLANE_MAX: u32 = 8u32; + const NUM_PLANE_MAX_LOG2: u32 = NUM_PLANE_MAX.ilog2(); + let plane_count_max_log2 = + core::cmp::min(NUM_PLANE_MAX_LOG2, u32::ilog2(plane_count_max)); + 2u32.pow(plane_count_max_log2) + } + } + } + + /// Create a new cube dim with x = y = z = 1. + pub const fn new_single() -> Self { + Self { x: 1, y: 1, z: 1 } + } + + /// Create a new cube dim with the given x, and y = z = 1. + pub const fn new_1d(x: u32) -> Self { + Self { x, y: 1, z: 1 } + } + + /// Create a new cube dim with the given x and y, and z = 1. + pub const fn new_2d(x: u32, y: u32) -> Self { + Self { x, y, z: 1 } + } + + /// Create a new cube dim with the given x, y and z. + /// This is equivalent to the [new](CubeDim::new) function. + pub const fn new_3d(x: u32, y: u32, z: u32) -> Self { + Self { x, y, z } + } + + /// Total numbers of units per cube + pub const fn num_elems(&self) -> u32 { + self.x * self.y * self.z + } + + /// Whether this `CubeDim` can fully contain `other` + pub const fn can_contain(&self, other: CubeDim) -> bool { + self.x >= other.x && self.y >= other.y && self.z >= other.z + } +} + +impl From<(u32, u32, u32)> for CubeDim { + fn from(value: (u32, u32, u32)) -> Self { + CubeDim::new_3d(value.0, value.1, value.2) + } +} + +impl From for (u32, u32, u32) { + fn from(val: CubeDim) -> Self { + (val.x, val.y, val.z) + } +} + +/// The kind of execution to be performed. +#[derive(Default, Hash, PartialEq, Eq, Clone, Debug, Copy)] +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +pub enum ExecutionMode { + /// Checked kernels are safe. + #[default] + Checked, + /// Validate OOB and alert if OOB access occurs + Validate, + /// Unchecked kernels are unsafe. + Unchecked, +} + +fn cube_count_spread(max: &(u32, u32, u32), num_cubes: u32) -> [u32; 3] { + let max_cube_counts = [max.0, max.1, max.2]; + let mut num_cubes = [num_cubes, 1, 1]; + let base = 2; + + let mut reduce_count = |i: usize| { + if num_cubes[i] <= max_cube_counts[i] { + return true; + } + + loop { + num_cubes[i] = num_cubes[i].div_ceil(base); + num_cubes[i + 1] *= base; + + if num_cubes[i] <= max_cube_counts[i] { + return false; + } + } + }; + + for i in 0..2 { + if reduce_count(i) { + break; + } + } + + num_cubes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test_log::test] + fn safe_num_cubes_even() { + let max = (32, 32, 32); + let required = 2048; + + let actual = cube_count_spread(&max, required); + let expected = [32, 32, 2]; + assert_eq!(actual, expected); + } + + #[test_log::test] + fn safe_num_cubes_odd() { + let max = (48, 32, 16); + let required = 3177; + + let actual = cube_count_spread(&max, required); + let expected = [25, 32, 4]; + assert_eq!(actual, expected); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/server/handle.rs b/third_party/cubecl-runtime-0.10.0-patched/src/server/handle.rs new file mode 100644 index 00000000..4ea5992b --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/server/handle.rs @@ -0,0 +1,160 @@ +use cubecl_common::stream_id::StreamId; +use cubecl_zspace::{Shape, Strides}; + +use crate::{ + memory_management::{ManagedMemoryBinding, ManagedMemoryHandle}, + server::CopyDescriptor, +}; + +/// Server handle containing the [memory handle](crate::server::Handle). +pub struct Handle { + /// Memory handle. + pub memory: ManagedMemoryHandle, + /// Memory offset in bytes. + pub offset_start: Option, + /// Memory offset in bytes. + pub offset_end: Option, + /// The stream where the data was created. + pub stream: StreamId, + /// Length of the underlying buffer ignoring offsets + pub(crate) size: u64, +} + +impl core::fmt::Debug for Handle { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Handle") + .field("id", &self.memory) + .field("offset_start", &self.offset_start) + .field("offset_end", &self.offset_end) + .field("stream", &self.stream) + .field("size", &self.size) + .finish() + } +} + +impl Clone for Handle { + fn clone(&self) -> Self { + Self { + memory: self.memory.clone(), + offset_start: self.offset_start, + offset_end: self.offset_end, + stream: self.stream, + size: self.size, + } + } +} + +impl Handle { + /// Creates a new handle of the given size. + pub fn from_memory(id: ManagedMemoryHandle, stream: StreamId, size: u64) -> Self { + Self { + memory: id, + offset_start: None, + offset_end: None, + stream, + size, + } + } + /// Creates a new handle of the given size. + pub fn new(stream: StreamId, size: u64) -> Self { + Self { + memory: ManagedMemoryHandle::new(), + offset_start: None, + offset_end: None, + stream, + size, + } + } + /// Checks whether the handle can be mutated in-place without affecting other computation. + pub fn can_mut(&self) -> bool { + self.memory.can_mut() + } + + /// Returns the [`Binding`] corresponding to the current handle. + pub fn binding(self) -> Binding { + Binding { + memory: self.memory.binding(), + offset_start: self.offset_start, + offset_end: self.offset_end, + stream: self.stream, + size: self.size, + } + } + + /// Add to the current offset in bytes. + pub fn offset_start(mut self, offset: u64) -> Self { + if let Some(val) = &mut self.offset_start { + *val += offset; + } else { + self.offset_start = Some(offset); + } + + self + } + /// Add to the current offset in bytes. + pub fn offset_end(mut self, offset: u64) -> Self { + if let Some(val) = &mut self.offset_end { + *val += offset; + } else { + self.offset_end = Some(offset); + } + + self + } + + /// Convert the [handle](Handle) into a [binding](Binding) with shape and stride metadata. + pub fn copy_descriptor( + self, + shape: Shape, + strides: Strides, + elem_size: usize, + ) -> CopyDescriptor { + CopyDescriptor { + shape, + strides, + elem_size, + handle: self.binding(), + } + } + /// Get the size of the handle, in bytes, accounting for offsets + pub fn size_in_used(&self) -> u64 { + self.size - self.offset_start.unwrap_or(0) - self.offset_end.unwrap_or(0) + } + /// Get the total size of the handle, in bytes. + pub fn size(&self) -> u64 { + self.size + } +} + +/// A binding represents a [Handle] that is bound to managed memory. +/// +/// The memory used is known by the compute server. +/// A binding is only valid after being initlized with [`super::ComputeServer::initialize_bindings`] +/// +/// # Notes +/// +/// A binding is detached from a [`Handle`], meaning that is won't affect [`Handle::can_mut`]. +#[derive(Clone, Debug)] +pub struct Binding { + /// The id of the handle the binding is bound to. + pub memory: ManagedMemoryBinding, + /// Memory offset in bytes. + pub offset_start: Option, + /// Memory offset in bytes. + pub offset_end: Option, + /// The stream where the data was created. + pub stream: StreamId, + /// Length of the underlying buffer ignoring offsets + pub size: u64, +} + +impl Binding { + /// Get the size of the handle, in bytes, accounting for offsets + pub fn size_in_used(&self) -> u64 { + self.size - self.offset_start.unwrap_or(0) - self.offset_end.unwrap_or(0) + } + /// Get the total size of the handle, in bytes. + pub fn size(&self) -> u64 { + self.size + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/server/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/src/server/mod.rs new file mode 100644 index 00000000..3dad0a39 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/server/mod.rs @@ -0,0 +1,5 @@ +mod base; +mod handle; + +pub use base::*; +pub use handle::*; diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/storage/base.rs b/third_party/cubecl-runtime-0.10.0-patched/src/storage/base.rs new file mode 100644 index 00000000..a99ea373 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/storage/base.rs @@ -0,0 +1,120 @@ +use crate::{memory_management::ManagedMemoryBinding, server::IoError, storage_id_type}; +use core::fmt::Debug; + +// This ID is used to map a handle to its actual data. +storage_id_type!(StorageId); + +impl core::fmt::Display for StorageId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_fmt(format_args!("StorageId({})", self.value)) + } +} + +/// Defines if data uses a full memory chunk or a slice of it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StorageUtilization { + /// The offset in bytes from the chunk start. + pub offset: u64, + /// The size of the slice in bytes. + pub size: u64, +} + +/// Contains the [storage id](StorageId) of a resource and the way it is used. +#[derive(new, Clone, Debug)] +pub struct StorageHandle { + /// Storage id. + pub id: StorageId, + /// How the storage is used. + pub utilization: StorageUtilization, +} + +impl StorageHandle { + /// Returns the size the handle is pointing to in memory. + /// + /// # Notes + /// + /// The result considers the offset. + pub fn size(&self) -> u64 { + self.utilization.size + } + + /// Returns the offset of the handle. + pub fn offset(&self) -> u64 { + self.utilization.offset + } + + /// Increase the current offset with the given value in bytes. + pub fn offset_start(&self, offset_bytes: u64) -> Self { + let utilization = StorageUtilization { + offset: self.offset() + offset_bytes, + size: self.size() - offset_bytes, + }; + + Self { + id: self.id, + utilization, + } + } + + /// Reduce the size of the memory handle.. + pub fn offset_end(&self, offset_bytes: u64) -> Self { + let utilization = StorageUtilization { + offset: self.offset(), + size: self.size() - offset_bytes, + }; + + Self { + id: self.id, + utilization, + } + } +} + +/// Storage types are responsible for allocating and deallocating memory. +pub trait ComputeStorage: Send { + /// The resource associated type determines the way data is implemented and how + /// it can be accessed by kernels. + type Resource: Send; + + /// The alignment memory is allocated with in this storage. + fn alignment(&self) -> usize; + + /// Returns the underlying resource for a specified storage handle + fn get(&mut self, handle: &StorageHandle) -> Self::Resource; + + /// Allocates `size` units of memory and returns a handle to it + fn alloc(&mut self, size: u64) -> Result; + + /// Deallocates the memory pointed by the given storage id. + /// + /// These deallocations might need to be flushed with [`Self::flush`]. + fn dealloc(&mut self, id: StorageId); + + /// Flush deallocations when required. + fn flush(&mut self); +} + +/// Access to the underlying resource. +#[derive(new, Debug)] +pub struct ManagedResource { + // This handle is here just to keep the underlying allocation alive. + // If the underlying allocation becomes invalid, someone else might + // allocate into this resource which could lead to bad behaviour. + #[allow(unused)] + binding: ManagedMemoryBinding, + resource: Resource, +} + +impl ManagedResource { + /// access the underlying resource. + /// + /// # Note + /// + /// The resource might be bigger than the part required. + /// (e.g. a big buffer where the handle only refers to a slice of it). + /// Only the part required by the handle is guaranteed to remain, + /// other parts of this resource *will* be re-used. + pub fn resource(&self) -> &Resource { + &self.resource + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/storage/bytes_cpu.rs b/third_party/cubecl-runtime-0.10.0-patched/src/storage/bytes_cpu.rs new file mode 100644 index 00000000..f456822d --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/storage/bytes_cpu.rs @@ -0,0 +1,246 @@ +use crate::server::IoError; + +use super::{ComputeStorage, StorageHandle, StorageId, StorageUtilization}; +use alloc::alloc::{Layout, alloc_zeroed, dealloc}; +use cubecl_common::backtrace::BackTrace; +use hashbrown::HashMap; + +/// The bytes storage maps ids to pointers of bytes in a contiguous layout. +#[derive(Default)] +pub struct BytesStorage { + memory: HashMap, +} + +impl core::fmt::Debug for BytesStorage { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("BytesStorage") + } +} + +/// Can send to other threads. +unsafe impl Send for BytesStorage {} +unsafe impl Send for BytesResource {} + +/// This struct is a pointer to a memory chunk or slice. +#[derive(Debug)] +pub struct BytesResource { + ptr: *mut u8, + utilization: StorageUtilization, +} + +/// This struct refers to a specific (contiguous) layout of bytes. +struct AllocatedBytes { + ptr: *mut u8, + layout: Layout, +} + +impl BytesResource { + /// Returns a mutable pointer to the start of the resource and its length. + pub fn get_write_ptr_and_length(&self) -> (*mut u8, usize) { + ( + // SAFETY: + // - The offset is created to be within the bounds of the allocation. + unsafe { self.ptr.add(self.utilization.offset as usize) }, + self.utilization.size as usize, + ) + } + + /// Returns the resource as a mutable slice of bytes. + /// + /// The lifetime `'a` is the lifetime of the underlying `BytesStorage` allocation, + /// not of `self`. The `&mut self` ensures only one mutable slice is created per + /// resource. Multiple resources may point to non-overlapping regions of the same + /// allocation (like `split_at_mut`); each resource owns its region exclusively. + pub fn write<'a>(&mut self) -> &'a mut [u8] { + let (ptr, len) = self.get_write_ptr_and_length(); + + // SAFETY: + // - ptr is non-null and aligned (from BytesStorage::alloc). + // - The region [ptr..ptr+len) is within a single allocation. + // - Memory is initialized (BytesStorage uses alloc_zeroed). + // - `&mut self` ensures exclusive access to this resource's region. + // - `StorageHandle` assigns non-overlapping regions per resource. + // - Systems must make sure this is the only `BytesResource` with an outstanding mutable borrow. + unsafe { core::slice::from_raw_parts_mut(ptr, len) } + } + + /// Returns the resource as an immutable slice of bytes. + /// + /// See [`write`](Self::write) for lifetime and safety notes. + pub fn read<'a>(&self) -> &'a [u8] { + let (ptr, len) = self.get_write_ptr_and_length(); + + // SAFETY: + // - ptr is non-null and aligned (from BytesStorage::alloc). + // - The region [ptr..ptr+len) is within a single allocation. + // - Memory is initialized (BytesStorage uses alloc_zeroed). + unsafe { core::slice::from_raw_parts(ptr, len) } + } +} + +impl ComputeStorage for BytesStorage { + type Resource = BytesResource; + + fn alignment(&self) -> usize { + 4 + } + + fn get(&mut self, handle: &StorageHandle) -> Self::Resource { + let allocated_bytes = self.memory.get(&handle.id).unwrap(); + + BytesResource { + ptr: allocated_bytes.ptr, + utilization: handle.utilization.clone(), + } + } + + #[cfg_attr( + feature = "tracing", + tracing::instrument(level = "trace", skip(self, size)) + )] + fn alloc(&mut self, size: u64) -> Result { + let id = StorageId::new(); + let handle = StorageHandle { + id, + utilization: StorageUtilization { offset: 0, size }, + }; + + if size == 0 { + // Zero-size allocations are valid handles but don't need real memory. + let memory = AllocatedBytes { + ptr: core::ptr::NonNull::dangling().as_ptr(), + layout: Layout::new::<()>(), + }; + self.memory.insert(id, memory); + } else { + unsafe { + let layout = Layout::array::(size as usize).unwrap(); + + // We allocate zeroed memory since we expose it as &[u8] / &mut [u8] + // which requires initialization. + let ptr = alloc_zeroed(layout); + if ptr.is_null() { + return Err(IoError::BufferTooBig { + size, + backtrace: BackTrace::capture(), + }); + } + let memory = AllocatedBytes { ptr, layout }; + self.memory.insert(id, memory); + } + } + + Ok(handle) + } + + #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))] + fn dealloc(&mut self, id: StorageId) { + if let Some(memory) = self.memory.remove(&id) + && memory.layout.size() > 0 + { + unsafe { + dealloc(memory.ptr, memory.layout); + } + } + } + + fn flush(&mut self) { + // We don't wait for dealloc. + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test_log::test] + fn test_can_alloc_and_dealloc() { + let mut storage = BytesStorage::default(); + let handle_1 = storage.alloc(64).unwrap(); + + assert_eq!(handle_1.size(), 64); + storage.dealloc(handle_1.id); + } + + #[test_log::test] + fn test_slices() { + let mut storage = BytesStorage::default(); + let handle_1 = storage.alloc(64).unwrap(); + let handle_2 = StorageHandle::new( + handle_1.id, + StorageUtilization { + offset: 24, + size: 8, + }, + ); + + storage + .get(&handle_1) + .write() + .iter_mut() + .enumerate() + .for_each(|(i, b)| { + *b = i as u8; + }); + + let bytes = storage.get(&handle_2).read().to_vec(); + + storage.dealloc(handle_1.id); + assert_eq!(bytes, &[24, 25, 26, 27, 28, 29, 30, 31]); + } + + /// Miri catches: "reading memory, but memory is uninitialized" + #[test_log::test] + fn test_read_after_alloc_without_write() { + let mut storage = BytesStorage::default(); + let handle = storage.alloc(16).unwrap(); + let resource = storage.get(&handle); + assert!(resource.read().iter().all(|&b| b == 0)); + storage.dealloc(handle.id); + } + + /// Miri catches: "creating allocation with size 0" + #[test_log::test] + fn test_zero_size_alloc_and_dealloc() { + let mut storage = BytesStorage::default(); + let handle = storage.alloc(0).unwrap(); + assert_eq!(handle.size(), 0); + storage.dealloc(handle.id); + } + + #[test_log::test] + fn test_alloc_dealloc_realloc() { + let mut storage = BytesStorage::default(); + let h1 = storage.alloc(32).unwrap(); + storage.get(&h1).write()[0] = 0xAA; + storage.dealloc(h1.id); + let h2 = storage.alloc(32).unwrap(); + storage.dealloc(h2.id); + } + + #[test_log::test] + fn test_multiple_non_overlapping_regions() { + let mut storage = BytesStorage::default(); + let base = storage.alloc(64).unwrap(); + + let regions: alloc::vec::Vec<_> = (0..4) + .map(|i| { + StorageHandle::new( + base.id, + StorageUtilization { + offset: i * 16, + size: 16, + }, + ) + }) + .collect(); + + for (i, region) in regions.iter().enumerate() { + storage.get(region).write().fill(i as u8); + } + for (i, region) in regions.iter().enumerate() { + assert!(storage.get(region).read().iter().all(|&b| b == i as u8)); + } + storage.dealloc(base.id); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/storage/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/src/storage/mod.rs new file mode 100644 index 00000000..0bf21bd3 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/storage/mod.rs @@ -0,0 +1,8 @@ +mod base; + +pub use base::*; + +#[cfg(feature = "storage-bytes")] +mod bytes_cpu; +#[cfg(feature = "storage-bytes")] +pub use bytes_cpu::*; diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/stream/base.rs b/third_party/cubecl-runtime-0.10.0-patched/src/stream/base.rs new file mode 100644 index 00000000..829dc1b8 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/stream/base.rs @@ -0,0 +1,103 @@ +use alloc::vec::Vec; +use cubecl_common::stream_id::StreamId; + +/// Trait for creating streams, used by the stream pool to generate streams as needed. +pub trait StreamFactory { + /// The type of stream produced by this factory. + type Stream; + /// Creates a new stream instance. + fn create(&mut self) -> Self::Stream; +} + +/// Represents a pool of streams, managing a collection of streams created by a factory. +#[derive(Debug)] +pub struct StreamPool { + /// Vector storing optional streams, where None indicates an uninitialized stream. + streams: Vec>, + /// The factory used to create new streams when needed. + factory: F, + /// Maximum number of regular streams (excludes special streams). + max_streams: usize, +} + +impl StreamPool { + /// Creates a new stream pool with the given backend factory and capacity constraints. + pub fn new(backend: F, max_streams: u8, num_special: u8) -> Self { + // Initialize a vector with capacity for regular and special streams. + let mut streams = Vec::with_capacity(max_streams as usize); + // Pre-populate the vector with None to reserve space for all streams. + for _ in 0..(max_streams.saturating_add(num_special)) { + streams.push(None); + } + + Self { + streams, + factory: backend, + max_streams: max_streams as usize, + } + } + + /// Retrieves a mutable reference to a stream for a given stream ID. + pub fn get_mut(&mut self, stream_id: &StreamId) -> &mut F::Stream { + // Calculate the index for the stream ID. + let index = self.stream_index(stream_id); + + // Use unsafe method to retrieve the stream, assuming the index is valid. + // + // # Safety + // + // * The `stream_index` function ensures the index is within bounds. + unsafe { self.get_mut_index(index) } + } + + /// Retrieves a mutable reference to a stream at the specified index, initializing it if needed. + /// + /// # Safety + /// + /// * Caller must ensure the index is valid (less than `max_streams + num_special`). + /// * Lifetimes still follow the Rust rules. + pub unsafe fn get_mut_index(&mut self, index: usize) -> &mut F::Stream { + unsafe { + // Access the stream entry without bounds checking for performance. + let entry = self.streams.get_unchecked_mut(index); + match entry { + // If the stream exists, return it. + Some(val) => val, + // If the stream is None, create a new one using the factory. + None => { + let stream = self.factory.create(); + // Store the new stream in the vector. + *entry = Some(stream); + + // Re-access the entry, which is now guaranteed to be Some. + match entry { + Some(val) => val, + // Unreachable because we just set it to Some. + None => unreachable!(), + } + } + } + } + } + + /// Retrieves a mutable reference to a special stream at the given index. + /// + /// # Safety + /// + /// * Caller must ensure the index corresponds to a valid special stream. + /// * Lifetimes still follow the Rust rules. + pub unsafe fn get_special(&mut self, index: u8) -> &mut F::Stream { + // Calculate the index for the special stream (offset by max_streams). + unsafe { self.get_mut_index(self.max_streams + index as usize) } + } + + /// Calculates the index for a given stream ID, mapping it to the pool's capacity. + pub fn stream_index(&mut self, id: &StreamId) -> usize { + stream_index(id, self.max_streams) + } +} + +/// Maps a stream ID to an index within the pool's capacity using modulo arithmetic. +pub fn stream_index(stream_id: &StreamId, max_streams: usize) -> usize { + stream_id.value as usize % max_streams +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/stream/event.rs b/third_party/cubecl-runtime-0.10.0-patched/src/stream/event.rs new file mode 100644 index 00000000..8529f18f --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/stream/event.rs @@ -0,0 +1,529 @@ +use crate::{ + config::streaming::StreamingLogLevel, + logging::ServerLogger, + memory_management::ManagedMemoryId, + server::{Binding, ServerError}, + stream::{StreamFactory, StreamPool}, +}; +use core::any::Any; +use cubecl_common::{backtrace::BackTrace, stream_id::StreamId}; +use hashbrown::HashMap; +use std::{ + boxed::Box, + format, + sync::{Arc, mpsc::SyncSender}, + vec::Vec, +}; + +/// Trait defining the backend operations for managing streams and events. +/// +/// This trait provides the necessary methods for initializing streams, flushing them to create events, +/// and waiting on events for synchronization purposes. +pub trait EventStreamBackend: 'static { + /// The type representing a stream in this backend. + type Stream: core::fmt::Debug; + /// The type representing an event in this backend. + type Event: Send + 'static; + + /// Initializes and returns a new stream associated with the given stream ID. + fn create_stream(&self) -> Self::Stream; + /// Returns the cursor of the given handle on the given stream. + fn handle_cursor(stream: &Self::Stream, handle: &Binding) -> u64; + /// Returns whether the stream can access new tasks. + fn is_healthy(stream: &Self::Stream) -> bool; + + /// Flushes the given stream, ensuring all pending operations are submitted, and returns an event + /// that can be used for synchronization. + fn flush(stream: &mut Self::Stream) -> Self::Event; + /// Makes the stream wait for the specified event to complete before proceeding with further operations. + fn wait_event(stream: &mut Self::Stream, event: Self::Event); + /// Wait for the given event synching the CPU. + fn wait_event_sync(event: Self::Event) -> Result<(), ServerError>; +} + +/// Manages multiple streams with synchronization logic based on shared bindings. +/// +/// This struct handles the creation and alignment of streams to ensure proper synchronization +/// when bindings (e.g., buffers) are shared across different streams. +#[derive(Debug)] +pub struct MultiStream { + /// The map of stream IDs to their corresponding stream wrappers. + streams: StreamPool>, + /// The logger used by the server. + pub logger: Arc, + max_streams: usize, + gc: GcThread, + shared_bindings_pool: Vec<(ManagedMemoryId, StreamId, u64)>, +} + +/// A wrapper around a backend stream that includes synchronization metadata. +/// +/// This includes the stream itself, a map of last synchronized cursors from other streams, +/// and the current cursor position for this stream. +pub(crate) struct StreamWrapper { + /// The underlying backend stream. + stream: B::Stream, + /// The current cursor position, representing the logical progress or version of operations on this stream. + cursor: u64, + /// A map tracking the last synchronized cursor positions from other streams. + last_synced: HashMap, +} + +/// Streams that are synchronized correctly after a [`MultiStream::resolve`] is called. +pub struct ResolvedStreams<'a, B: EventStreamBackend> { + /// The cursor on the current stream. + /// + /// This cursor should be use for new allocations happening on the current stream. + pub cursor: u64, + streams: &'a mut StreamPool>, + analysis: SharedBindingAnalysis, + gc: &'a GcThread, + /// The current stream where new tasks can be sent safely. + pub current: StreamId, +} + +#[derive(Debug)] +/// A task to be enqueue on the gc stream that will be clearned after an event is reached. +pub struct GcTask { + to_drop: Box, + /// The event to sync making sure the bindings in the batch are ready to be reused by other streams. + event: B::Event, +} + +impl GcTask { + /// Creates a new task that will be clearned when the event is reached. + pub fn new(to_drop: T, event: B::Event) -> Self { + Self { + to_drop: Box::new(to_drop), + event, + } + } +} + +#[derive(Debug)] +struct EventStreamBackendWrapper { + backend: B, +} + +impl StreamFactory for EventStreamBackendWrapper { + type Stream = StreamWrapper; + + fn create(&mut self) -> Self::Stream { + StreamWrapper { + stream: self.backend.create_stream(), + cursor: 0, + last_synced: Default::default(), + } + } +} + +#[derive(Debug)] +struct GcThread { + sender: SyncSender>, +} + +impl GcThread { + fn new() -> GcThread { + let (sender, recv) = std::sync::mpsc::sync_channel::>(8); + + std::thread::spawn(move || { + while let Ok(event) = recv.recv() { + B::wait_event_sync(event.event).unwrap(); + core::mem::drop(event.to_drop); + } + }); + + GcThread { sender } + } + fn register(&self, task: GcTask) { + self.sender.send(task).unwrap() + } +} + +fn stream_index(stream_id: &StreamId, max_streams: usize) -> usize { + stream_id.value as usize % max_streams +} + +impl<'a, B: EventStreamBackend> ResolvedStreams<'a, B> { + /// Get the stream associated to the given [`stream_id`](StreamId). + pub fn get(&mut self, stream_id: &StreamId) -> &mut B::Stream { + let stream = self.streams.get_mut(stream_id); + &mut stream.stream + } + + /// Get the stream associated to the [current `stream_id`](StreamId). + pub fn current(&mut self) -> &mut B::Stream { + let stream = self.streams.get_mut(&self.current); + &mut stream.stream + } + + /// Enqueue a task to be cleaned. + pub fn gc(&mut self, gc: GcTask) { + self.gc.sender.send(gc).unwrap(); + } +} + +impl<'a, B: EventStreamBackend> Drop for ResolvedStreams<'a, B> { + fn drop(&mut self) { + if self.analysis.slices.is_empty() { + return; + } + + let stream = self.streams.get_mut(&self.current); + let event_origin = B::flush(&mut stream.stream); + + let stream_gc = &mut unsafe { self.streams.get_special(0) }.stream; + B::wait_event(stream_gc, event_origin); + let event = B::flush(stream_gc); + + let mut ids = Vec::new(); + self.analysis + .slices + .drain() + .for_each(|item| ids.extend(item.1)); + + self.gc.register(GcTask::new(ids, event)); + } +} + +impl MultiStream { + /// Creates an empty multi-stream. + pub fn new(logger: Arc, backend: B, max_streams: u8) -> Self { + let wrapper = EventStreamBackendWrapper { backend }; + Self { + streams: StreamPool::new(wrapper, max_streams, 1), + logger, + max_streams: max_streams as usize, + gc: GcThread::new(), + shared_bindings_pool: Vec::new(), + } + } + + /// Enqueue a task to be cleaned. + pub fn gc(&mut self, gc: GcTask) { + self.gc.sender.send(gc).unwrap(); + } + + /// Resolves and returns a mutable reference to the stream for the given ID, performing any necessary + /// alignment based on the provided bindings. + /// + /// This method ensures that the stream is synchronized with any shared bindings from other streams + /// before returning the stream reference. + pub fn resolve<'a>( + &mut self, + stream_id: StreamId, + handles: impl Iterator, + enforce_healthy: bool, + ) -> Result, ServerError> { + let analysis = self.align_streams(stream_id, handles); + + let stream = self.streams.get_mut(&stream_id); + stream.cursor += 1; + + if enforce_healthy && !B::is_healthy(&stream.stream) { + return Err(ServerError::Generic { + reason: "Can't resolve the stream since it is currently in an error state".into(), + backtrace: BackTrace::capture(), + }); + } + + Ok(ResolvedStreams { + cursor: stream.cursor, + streams: &mut self.streams, + current: stream_id, + analysis, + gc: &self.gc, + }) + } + + /// Aligns the target stream with other streams based on shared bindings. + /// + /// This initializes the stream if it doesn't exist, analyzes which originating streams need flushing + /// for synchronization, flushes them, and waits on the events in the target stream. + fn align_streams<'a>( + &mut self, + stream_id: StreamId, + handles: impl Iterator, + ) -> SharedBindingAnalysis { + let analysis = self.update_shared_bindings(stream_id, handles); + + self.apply_analysis(stream_id, analysis) + } + + /// Update and analyzes the bindings to determine which streams need alignment (flushing and waiting). + /// + /// This checks for shared bindings from other streams and determines if synchronization is needed + /// based on cursor positions. + pub(crate) fn update_shared_bindings<'a>( + &mut self, + stream_id: StreamId, + handles: impl Iterator, + ) -> SharedBindingAnalysis { + // We reset the memory pool for the info. + self.shared_bindings_pool.clear(); + + for handle in handles { + let index = stream_index(&handle.stream, self.max_streams); + let stream = unsafe { self.streams.get_mut_index(index) }; + let cursor_handle = B::handle_cursor(&stream.stream, handle); + + // We only add the info to be consider if the handle stream is different from the current + // stream. + if handle.stream != stream_id { + self.shared_bindings_pool.push(( + handle.memory.descriptor().id, + handle.stream, + cursor_handle, + )); + } + } + + let mut analysis = SharedBindingAnalysis::default(); + let current = self.streams.get_mut(&stream_id); + + for (handle_id, stream, cursor) in self.shared_bindings_pool.iter() { + let index = stream_index(stream, self.max_streams); + + if let Some(last_synced) = current.last_synced.get(&index) { + if last_synced < cursor { + self.logger.log_streaming( + |level| matches!(level, StreamingLogLevel::Full), + || { + format!( + "Binding on {} is shared on {} since it's not sync {} < {}", + stream, stream_id, last_synced, cursor + ) + }, + ); + analysis.shared(*handle_id, index); + } + } else { + self.logger.log_streaming( + |level| matches!(level, StreamingLogLevel::Full), + || { + format!( + "Binding on {} is shared on {} since it was never synced.", + stream, stream_id, + ) + }, + ); + analysis.shared(*handle_id, index); + } + } + + analysis + } + + pub(crate) fn apply_analysis( + &mut self, + stream_id: StreamId, + analysis: SharedBindingAnalysis, + ) -> SharedBindingAnalysis { + if analysis.slices.is_empty() { + return analysis; + } + + let mut events = Vec::with_capacity(analysis.slices.len()); + + unsafe { + for origin in analysis.slices.keys() { + let stream = self.streams.get_mut_index(*origin); + let event = B::flush(&mut stream.stream); + + events.push(((origin, stream.cursor), event)); + } + } + + let stream = self.streams.get_mut(&stream_id); + + for ((stream_origin, cursor_origin), event) in events { + stream.last_synced.insert(*stream_origin, cursor_origin); + + self.logger.log_streaming( + |level| !matches!(level, StreamingLogLevel::Disabled), + || format!("Waiting on {stream_origin} from {stream_id}",), + ); + + B::wait_event(&mut stream.stream, event); + } + + analysis + } +} + +impl core::fmt::Debug for StreamWrapper { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("StreamWrapper") + .field("stream", &self.stream) + .field("cursor", &self.cursor) + .field("last_synced", &self.last_synced) + .finish() + } +} + +#[derive(Default, Debug, PartialEq, Eq)] +pub(crate) struct SharedBindingAnalysis { + slices: HashMap>, +} + +impl SharedBindingAnalysis { + fn shared(&mut self, id: ManagedMemoryId, index: usize) { + match self.slices.get_mut(&index) { + Some(bindings) => bindings.push(id), + None => { + self.slices.insert(index, alloc::vec![id]); + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::server::Handle; + + use super::*; + + const MAX_STREAMS: u8 = 4; + + #[test_log::test] + fn test_analysis_shared_bindings_1() { + let logger = Arc::new(ServerLogger::default()); + let stream_1 = StreamId { value: 1 }; + let stream_2 = StreamId { value: 2 }; + + let binding_1 = handle(stream_1); + let binding_2 = handle(stream_2); + + let mut ms = MultiStream::new(logger, TestBackend, MAX_STREAMS); + ms.resolve(stream_1, [].into_iter(), false).unwrap(); + ms.resolve(stream_2, [].into_iter(), false).unwrap(); + + let analysis = ms.update_shared_bindings(stream_1, [&binding_1, &binding_2].into_iter()); + + let mut expected = SharedBindingAnalysis::default(); + expected.shared( + binding_2.memory.descriptor().id, + ms.streams.stream_index(&binding_2.stream), + ); + + assert_eq!(analysis, expected); + } + + #[test_log::test] + fn test_analysis_shared_bindings_2() { + let logger = Arc::new(ServerLogger::default()); + let stream_1 = StreamId { value: 1 }; + let stream_2 = StreamId { value: 2 }; + + let binding_1 = handle(stream_1); + let binding_2 = handle(stream_2); + let binding_3 = handle(stream_1); + + let mut ms = MultiStream::new(logger, TestBackend, 4); + ms.resolve(stream_1, [].into_iter(), false).unwrap(); + ms.resolve(stream_2, [].into_iter(), false).unwrap(); + + let analysis = + ms.update_shared_bindings(stream_1, [&binding_1, &binding_2, &binding_3].into_iter()); + + let mut expected = SharedBindingAnalysis::default(); + expected.shared( + binding_2.memory.descriptor().id, + ms.streams.stream_index(&binding_2.stream), + ); + + assert_eq!(analysis, expected); + } + + #[test_log::test] + fn test_analysis_no_shared() { + let logger = Arc::new(ServerLogger::default()); + let stream_1 = StreamId { value: 1 }; + let stream_2 = StreamId { value: 2 }; + + let binding_1 = handle(stream_1); + let binding_2 = handle(stream_1); + let binding_3 = handle(stream_1); + + let mut ms = MultiStream::new(logger, TestBackend, MAX_STREAMS); + ms.resolve(stream_1, [].into_iter(), false).unwrap(); + ms.resolve(stream_2, [].into_iter(), false).unwrap(); + + let analysis = + ms.update_shared_bindings(stream_1, [&binding_1, &binding_2, &binding_3].into_iter()); + + let expected = SharedBindingAnalysis::default(); + + assert_eq!(analysis, expected); + } + + #[test_log::test] + fn test_state() { + let logger = Arc::new(ServerLogger::default()); + let stream_1 = StreamId { value: 1 }; + let stream_2 = StreamId { value: 2 }; + + let binding_1 = handle(stream_1); + let binding_2 = handle(stream_2); + let binding_3 = handle(stream_1); + + let mut ms = MultiStream::new(logger, TestBackend, MAX_STREAMS); + ms.resolve(stream_1, [].into_iter(), false).unwrap(); + ms.resolve(stream_2, [].into_iter(), false).unwrap(); + + ms.resolve( + stream_1, + [&binding_1, &binding_2, &binding_3].into_iter(), + false, + ) + .unwrap(); + + let stream1 = ms.streams.get_mut(&stream_1); + let index_2 = stream_index(&stream_2, MAX_STREAMS as usize); + assert_eq!(stream1.last_synced.get(&index_2), Some(&1)); + assert_eq!(stream1.cursor, 2); + + let stream2 = ms.streams.get_mut(&stream_2); + assert!(stream2.last_synced.is_empty()); + assert_eq!(stream2.cursor, 1); + } + + fn handle(stream: StreamId) -> Binding { + Handle::new(stream, 10).binding() + } + + struct TestBackend; + + #[derive(Debug)] + struct TestStream {} + + #[derive(Debug)] + struct TestEvent {} + + impl EventStreamBackend for TestBackend { + type Stream = TestStream; + type Event = TestEvent; + + fn create_stream(&self) -> Self::Stream { + TestStream {} + } + + fn flush(_stream: &mut Self::Stream) -> Self::Event { + TestEvent {} + } + + fn wait_event(_stream: &mut Self::Stream, _event: Self::Event) {} + + fn wait_event_sync(_event: Self::Event) -> Result<(), ServerError> { + Ok(()) + } + + fn handle_cursor(_stream: &Self::Stream, _handle: &Binding) -> u64 { + 0 + } + + fn is_healthy(_stream: &Self::Stream) -> bool { + true + } + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/stream/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/src/stream/mod.rs new file mode 100644 index 00000000..2dafd8c6 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/stream/mod.rs @@ -0,0 +1,12 @@ +/// Scheduler based multi-stream support. +pub mod scheduler; + +mod base; + +#[cfg(multi_threading)] +mod event; + +pub use base::*; + +#[cfg(multi_threading)] +pub use event::*; diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/stream/scheduler.rs b/third_party/cubecl-runtime-0.10.0-patched/src/stream/scheduler.rs new file mode 100644 index 00000000..c8a9875c --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/stream/scheduler.rs @@ -0,0 +1,266 @@ +use crate::{ + config::streaming::StreamingLogLevel, + logging::ServerLogger, + stream::{StreamFactory, StreamPool}, +}; +use alloc::{format, sync::Arc, vec, vec::Vec}; +use cubecl_common::stream_id::StreamId; + +/// Defines a trait for a scheduler stream backend, specifying the types and behavior for task scheduling. +pub trait SchedulerStreamBackend { + /// Type representing a task. + type Task: core::fmt::Debug; + /// Type representing a stream. + type Stream: core::fmt::Debug; + /// Type for the stream factory, which creates streams of type `Self::Stream`. + type Factory: StreamFactory; + + /// Enqueues a task onto a given stream for execution. + fn enqueue(task: Self::Task, stream: &mut Self::Stream); + /// Flush the inner stream queue to ensure ordering between different streams. + fn flush(stream: &mut Self::Stream); + /// Returns a mutable reference to the stream factory. + fn factory(&mut self) -> &mut Self::Factory; +} + +/// Represents a multi-stream scheduler that manages task execution across multiple streams. +#[derive(Debug)] +pub struct SchedulerMultiStream { + /// Pool of streams managed by the scheduler. + pool: StreamPool>, + /// Strategy for scheduling tasks (e.g., Interleave or Sequential). + strategy: SchedulerStrategy, + /// Maximum number of tasks allowed per stream before execution is triggered. + max_tasks: usize, + /// Server logger. + pub logger: Arc, +} + +/// Defines the scheduling strategy for task execution. +#[derive(Debug)] +pub enum SchedulerStrategy { + /// Tasks from different streams are interleaved during execution. + Interleave, + /// Tasks from each stream are executed sequentially. + Sequential, +} + +/// Represents a single stream that holds tasks and a backend stream. +#[derive(Debug)] +pub struct Stream { + /// List of tasks queued for execution in this stream. + tasks: Vec, + /// The backend stream used for task execution. + stream: B::Stream, +} + +impl Stream { + /// Flushes all tasks from the stream, returning them and clearing the internal task list. + fn flush(&mut self) -> Vec { + let mut returned = Vec::with_capacity(self.tasks.capacity()); + core::mem::swap(&mut returned, &mut self.tasks); + returned + } +} + +#[derive(Debug)] +struct SchedulerPoolMarker { + backend: B, +} + +impl StreamFactory for SchedulerPoolMarker { + // The type of stream produced by this factory. + type Stream = Stream; + + // Creates a new stream with an empty task list and a backend stream. + fn create(&mut self) -> Self::Stream { + Stream { + tasks: Vec::new(), + // Uses the backend's factory to create a new stream. + stream: self.backend.factory().create(), + } + } +} + +/// Options for configuring a `SchedulerMultiStream`. +#[derive(Debug)] +pub struct SchedulerMultiStreamOptions { + /// Maximum number of streams allowed in the pool. + pub max_streams: u8, + /// Maximum number of tasks per stream before execution is triggered. + pub max_tasks: usize, + /// The scheduling strategy to use. + pub strategy: SchedulerStrategy, +} + +impl SchedulerMultiStream { + /// Creates a new `SchedulerMultiStream` with the given backend and options. + pub fn new( + logger: Arc, + backend: B, + options: SchedulerMultiStreamOptions, + ) -> Self { + Self { + pool: StreamPool::new(SchedulerPoolMarker { backend }, options.max_streams, 0), + max_tasks: options.max_tasks, + strategy: options.strategy, + logger, + } + } + + /// Returns a mutable reference to the backend stream for a given stream ID. + pub fn stream(&mut self, stream_id: &StreamId) -> &mut B::Stream { + let stream = self.pool.get_mut(stream_id); + &mut stream.stream + } + + /// Registers a task for execution on a specific stream, ensuring stream alignment. + pub fn register(&mut self, stream_id: StreamId, task: B::Task, args_streams: &[StreamId]) { + // Align streams to ensure dependencies are handled correctly. + self.align_streams(stream_id, args_streams); + + // Get the stream for the given stream ID and add the task to its queue. + let current = self.pool.get_mut(&stream_id); + current.tasks.push(task); + + // If the task queue exceeds the maximum, execute the stream. + if current.tasks.len() >= self.max_tasks { + self.execute_streams(vec![stream_id]); + } + } + + /// Aligns streams by flushing tasks from streams that conflict with the given bindings. + pub(crate) fn align_streams(&mut self, stream_id: StreamId, args_streams: &[StreamId]) { + let mut to_flush = Vec::new(); + // Get the index of the target stream. + let index = self.pool.stream_index(&stream_id); + + // Identify streams that need to be flushed due to conflicting bindings. + for arg_stream in args_streams { + let index_stream = self.pool.stream_index(arg_stream); + if index != index_stream { + to_flush.push(*arg_stream); + + self.logger.log_streaming( + |level| matches!(level, StreamingLogLevel::Full), + || format!("Binding on {} is shared on {}", arg_stream, stream_id), + ); + } + } + + // If no streams need flushing, return early. + if to_flush.is_empty() { + return; + } + + self.logger.log_streaming( + |level| !matches!(level, StreamingLogLevel::Disabled), + || { + format!( + "Flushing streams {to_flush:?} before registering more tasks on {stream_id}" + ) + }, + ); + // Execute the streams that need to be flushed. + self.execute_streams(to_flush); + } + + /// Executes tasks from the specified streams based on the scheduling strategy. + pub fn execute_streams(&mut self, stream_ids: Vec) { + let mut indices = Vec::with_capacity(stream_ids.len()); + + // Collect unique stream indices to avoid redundant processing. + for id in stream_ids { + let index = self.pool.stream_index(&id); + if !indices.contains(&index) { + indices.push(index); + } + } + + // Create schedules for each stream to be executed. + let mut schedules = Vec::new(); + for index in indices { + let stream = unsafe { self.pool.get_mut_index(index) }; // Note: `unsafe` usage assumes valid index. + let tasks = stream.flush(); + let num_tasks = tasks.len(); + + schedules.push(Schedule { + tasks: tasks.into_iter(), + num_tasks, + stream_index: index, + }); + } + + // If no schedules were created, return early. + if schedules.is_empty() { + return; + } + + // Execute schedules based on the configured strategy. + match self.strategy { + SchedulerStrategy::Interleave => self.execute_schedules_interleave(schedules), + SchedulerStrategy::Sequential => self.execute_schedules_sequence(schedules), + } + } + + /// Executes schedules sequentially, processing each stream's tasks in order. + fn execute_schedules_sequence(&mut self, schedules: Vec>) { + for schedule in schedules { + let stream = unsafe { self.pool.get_mut_index(schedule.stream_index) }; // Note: `unsafe` usage assumes valid index. + for task in schedule.tasks { + // Enqueue each task on the stream. + B::enqueue(task, &mut stream.stream); + } + + // Makes sure the tasks are ordered on the compute queue. + B::flush(&mut stream.stream); + } + } + + //// Executes schedules in an interleaved manner, alternating tasks from different streams. + /// + /// We chose the first stream as the one executing the tasks, ensuring proper ordering by + /// flushing all other streams first and flushing the execution stream at the end. + /// This way, we ensure that most tasks are actually interleaved on the real compute queue + /// shared across all streams. + fn execute_schedules_interleave(&mut self, mut schedules: Vec>) { + // Makes sure the tasks are ordered on the compute queue. + for schedule in schedules.iter_mut().skip(1) { + let stream = unsafe { self.pool.get_mut_index(schedule.stream_index) }; + B::flush(&mut stream.stream); + } + + let execution_index = schedules.first().expect("At least one stream").stream_index; + let stream = unsafe { self.pool.get_mut_index(execution_index) }; + + // Find the maximum number of tasks across all schedules. + let num_tasks_max = schedules + .iter() + .map(|s| s.num_tasks) + .max() + .expect("At least one schedule"); + + // Iterate through tasks, interleaving them across streams. + for _ in 0..num_tasks_max { + for schedule in schedules.iter_mut() { + // If there are tasks remaining in the schedule, enqueue the next one. + if let Some(task) = schedule.tasks.next() { + B::enqueue(task, &mut stream.stream); + } + } + } + + // Making sure all tasks are registered to the queue. + B::flush(&mut stream.stream); + } +} + +// Represents a schedule for executing tasks on a specific stream. +struct Schedule { + // Iterator over the tasks to be executed. + tasks: alloc::vec::IntoIter, + // Number of tasks in the schedule. + num_tasks: usize, + // Index of the stream in the pool. + stream_index: usize, +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/timestamp_profiler.rs b/third_party/cubecl-runtime-0.10.0-patched/src/timestamp_profiler.rs new file mode 100644 index 00000000..a843f522 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/timestamp_profiler.rs @@ -0,0 +1,59 @@ +use cubecl_common::{ + backtrace::BackTrace, + profile::{Instant, ProfileDuration}, +}; +use hashbrown::HashMap; + +use crate::server::{ProfileError, ProfilingToken}; + +#[derive(Default, Debug)] +/// A simple struct to keep track of timestamps for kernel execution. +/// This should be used for servers that do not have native device profiling. +pub struct TimestampProfiler { + state: HashMap, + counter: u64, +} + +#[derive(Debug)] +enum State { + Start(Instant), + Error(ProfileError), +} + +impl TimestampProfiler { + /// If there is some profiling registered. + pub fn is_empty(&self) -> bool { + self.state.is_empty() + } + /// Start measuring + pub fn start(&mut self) -> ProfilingToken { + let token = ProfilingToken { id: self.counter }; + self.counter += 1; + self.state.insert(token, State::Start(Instant::now())); + token + } + + /// Stop measuring + pub fn stop(&mut self, token: ProfilingToken) -> Result { + let state = self.state.remove(&token); + let start = match state { + Some(val) => match val { + State::Start(instant) => instant, + State::Error(profile_error) => return Err(profile_error), + }, + None => { + return Err(ProfileError::NotRegistered { + backtrace: BackTrace::capture(), + }); + } + }; + Ok(ProfileDuration::new_system_time(start, Instant::now())) + } + + /// Register an error during profiling. + pub fn error(&mut self, error: ProfileError) { + self.state + .iter_mut() + .for_each(|(_, state)| *state = State::Error(error.clone())); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tma.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tma.rs new file mode 100644 index 00000000..9dc39c0a --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tma.rs @@ -0,0 +1,130 @@ +use alloc::vec::Vec; +use cubecl_zspace::Shape; +#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))] +use serde::{Deserialize, Serialize}; + +/// Args for tiled tensor maps +#[cfg_attr( + any(target_os = "windows", target_os = "linux", target_os = "macos"), + derive(Serialize, Deserialize) +)] +#[derive(Hash, PartialEq, Eq, Clone, Debug, new)] +pub struct TiledArgs { + /// Tile size that's loaded from memory in each copy operation. Must have `rank` elements. + /// In matmul, for example, this might be `batch x m x k`, or whatever the stage size is. + /// If a dimension isn't present in the tile, it should just be set to `1`. + /// + /// For CUDA, this must be a power of two and `<= 256` on each dimension. + pub tile_size: Shape, +} + +/// Args for im2col tensor maps +#[derive(Hash, PartialEq, Eq, Clone, Debug, new)] +pub struct Im2colArgs { + /// Pixel box lower corner. This is the logical upper left corner in the input tensor, + /// when offset is 0. The length of this value should equal the *spatial* dimensions of + /// the input tensor (i.e. `h, w` for an NHWC tensor). Should normally be set to `-padding`. + pub pixel_box_lower_corner: Vec, + /// Pixel box top corner. This is the logical lower right corner in the input tensor, + /// when offset is 0. The length of this value should equal the *spatial* dimensions of + /// the input tensor (i.e. `h, w` for an NHWC tensor). Should normally be set to + /// `padding - kernel_size - 1` (where `kernel_size` accounts for dilation). This is not + /// equal to padding, it's equal to the bounding box for the *top left corner of the kernel*. + pub pixel_box_upper_corner: Vec, + /// Channels to load per pixel, should be a multiple or divisor of the matmul tile size. + /// This is not the total number of channels in the tensor, but only the number loaded in + /// each load. Must be <= 256 and aligned to 16 bytes. + pub channels_per_pixel: u32, + /// Pixels per column, equivalent to the `m`/`n` dimension of each tile in the matrix + /// multiplication. i.e. `NHW` for a 4D tensor. + /// Must be <= 256 and aligned to 16 bytes + pub pixels_per_column: u32, +} + +/// Args for im2col wide tensor maps +#[derive(Hash, PartialEq, Eq, Clone, Debug, new)] +pub struct Im2colWideArgs { + /// Pixel box lower corner width. TODO: How does this work? + pub pixel_box_lower_corner_width: i32, + /// Pixel box upper corner width. TODO: How does this work? + pub pixel_box_upper_corner_width: i32, + /// Channels per pixel + pub channels_per_pixel: u32, + /// Pixels per column + pub pixels_per_column: u32, +} + +/// Format of ``TensorMap`` +#[derive(Hash, PartialEq, Eq, Clone, Debug)] +pub enum TensorMapFormat { + /// Simple tiling + Tiled(TiledArgs), + /// Im2col indexing. + Im2col(Im2colArgs), + /// Wide im2col + Im2colWide(Im2colWideArgs), +} + +/// Interleave setting for ``TensorMap`` +#[derive(Default, Hash, PartialEq, Eq, Clone, Debug, Copy)] +pub enum TensorMapInterleave { + /// No interleaving + #[default] + None, + /// Interleaved with 16 bytes chunks in the last dim. + /// i.e. NC/8HWC8 with f16 + B16, + /// Interleaved with 32 bytes chunks in the last dim. + /// i.e. NC/16HWC16 with f16 + B32, +} + +/// Data are organized in a specific order in global memory; however, this may not match the order +/// in which the application accesses data in shared memory. This difference in data organization +/// may cause bank conflicts when shared memory is accessed. In order to avoid this problem, data +/// can be loaded to shared memory with shuffling across shared memory banks. When interleave is +/// [`TensorMapInterleave::B32`], swizzle must be [`TensorMapSwizzle::B32`]. +/// Other interleave modes can have any swizzling pattern. +#[derive(Default, Hash, PartialEq, Eq, Clone, Debug, Copy)] +pub enum TensorMapSwizzle { + /// No swizzling + #[default] + None, + /// Swizzle 16B chunks within 32B span + B32, + /// Swizzle 16B chunks within 64B span + B64, + /// Swizzle 16B chunks within 128B span + B128, + /// Swizzle 32B chunks within 128B span + B128Atom32B, + /// Swizzle 32B chunks within 128B span, additionally swap lower 8B with upper 8B within each 16B for every alternate row + B128Atom32BFlip8B, + /// Swizzle 64B chunks within 128B span + B128Atom64B, +} + +/// Additional prefetching to perform during load +/// Specifies L2 fetch size which indicates the byte granularity at which L2 requests are filled from DRAM +#[derive(Default, Hash, PartialEq, Eq, Clone, Debug, Copy)] +pub enum TensorMapPrefetch { + /// No extra prefetch + #[default] + None, + /// Prefetch 64 bytes + B64, + /// Prefetch 128 bytes + B128, + /// Prefetch 256 bytes + B256, +} + +/// What value to use when filling out of bounds values +#[derive(Default, Hash, PartialEq, Eq, Clone, Debug, Copy)] +pub enum OobFill { + /// Fill zeroes + #[default] + Zero, + /// Fill NaN + NaN, +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/base.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/base.rs new file mode 100644 index 00000000..e36e18b8 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/base.rs @@ -0,0 +1,592 @@ +use super::{AutotuneError, AutotuneKey, TuneFn, TuneInputs}; +use alloc::boxed::Box; +use alloc::string::ToString; +use alloc::{format, string::String, sync::Arc, vec, vec::Vec}; +use core::sync::atomic::{AtomicU32, Ordering}; +use hashbrown::HashMap; + +/// A single candidate for autotune: a named [`TuneFn`] plus the [groups](TuneGroup) it +/// belongs to. A tunable is autotuned whenever any of its groups is prioritized. +pub struct Tunable { + pub(crate) function: TuneFn, + groups: Vec<(TuneGroup, PriorityFunc)>, +} + +impl Tunable { + /// Create a tunable from a closure. + /// + /// The `for<'a> Fn(F::At<'a>) -> _` bound is spelled out directly in the + /// `where`-clause (rather than hidden behind a helper trait) so that Rust closure + /// inference sees it: otherwise `move |input| …` picks a single concrete lifetime + /// and fails with `implementation of FnOnce is not general enough` whenever + /// `F::At<'a>` actually depends on `'a`. + /// + /// For multi-input kernels, destructure a tuple: + /// `Tunable::new("name", |(lhs, rhs, out)| body)`. + pub fn new(name: &str, func: Func) -> Self + where + Err: Into + 'static, + Func: for<'a> Fn(::At<'a>) -> Result + Send + Sync + 'static, + { + let name: String = name.into(); + let name_for_err = name.clone(); + Self { + function: TuneFn::new( + name, + Box::new(move |inputs| { + func(inputs).map_err(|err| AutotuneError::Unknown { + name: name_for_err.to_string(), + err: err.into(), + }) + }), + ), + groups: Vec::new(), + } + } + + /// Add this tunable to a [`TuneGroup`] with the given intra-group priority. + /// + /// Groups are autotuned in order of their priority; within each group, tunables are + /// tried in order of `priority(key)`. A negative priority skips the tunable for this + /// key. + pub fn group( + mut self, + group: &TuneGroup, + priority: impl Fn(&K) -> i8 + Send + Sync + 'static, + ) -> Self { + self.groups.push((group.clone(), Arc::new(priority))); + self + } +} + +/// A priority bucket for tunables, computed from the [autotune key](AutotuneKey). +/// +/// Higher-priority groups are autotuned first; once any tunable in a group returns a +/// valid result, no later groups are tried. +pub struct TuneGroup { + id: u32, + name: Arc, + pub(crate) priority: PriorityFunc, +} + +impl core::fmt::Debug for TuneGroup { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TuneGroup").field("id", &self.id).finish() + } +} + +impl Clone for TuneGroup { + fn clone(&self) -> Self { + Self { + id: self.id, + name: self.name.clone(), + priority: self.priority.clone(), + } + } +} + +impl TuneGroup { + /// Create a new group based on a priority function. + pub fn new(name: &str, f: impl Fn(&K) -> i8 + Send + Sync + 'static) -> Self { + let id = GROUP_COUNTER.fetch_add(1, Ordering::Relaxed); + + Self { + id, + name: Arc::new(name.into()), + priority: Arc::new(f), + } + } +} + +#[derive(Debug)] +/// A group plan dictates which [tunables](Tunable) should be executed, and in what order. +pub(crate) struct TunePlan { + priorities: Vec, + no_groups: Vec, + groups: HashMap, + returned: Vec, +} + +#[derive(Default, Debug)] +struct GroupPlan { + priorities: Vec, + indices: HashMap)>>, +} + +#[derive(Debug)] +struct Cleanup { + groups: Vec, + tunables: Vec<(i8, i8)>, + /// Within group priority is too low to even try. + skipped: bool, +} + +impl TunePlan { + pub fn new( + key: &K, + tunables: &[Tunable], + ) -> Self { + let mut priorities = Vec::::new(); + let mut no_groups = Vec::new(); + let mut groups = HashMap::::new(); + + for (index, tunable) in tunables.iter().enumerate() { + if tunable.groups.is_empty() { + no_groups.push(index); + } else { + for (group, within_group_priority_fn) in tunable.groups.iter() { + let priority_fn = &group.priority; + let priority = priority_fn(key); + if !priorities.contains(&priority) { + priorities.push(priority); + } + + let group_priorities = match groups.get_mut(&priority) { + Some(val) => val, + None => { + groups.insert(priority, GroupPlan::default()); + groups.get_mut(&priority).unwrap() + } + }; + let priority = within_group_priority_fn(key); + + if group_priorities.priorities.contains(&priority) { + group_priorities + .indices + .get_mut(&priority) + .unwrap() + .push((index, group.name.clone())); + } else { + group_priorities.priorities.push(priority); + group_priorities + .indices + .insert(priority, vec![(index, group.name.clone())]); + } + } + } + } + + priorities.sort(); + + for group in groups.iter_mut() { + group.1.priorities.sort(); + } + + Self { + priorities, + no_groups, + groups, + returned: Vec::new(), + } + } + + /// Get the next batch of [tunable](Tunable) index to be autotuned. + /// + /// Note that if the list is empty, it means no more autotuned entry can be executed. + pub(crate) fn next(&mut self, mut context_logs: Option<&mut String>) -> Vec { + let mut indices = core::mem::take(&mut self.no_groups); + let priority = self.priorities.last(); + + let priority = match priority { + Some(val) => *val, + None => return indices, + }; + + let (group_indices, cleanup) = self.group_plan_next(priority); + // Some entries are skipped for this round of prioritizing. + let skipped = cleanup.skipped || priority < 0; + let mut all_skip = true; + + self.cleanup(cleanup); + + if priority >= 0 { + if let Some(ctx) = context_logs.take() { + *ctx += format!("\n - Tuning: {group_indices:?}").as_str(); + context_logs = Some(ctx); + } + for (index, _name) in group_indices { + if !self.returned.contains(&index) { + all_skip = false; + indices.push(index); + } + } + } + + // The indices list is empty, but it doesn't mean we should stop + // autotuning, since some entries were skipped. + + if indices.is_empty() && (skipped || all_skip) { + self.next(context_logs) + } else { + for i in indices.iter() { + self.returned.push(*i); + } + indices + } + } + + fn cleanup(&mut self, cleanup: Cleanup) { + for group_p in cleanup.groups { + let index = self + .priorities + .iter() + .enumerate() + .find(|p| *p.1 == group_p) + .unwrap(); + + self.priorities.remove(index.0); + self.groups.remove(&group_p); + } + + for (group_p, tunable_p) in cleanup.tunables { + if let Some(group) = self.groups.get_mut(&group_p) { + let index = group + .priorities + .iter() + .enumerate() + .find(|p| *p.1 == tunable_p) + .unwrap(); + group.priorities.remove(index.0); + group.indices.remove(&tunable_p); + } + } + } + + fn group_plan_next(&mut self, priority: i8) -> (Vec<(usize, Arc)>, Cleanup) { + let group_plan = self.groups.get_mut(&priority).expect("To be filled"); + let within_group_prio = group_plan.priorities.pop().unwrap(); + let mut next_indices = group_plan.indices.remove(&within_group_prio).unwrap(); + + let mut cleanup_groups = Vec::new(); + let mut cleanup_tunables = Vec::new(); + + for (pg, group) in self.groups.iter_mut() { + let mut num_empty_tunables = 0; + let num_tunables = group.priorities.len(); + + for (pt, indices) in group.indices.iter_mut() { + for n in &next_indices { + let entry = indices.iter().enumerate().find(|p| *p.1 == *n); + if let Some(entry) = entry { + indices.remove(entry.0); + } + } + + if indices.is_empty() { + num_empty_tunables += 1; + cleanup_tunables.push((*pg, *pt)); + } + } + + if num_empty_tunables == num_tunables { + cleanup_groups.push(*pg); + } + } + + if within_group_prio < 0 { + // Discard algorithms with negative priority + next_indices.clear(); + } + + ( + next_indices, + Cleanup { + groups: cleanup_groups, + tunables: cleanup_tunables, + skipped: within_group_prio < 0, + }, + ) + } +} + +type PriorityFunc = Arc i8 + Send + Sync>; + +static GROUP_COUNTER: AtomicU32 = AtomicU32::new(0); + +#[cfg(test)] +mod tests { + use core::fmt::Display; + + use serde::{Deserialize, Serialize}; + + use super::*; + + #[derive(Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, Debug)] + struct FakeAutotuneKey; + + impl Display for FakeAutotuneKey { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("FakeAutotuneKey") + } + } + + impl AutotuneKey for FakeAutotuneKey {} + + #[test_log::test] + fn test_plan_order() { + let group0 = TuneGroup::::new("group0", |_| 2); + let group1 = TuneGroup::::new("group1", |_| 1); + + let tunable0 = Tunable::::new("fake", fake_kernel); + let tunable1 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 1); + let tunable2 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 2); + let tunable3 = + Tunable::::new("fake", fake_kernel).group(&group1, |_| 2); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1, tunable2, tunable3]); + + assert_eq!(plan.next(None), vec![0, 2]); + assert_eq!(plan.next(None), vec![1]); + assert_eq!(plan.next(None), vec![3]); + assert!(plan.next(None).is_empty()); + } + + #[test_log::test] + fn test_plan_order_multi_groups_same_priority() { + let group0 = TuneGroup::::new("group0", |_| 2); + let group1 = TuneGroup::::new("group1", |_| 1); + let group2 = TuneGroup::::new("group2", |_| 1); + + let tunable0 = Tunable::::new("fake", fake_kernel); + let tunable1 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 1); + let tunable2 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 2); + let tunable3 = + Tunable::::new("fake", fake_kernel).group(&group1, |_| 2); + let tunable4 = + Tunable::::new("fake", fake_kernel).group(&group2, |_| 2); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1, tunable2, tunable3, tunable4]); + + assert_eq!(plan.next(None), vec![0, 2]); + assert_eq!(plan.next(None), vec![1]); + assert_eq!(plan.next(None), vec![3, 4]); + assert!(plan.next(None).is_empty()); + } + + #[test_log::test] + fn test_plan_order_tunable_multiple_groups() { + let group0 = TuneGroup::::new("group0", |_| 1); + let group1 = TuneGroup::::new("group1", |_| 2); + + let tunable0 = Tunable::::new("fake", fake_kernel); + let tunable1 = Tunable::::new("fake", fake_kernel) + .group(&group0, |_| 1) + .group(&group1, |_| 2); + let tunable2 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 2); + let tunable3 = + Tunable::::new("fake", fake_kernel).group(&group1, |_| 3); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1, tunable2, tunable3]); + + assert_eq!(plan.next(None), vec![0, 3]); + assert_eq!(plan.next(None), vec![1]); + assert_eq!(plan.next(None), vec![2]); + assert!(plan.next(None).is_empty()); + } + + #[test_log::test] + fn test_plan_negative_priority() { + let group0 = TuneGroup::::new("group0", |_| 2); + let group1 = TuneGroup::::new("group1", |_| 1); + + let tunable0 = Tunable::::new("fake", fake_kernel); + let tunable1 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| -1); + let tunable2 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 2); + let tunable3 = + Tunable::::new("fake", fake_kernel).group(&group1, |_| 2); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1, tunable2, tunable3]); + + assert_eq!(plan.next(None), vec![0, 2]); + assert_eq!(plan.next(None), vec![3]); + assert!(plan.next(None).is_empty()); + } + + #[test_log::test] + fn test_plan_no_group() { + let tunable0 = Tunable::::new("fake", fake_kernel); + let tunable1 = Tunable::::new("fake", fake_kernel); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1]); + + assert_eq!(plan.next(None), vec![0, 1]); + assert!(plan.next(None).is_empty()); + } + + #[test_log::test] + fn test_plan_falls_through_when_all_group_tunables_fail() { + // Every tunable lives in exactly one group; the caller treats every batch as a failure + // by continuing to call next(). The plan must still surface every tunable, in priority + // order, before going empty. + let group0 = TuneGroup::::new("group0", |_| 2); + let group1 = TuneGroup::::new("group1", |_| 1); + + let tunable0 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 1); + let tunable1 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 2); + let tunable2 = + Tunable::::new("fake", fake_kernel).group(&group1, |_| 1); + let tunable3 = + Tunable::::new("fake", fake_kernel).group(&group1, |_| 2); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1, tunable2, tunable3]); + + let mut all_returned: Vec = Vec::new(); + loop { + let batch = plan.next(None); + if batch.is_empty() { + break; + } + all_returned.extend(batch); + } + + // Highest group (prio 2) drains first from highest intra-priority down, then next group. + assert_eq!(all_returned, vec![1, 0, 3, 2]); + } + + #[test_log::test] + fn test_plan_single_group_exhausts_all_intra_priorities() { + // A single group with multiple intra-priorities should yield each batch separately, + // allowing the caller to continue on failures until the group is exhausted. + let group0 = TuneGroup::::new("group0", |_| 0); + + let tunable0 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 1); + let tunable1 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 2); + let tunable2 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 3); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1, tunable2]); + + assert_eq!(plan.next(None), vec![2]); + assert_eq!(plan.next(None), vec![1]); + assert_eq!(plan.next(None), vec![0]); + assert!(plan.next(None).is_empty()); + } + + #[test_log::test] + fn test_plan_all_negative_group_advances_to_next_group() { + // A group whose every tunable has a negative intra-priority should be skipped entirely + // without stopping autotuning — the next group must still be reached. + let group0 = TuneGroup::::new("group0", |_| 2); + let group1 = TuneGroup::::new("group1", |_| 1); + + let tunable0 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| -1); + let tunable1 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| -2); + let tunable2 = + Tunable::::new("fake", fake_kernel).group(&group1, |_| 1); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1, tunable2]); + + assert_eq!(plan.next(None), vec![2]); + assert!(plan.next(None).is_empty()); + } + + #[test_log::test] + fn test_plan_no_group_tunables_only_emitted_once_even_on_failures() { + // The ungrouped tunables are emitted together with the first group batch. If the caller + // keeps calling next() (treating the first batch as failing), they must not be + // re-emitted, and the plan must still advance to later groups. + let group0 = TuneGroup::::new("group0", |_| 2); + let group1 = TuneGroup::::new("group1", |_| 1); + + let tunable0 = Tunable::::new("fake", fake_kernel); + let tunable1 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 1); + let tunable2 = + Tunable::::new("fake", fake_kernel).group(&group1, |_| 1); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1, tunable2]); + + assert_eq!(plan.next(None), vec![0, 1]); + assert_eq!(plan.next(None), vec![2]); + assert!(plan.next(None).is_empty()); + } + + #[test_log::test] + fn test_plan_multi_group_tunable_not_duplicated_across_failed_groups() { + // tunable1 belongs to both group0 and group1. It must be returned exactly once (via its + // higher-priority group), even if the caller continues iterating after failures. + let group0 = TuneGroup::::new("group0", |_| 1); + let group1 = TuneGroup::::new("group1", |_| 2); + + let tunable0 = Tunable::::new("fake", fake_kernel) + .group(&group0, |_| 1) + .group(&group1, |_| 1); + let tunable1 = + Tunable::::new("fake", fake_kernel).group(&group0, |_| 2); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1]); + + let mut all_returned: Vec = Vec::new(); + loop { + let batch = plan.next(None); + if batch.is_empty() { + break; + } + all_returned.extend(batch); + } + + // tunable0 comes from group1 (higher priority). tunable1 is the sole member of group0 + // after cross-group dedup. No duplicates. + assert_eq!(all_returned, vec![0, 1]); + } + + #[test_log::test] + fn test_plan_recurses_when_batch_is_fully_already_returned() { + // Regression test: a tunable that lives in multiple groups was already emitted via its + // higher-priority group, so when its lower-priority group's batch fires the only index + // is one already present in `returned`. The plan must NOT return an empty batch here + // (that signals "no more work" to the caller and aborts with NoValidKernelFound); it + // must recurse to the next intra-priority and surface the remaining tunable. + // + // Cross-group dedup in group_plan_next compares (index, Arc group_name), so a + // tunable appearing in both group_hi and group_lo isn't auto-removed from group_lo + // when popped from group_hi — the `returned` + `all_skip` path is the only guard. + let group_hi = TuneGroup::::new("hi", |_| 2); + let group_lo = TuneGroup::::new("lo", |_| 1); + + // tunable0 is in both groups. tunable1 is only in group_lo at a lower intra-priority. + let tunable0 = Tunable::::new("fake", fake_kernel) + .group(&group_hi, |_| 1) + .group(&group_lo, |_| 2); + let tunable1 = + Tunable::::new("fake", fake_kernel).group(&group_lo, |_| 1); + + let key = FakeAutotuneKey; + let mut plan = TunePlan::new(&key, &[tunable0, tunable1]); + + // First call: group_hi yields tunable0. + assert_eq!(plan.next(None), vec![0]); + // Second call: group_lo's higher intra-priority batch is just tunable0 (already + // returned). Without the fix this returns [] and the autotuner aborts. With the fix + // the plan recurses and yields tunable1. + assert_eq!(plan.next(None), vec![1]); + assert!(plan.next(None).is_empty()); + } + + fn fake_kernel(_: ()) -> Result<(), String> { + Ok(()) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/input_generator.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/input_generator.rs new file mode 100644 index 00000000..3c094d44 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/input_generator.rs @@ -0,0 +1,50 @@ +use super::TuneInputs; + +/// Produces the benchmark inputs for a given key and reference inputs. +/// +/// A tuner runs each candidate on the value `generate` returns, not on the real call +/// inputs directly, so callers can synthesize test inputs (for example, allocate fresh +/// output buffers) without mutating the real ones. The common case is just cloning the +/// reference inputs; use [`CloneInputGenerator`] for that. +/// +/// There's no `Fn`-based blanket impl for arbitrary [`TuneInputs`] families because +/// `for<'a> Fn(&K, &I::At<'a>) -> I::At<'a>` runs into E0582 (`Fn`'s `Output` cannot +/// depend on the higher-ranked lifetime). For the owned-input case the HRTB collapses +/// and the `Fn` blanket below works; borrowed-input families implement this trait +/// directly. +#[diagnostic::on_unimplemented( + message = "`{Self}` is not a valid input generator", + label = "invalid input generator" +)] +pub trait InputGenerator: Send + Sync + 'static { + /// Generate a set of inputs for a given key and reference inputs. + fn generate<'a>(&self, key: &K, inputs: &I::At<'a>) -> I::At<'a>; +} + +/// [`InputGenerator`] that clones the reference inputs verbatim. +#[derive(Copy, Clone, Debug, Default)] +pub struct CloneInputGenerator; + +impl InputGenerator for CloneInputGenerator { + fn generate<'a>(&self, _key: &K, inputs: &I::At<'a>) -> I::At<'a> { + inputs.clone() + } +} + +/// `Fn(&K, &A) -> A` acts as an [`InputGenerator`] when `A` is an owned type. For +/// multi-input kernels, `A` is a tuple that the closure destructures internally. +impl InputGenerator for Func +where + A: Clone + Send + Sync + 'static, + K: 'static, + Func: Send + Sync + 'static + Fn(&K, &A) -> A, +{ + #[inline] + fn generate<'a>( + &self, + key: &K, + inputs: &::At<'a>, + ) -> ::At<'a> { + (self)(key, inputs) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/key_generator.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/key_generator.rs new file mode 100644 index 00000000..ce893b85 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/key_generator.rs @@ -0,0 +1,24 @@ +use super::TuneInputs; + +/// Produces an autotune key from a borrowed view of the tuning inputs. +#[diagnostic::on_unimplemented( + message = "`{Self}` is not a valid key generator", + label = "invalid key generator" +)] +pub trait KeyGenerator: Send + Sync + 'static { + /// Generate a key from a set of inputs. + fn generate(&self, inputs: &I::At<'_>) -> K; +} + +/// Any `for<'a> Fn(&I::At<'a>) -> K` is a [`KeyGenerator`]. +impl KeyGenerator for Func +where + I: TuneInputs, + Func: Send + Sync + 'static, + for<'a> Func: Fn(&I::At<'a>) -> K, +{ + #[inline] + fn generate<'a>(&self, inputs: &I::At<'a>) -> K { + (self)(inputs) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/local.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/local.rs new file mode 100644 index 00000000..dcec7435 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/local.rs @@ -0,0 +1,190 @@ +use super::{AutotuneKey, AutotuneOutput, TunableSet, TuneInputs, Tuner}; +use crate::{client::ComputeClient, runtime::Runtime, tune::TuneCacheResult}; +use alloc::string::ToString; +use alloc::sync::Arc; +use core::{ + any::{Any, TypeId}, + fmt::Display, + hash::Hash, +}; +use hashbrown::HashMap; +use spin::Mutex; + +/// A local tuner allows to create a tuner for a specific key that can be different from the server +/// key. +pub struct LocalTuner { + state: Mutex>>>>, + name: &'static str, + sets: spin::RwLock>>>, +} + +/// Create a local tuner with the provided name. +#[macro_export] +macro_rules! local_tuner { + ($name:expr) => { + LocalTuner::new(concat!(module_path!(), "-", $name)); + }; + () => { + LocalTuner::new(module_path!()); + }; +} + +pub use local_tuner; + +impl LocalTuner +where + AK: AutotuneKey + 'static, + ID: Hash + PartialEq + Eq + Clone + Display, +{ + /// Create a new local tuner. + pub const fn new(name: &'static str) -> Self { + Self { + state: Mutex::new(None), + name, + sets: spin::RwLock::new(None), + } + } + + /// Get or initialize the [`TunableSet`] for this tuner. + /// + /// Returns a cached `Arc` keyed by the `TypeId` of `init_set`. The + /// initializer runs at most once per process. + pub fn init(&self, init_set: F) -> Arc> + where + F: Fn() -> TunableSet + 'static + Send + Sync, + I: TuneInputs, + Out: AutotuneOutput, + { + let sets = self.sets.read(); + let type_id = TypeId::of::(); + + static DOWNCAST_ERROR: &str = "Local tuner only support one set of tunable that must work on the same input and output declared with the init function."; + + if let Some(sets) = sets.as_ref() + && let Some(set) = sets.get(&type_id) + { + return set.clone().downcast().expect(DOWNCAST_ERROR); + }; + + core::mem::drop(sets); + + let mut sets = self.sets.write(); + + if let Some(sets) = sets.as_ref() + && let Some(set) = sets.get(&type_id) + { + return set.clone().downcast().expect(DOWNCAST_ERROR); + }; + + let content = Arc::new(init_set()); + + if let Some(sets) = sets.as_mut() { + sets.insert(type_id, content.clone()); + } else { + let mut map = HashMap::>::new(); + map.insert(type_id, content.clone()); + *sets = Some(map); + }; + + content + } + + /// Clear the autotune state. + pub fn clear(&self) { + if let Some(s) = self.state.lock().as_mut() { + s.clear() + } + } + + #[cfg(feature = "autotune-checks")] + fn checks<'a, I: TuneInputs, Out: AutotuneOutput>( + &self, + operations: &TunableSet, + inputs: &::At<'a>, + ) where + ::At<'a>: Clone + Send, + { + use alloc::vec::Vec; + + let mut checks_outputs = Vec::new(); + for i in 0..operations.len() { + let op = operations.fastest(i); + let result = op.execute(inputs.clone()); + checks_outputs.push(result); + } + super::check_autotune_outputs(checks_outputs); + } + + /// Execute the fastest operation in a [`TunableSet`], triggering a tuning pass on + /// the first call for a given key. + pub fn execute<'a, R: Runtime, I: TuneInputs, Out>( + &self, + id: &ID, + client: &ComputeClient, + operations: Arc>, + inputs: ::At<'a>, + ) -> Out + where + ::At<'a>: Clone + Send, + Out: AutotuneOutput, + { + let key = operations.generate_key(&inputs); + + let tuner = { + let mut state_lock = self.state.lock(); + let state_map = state_lock.get_or_insert_with(|| HashMap::new()); + state_map + .entry(id.clone()) + .or_insert_with(move || { + let name = self.name.replace("::", "-"); + Arc::new(Tuner::new(&name, &id.to_string())) + }) + .clone() + }; + + // First, check for a cache hit under a read lock. + if let TuneCacheResult::Hit { fastest_index } = tuner.fastest(&key) { + #[cfg(feature = "autotune-checks")] + self.checks::(&operations, &inputs); + return operations + .fastest(fastest_index) + .execute(inputs) + .expect("Should run when selected by autotune."); + } + + let fastest = tuner.check_tune::( + &key, + &inputs, + &operations, + || operations.compute_checksum(), + client, + ); + + // Run the execution depending on the cache state. + match fastest { + TuneCacheResult::Hit { fastest_index } => { + #[cfg(feature = "autotune-checks")] + self.checks::(&operations, &inputs); + + operations + .fastest(fastest_index) + .execute(inputs) + .expect("Should run when selected by autotune.") + } + TuneCacheResult::Unchecked | TuneCacheResult::Miss => { + panic!( + "Somehow we STILL didn't check a tuning checksum or start tuning, something has gone wrong." + ) + } + TuneCacheResult::Pending => { + // Still waiting (e.g. on wasm). Try all operations as a fallback. + for i in 0..operations.len() { + if let Ok(output) = operations.fastest(i).execute(inputs.clone()) { + return output; + } + } + panic!("All autotune operations failed, no viable operation found."); + } + } + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/mod.rs new file mode 100644 index 00000000..16cb08ae --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/mod.rs @@ -0,0 +1,49 @@ +//! # Autotuning +//! +//! Autotuning runs several candidate kernels on reference inputs and caches the fastest +//! one per key. +//! +//! ```ignore +//! #[derive(AutotuneKey)] +//! struct KernelKey { size: u32 } +//! +//! fn run_kernel_tuned(lhs: Tensor, rhs: Tensor) -> Tensor { +//! static TUNER: LocalTuner = local_tuner!(); +//! +//! let tunables = TUNER.init(|| { +//! TunableSet::new(KernelKey::new, |_key, (lhs, rhs)| (lhs.clone(), rhs.clone())) +//! .with(Tunable::new("k1", |(lhs, rhs)| kernel_1(lhs, rhs))) +//! .with(Tunable::new("k2", |(lhs, rhs)| kernel_2(lhs, rhs))) +//! }); +//! +//! TUNER.execute(&device_id, &lhs.client, tunables, (lhs, rhs)); +//! } +//! ``` +//! +//! Kernels are closures returning `Result>`. Multi-input kernels +//! take a single tuple argument and destructure: `|(lhs, rhs, out)| body`. +//! +//! See [`TuneInputs`] for the borrowed-inputs story, and [`Tunable::new`] for why its +//! HRTB bound is spelled out directly (closure inference). + +mod base; +mod input_generator; +mod key_generator; +mod local; +mod operation; +mod tune_benchmark; +mod tune_cache; +mod tune_inputs; +mod tuner; +mod util; + +pub use base::*; +pub use input_generator::*; +pub use key_generator::*; +pub use local::*; +pub use operation::*; +pub use tune_benchmark::*; +pub use tune_cache::*; +pub use tune_inputs::*; +pub use tuner::*; +pub use util::*; diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/operation.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/operation.rs new file mode 100644 index 00000000..cab28236 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/operation.rs @@ -0,0 +1,138 @@ +use alloc::boxed::Box; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::{Debug, Display}; +use core::hash::Hash; + +use alloc::format; + +use super::{ + AutotuneError, input_generator::InputGenerator, key_generator::KeyGenerator, + tune_inputs::TuneInputs, +}; +use super::{Tunable, TunePlan}; + +/// A type-erased delegate for a tunable function. +/// +/// The lifetime `'inp` is the lifetime of the input data, the function must be defined such that +/// it can be called for any lifetime `inp` and produce a `Result`. +type TuneDelegate = + dyn for<'inp> Fn(::At<'inp>) -> Result + Send + Sync; + +/// A named, type-erased tunable function stored in a [`TunableSet`]. Constructed via +/// [`Tunable::new`](super::Tunable::new); callers don't name this type directly. +#[derive(new)] +pub struct TuneFn { + pub(crate) name: String, + func: Box>, +} + +impl TuneFn { + /// Run the wrapped function on the given inputs. + pub fn execute<'a>(&self, inputs: ::At<'a>) -> Result { + (self.func)(inputs) + } +} + +/// A set of candidate tunable functions for autotune, sharing a key generator and an +/// input generator. See [`TuneInputs`] for the `F` parameter. +pub struct TunableSet { + tunables: Vec>, + key_gen: Arc + Send + Sync>, + input_gen: Arc + Send + Sync>, +} + +impl TunableSet { + /// The number of tunables in the set. + pub fn len(&self) -> usize { + self.tunables.len() + } + + /// Whether this set contains no tunables. + pub fn is_empty(&self) -> bool { + self.tunables.is_empty() + } + + /// Create a tunable set from a key generator and an input generator. + pub fn new(key_gen: impl KeyGenerator, input_gen: impl InputGenerator) -> Self { + Self { + tunables: Default::default(), + input_gen: Arc::new(input_gen), + key_gen: Arc::new(key_gen), + } + } + + /// Shorthand for [`new`](Self::new) with a [`CloneInputGenerator`]: benchmarks run + /// on clones of the real call inputs. + pub fn new_cloning_inputs(key_gen: impl KeyGenerator) -> Self { + Self::new(key_gen, super::CloneInputGenerator) + } + + /// Register a tunable with this tunable set. + pub fn with(mut self, tunable: Tunable) -> Self { + self.tunables.push(tunable); + self + } + + /// All candidate operations in this set, in registration order. + pub fn autotunables(&self) -> impl Iterator> { + self.tunables.iter().map(|tunable| &tunable.function) + } + + /// Returns the [autotune plan](TunePlan) for the given set. + pub(crate) fn plan(&self, key: &K) -> TunePlan { + TunePlan::new(key, &self.tunables) + } + + /// Returns the operation for the given index, matching the order returned by + /// `autotunables`. Tunables are tried in order, so index 0 should be a good default. + pub fn fastest(&self, fastest_index: usize) -> &TuneFn { + &self.tunables[fastest_index].function + } + + /// Compute a checksum that invalidates outdated cached auto-tune results when the + /// set of tunable names changes. + pub fn compute_checksum(&self) -> String { + let mut checksum = String::new(); + for tune in &self.tunables { + checksum += &tune.function.name; + } + format!("{:x}", md5::compute(checksum)) + } + + /// Generate a key from a set of inputs + pub fn generate_key<'a>(&self, inputs: &F::At<'a>) -> K { + self.key_gen.generate(inputs) + } + + /// Generate a set of test inputs from a key and reference inputs. + pub fn generate_inputs<'a>(&self, key: &K, inputs: &F::At<'a>) -> F::At<'a> { + self.input_gen.generate(key, inputs) + } +} + +#[cfg(std_io)] +/// Trait alias with support for persistent caching +pub trait AutotuneKey: + Clone + + Debug + + PartialEq + + Eq + + Hash + + Display + + serde::Serialize + + serde::de::DeserializeOwned + + Send + + Sync + + 'static +{ +} +#[cfg(not(std_io))] +/// Trait alias +pub trait AutotuneKey: + Clone + Debug + PartialEq + Eq + Hash + Display + Send + Sync + 'static +{ +} + +impl AutotuneKey for String {} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_benchmark.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_benchmark.rs new file mode 100644 index 00000000..dc60c232 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_benchmark.rs @@ -0,0 +1,126 @@ +use super::{AutotuneError, TuneFn, TuneInputs}; +use crate::{client::ComputeClient, runtime::Runtime}; +use alloc::string::ToString; +use alloc::vec::Vec; +use cubecl_common::profile::ProfileDuration; + +/// The trait to be implemented by an autotune output. +pub trait AutotuneOutput: Send + 'static { + #[cfg(feature = "autotune-checks")] + /// Checks if the output of an autotune operation is the same as another one on the same + /// problem. + fn check_equivalence(&self, other: Self); +} + +impl AutotuneOutput for () { + #[cfg(feature = "autotune-checks")] + fn check_equivalence(&self, _other: Self) { + // + } +} + +/// Benchmark how long this operation takes for a number of samples. +/// +/// Returns at least one duration, otherwise an error is returned. +pub fn tune_benchmark<'a, R: Runtime, F: TuneInputs, Out: AutotuneOutput>( + operation: &TuneFn, + inputs: ::At<'a>, + client: ComputeClient, +) -> Result, AutotuneError> { + // `scoped` holds exclusive device access for the whole benchmark loop and + // accepts non-`'static` closures. + client + .clone() + .exclusive(move || profile_exclusive(operation, inputs, client)) + .map_err(|err| AutotuneError::Unknown { + name: operation.name.to_string(), + err: err.to_string(), + })? +} + +fn profile_exclusive<'a, R: Runtime, F: TuneInputs, Out: AutotuneOutput>( + operation: &TuneFn, + inputs: ::At<'a>, + client: ComputeClient, +) -> Result, AutotuneError> { + warmup(operation, inputs.clone(), client.clone())?; + + let num_samples = 10; + let mut durations = Vec::new(); + + for _ in 0..num_samples { + let result: Result< + (Result, ProfileDuration), + crate::server::ProfileError, + > = { + let inputs = inputs.clone(); + + client.profile( + move || { + // It is important to return the output since otherwise deadcode elimination + // might optimize away code that needs to be profiled. + operation.execute(inputs) + }, + &operation.name, + ) + }; + + let result = match result { + Ok((out, duration)) => match out { + Ok(_) => Some(duration), + Err(err) => { + log::trace!("Error while autotuning {err:?}"); + None + } + }, + Err(err) => { + log::trace!("Error while autotuning {err:?}"); + None + } + }; + + if let Some(item) = result { + durations.push(item); + } + } + + if durations.is_empty() { + Err(AutotuneError::InvalidSamples { + name: operation.name.to_string(), + }) + } else { + Ok(durations) + } +} + +fn warmup<'a, R: Runtime, F: TuneInputs, Out: AutotuneOutput>( + operation: &TuneFn, + inputs: ::At<'a>, + client: ComputeClient, +) -> Result<(), AutotuneError> { + let num_warmup = 3; + + let mut errors = Vec::with_capacity(num_warmup); + // We make sure the server is in a correct state. + let _errs = client.flush(); + + for _ in 0..num_warmup { + let inputs = inputs.clone(); + let profiled = client.profile(move || operation.execute(inputs), &operation.name); + + match profiled { + Ok(_) => {} + Err(err) => errors.push(err), + } + } + + if errors.len() < num_warmup { + Ok(()) + } else { + let msg = alloc::format!("{:?}", errors.remove(num_warmup - 1)); + Err(AutotuneError::Unknown { + name: operation.name.to_string(), + err: msg, + }) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_cache.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_cache.rs new file mode 100644 index 00000000..6f18862a --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_cache.rs @@ -0,0 +1,265 @@ +#[cfg(std_io)] +use std::vec::Vec; + +#[cfg(std_io)] +use cubecl_common::cache::Cache; +#[cfg(std_io)] +use cubecl_common::cache::CacheError; +#[cfg(std_io)] +use serde::{Deserialize, Serialize}; + +use super::{AutotuneError, AutotuneKey, AutotuneOutcome}; +use alloc::string::String; +use hashbrown::HashMap; + +#[derive(Debug)] +pub(crate) enum CacheEntry { + Done { + checksum: ChecksumState, + fastest_index: usize, + }, + Pending, +} + +#[derive(Debug)] +#[allow(dead_code)] // Some variants are not created when the cache isn't saved. +pub(crate) enum ChecksumState { + Match, + NoMatch, + ToBeVerified(String), +} + +/// Persistent cache key +#[cfg(std_io)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)] +pub(crate) struct PersistentCacheKey { + key: K, + checksum: String, +} + +/// Persistent cache entry +#[cfg(std_io)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] +pub(crate) struct PersistentCacheValue { + fastest_index: usize, + results: Vec, +} + +#[cfg_attr(std_io, derive(Serialize, Deserialize))] +#[derive(Debug, Clone)] +/// The result of an autotune job. +pub struct AutotuneResult { + pub(crate) outcome: Result, +} + +impl AutotuneResult { + pub(crate) fn error(error: AutotuneError) -> Self { + Self { + outcome: Err(error), + } + } + pub(crate) fn success(outcome: AutotuneOutcome) -> Self { + Self { + outcome: Ok(outcome), + } + } +} + +impl Eq for AutotuneResult {} +impl PartialEq for AutotuneResult { + fn eq(&self, other: &Self) -> bool { + match (&self.outcome, &other.outcome) { + (Ok(lhs), Ok(rhs)) => lhs == rhs, + (Ok(_), Err(_)) => false, + (Err(_), Ok(_)) => false, + // We don't have to check the error + (Err(_), Err(_)) => true, + } + } +} + +/// Use to find and reuse the best kernel for some input +#[derive(Debug)] +pub(crate) struct TuneCache { + in_memory_cache: HashMap, + #[cfg(std_io)] + persistent_cache: Cache, PersistentCacheValue>, +} + +/// Result of the cache try +#[derive(Debug)] +pub enum TuneCacheResult { + /// An operation is found. + Hit { + /// The index of the fastest operation to execute. + fastest_index: usize, + }, + /// The operation might be cached, but we don't know yet whether the checksum is valid. + Unchecked, + /// A tuning job is in flight for this key — the worker hasn't published a result yet. + /// The receiver wakes (with `Err(RecvError)`) when the worker commits the result. Native + /// callers `block_on` it and re-query; wasm callers drop it and fall back. + Pending, + /// No operation is found yet. + Miss, +} + +impl TuneCache { + pub(crate) fn new( + #[cfg_attr(not(std_io), allow(unused_variables))] name: &str, + #[cfg_attr(not(std_io), allow(unused_variables))] device_id: &str, + ) -> Self { + #[cfg(std_io)] + { + use crate::config::RuntimeConfig; + use std::format; + + let root = crate::config::CubeClRuntimeConfig::get() + .autotune + .cache + .root(); + let options = cubecl_common::cache::CacheOption::default(); + let mut cache = TuneCache { + in_memory_cache: HashMap::new(), + persistent_cache: Cache::new( + format!("{device_id}/{name}"), + options.root(root).name("autotune"), + ), + }; + cache.load(); + cache + } + + #[cfg(not(std_io))] + { + TuneCache { + in_memory_cache: HashMap::new(), + } + } + } + + pub fn fastest(&self, key: &K) -> TuneCacheResult { + let Some(val) = self.in_memory_cache.get(key) else { + return TuneCacheResult::Miss; + }; + + let CacheEntry::Done { + checksum, + fastest_index, + } = val + else { + // Pending: clone the receiver so the caller can subscribe to the in-flight tune. + let CacheEntry::Pending = val else { + unreachable!() + }; + return TuneCacheResult::Pending; + }; + + if cfg!(std_io) { + match checksum { + ChecksumState::ToBeVerified(..) => TuneCacheResult::Unchecked, // Don't know yet. + ChecksumState::NoMatch => TuneCacheResult::Miss, // Can't use this. + ChecksumState::Match => TuneCacheResult::Hit { + fastest_index: *fastest_index, + }, + } + } else { + // Clippy; + let _ = checksum; + TuneCacheResult::Hit { + fastest_index: *fastest_index, + } + } + } + + #[cfg(std_io)] + pub fn validate_checksum(&mut self, key: &K, checksum: &str) -> TuneCacheResult { + let Some(val) = self.in_memory_cache.get_mut(key) else { + return TuneCacheResult::Miss; + }; + + if let CacheEntry::Done { + checksum: checksum_state, + .. + } = val + && let ChecksumState::ToBeVerified(checksum_expected) = checksum_state + { + if checksum_expected == checksum { + *checksum_state = ChecksumState::Match; + } else { + *checksum_state = ChecksumState::NoMatch; + } + } + + self.fastest(key) + } + + /// Mark a key as being tuned. Used by [`Tuner::tune`] under the cache mutex so that + /// concurrent callers see [`TuneCacheResult::Pending`] and wait on the same job instead of + /// starting a second one. Returns `(Sender, Receiver)`: + pub(crate) fn mark_pending(&mut self, key: K) { + self.in_memory_cache.insert(key, CacheEntry::Pending); + } + + pub(crate) fn cache_insert(&mut self, key: K, fastest_index: usize) { + self.in_memory_cache.insert( + key, + CacheEntry::Done { + checksum: ChecksumState::Match, + fastest_index, + }, + ); + } +} + +#[cfg(std_io)] +impl TuneCache { + pub(crate) fn persistent_cache_insert( + &mut self, + key: K, + checksum: String, + fastest_index: usize, + results: Vec, + ) { + if let Err(err) = self.persistent_cache.insert( + PersistentCacheKey { key, checksum }, + PersistentCacheValue { + fastest_index, + results, + }, + ) { + match err { + CacheError::DuplicatedKey { + key, + value_previous, + value_updated, + } => { + log::warn!( + "Autotune the same function multiple times for key {key:?} => old {value_previous:?}, new {value_updated:?}" + ); + } + CacheError::KeyOutOfSync { .. } => { + // This is OK. + } + } + } + // .expect(); + } + + /// Load the persistent cache data from disk + pub(crate) fn load(&mut self) { + log::info!("Load autotune cache ..."); + let mut loaded = 0; + self.persistent_cache.for_each(|key, value| { + loaded += 1; + self.in_memory_cache.insert( + key.key.clone(), + CacheEntry::Done { + checksum: ChecksumState::ToBeVerified(key.checksum.clone()), + fastest_index: value.fastest_index, + }, + ); + }); + log::info!("Loaded {loaded} autotune cached entries"); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_inputs.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_inputs.rs new file mode 100644 index 00000000..bd62e42a --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/tune_inputs.rs @@ -0,0 +1,22 @@ +/// Describes the set of inputs a [`TunableSet`](super::TunableSet) accepts. +/// +/// The associated type [`At`](Self::At) gives the concrete input type at a specific +/// lifetime. The indirection exists because [`TunableSet`](super::TunableSet) must be +/// `'static` (to live in [`LocalTuner`](super::LocalTuner)'s `Arc` cache), but +/// some callers want to pass *borrowed* inputs. A `'static` marker type with a +/// lifetime-parameterized works here: the set is `'static`, but every +/// tunable function accepts `Self::At<'a>` for any `'a` via HRTB. +/// +/// Implementing it manually is needed when the inputs genuinely borrowed. Example: +/// burn's fusion autotune passes `TuneInput<'a, R, O>` (which wraps `&'a mut Context`) +/// by defining a `FusionTuneInputs` marker with +/// `At<'a> = TuneInput<'a, R, O>`. Such a marker must not implement `Clone`, otherwise +/// it would overlap with the blanket impl below. +pub trait TuneInputs: Send + Sync + 'static { + /// The concrete input type at lifetime `'a`. + type At<'a>: Clone + Send; +} + +impl TuneInputs for T { + type At<'a> = T; +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/tuner.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/tuner.rs new file mode 100644 index 00000000..d74092f6 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/tuner.rs @@ -0,0 +1,406 @@ +use alloc::format; +use alloc::sync::Arc; +use alloc::vec::Vec; +use cubecl_common::profile::ProfileDuration; + +use core::time::Duration; + +use alloc::string::{String, ToString}; +use cubecl_common::benchmark::{BenchmarkComputations, BenchmarkDurations}; + +use crate::config::{Logger, autotune::AutotuneLogLevel}; +use crate::server::LaunchError; +use crate::tune::{AutotuneResult, TuneCache, tune_benchmark}; +use crate::{client::ComputeClient, runtime::Runtime}; + +use super::{AutotuneKey, AutotuneOutput, TunableSet, TuneCacheResult, TuneInputs}; + +#[derive(Debug)] +/// Runs autotune benchmarks for a single device and caches the results. +/// +/// On wasm, [`tune`](Self::tune) spawns its work on the browser event loop; elsewhere +/// it blocks inline. Either way the benchmarking itself is synchronous; only the +/// per-sample profile resolution is awaited. +pub struct Tuner { + cache: Arc>>, + logger: Arc>, +} + +/// The measured outcome for a given autotune invocation. +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +#[derive(new, Debug, Clone, PartialEq, Eq)] +pub struct AutotuneOutcome { + name: String, + index: usize, + computation: BenchmarkComputations, +} + +impl core::fmt::Display for AutotuneOutcome { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "Autotune[{}] name {} => {:?}", + self.index, self.name, self.computation + ) + } +} + +/// Error from running autotune. +#[derive(Debug, Clone)] +#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))] +pub enum AutotuneError { + /// An unknown error happened. + Unknown { + /// The name of the tunable. + name: String, + /// The unknown error, + err: String, + }, + /// All samples are invalid. + InvalidSamples { + /// The name of the tunable. + name: String, + }, + /// No autotune was flagged as valid for the problem. + /// + /// # Warning + /// + /// This is an unrecoverable error and will cause a panic. + NoValidKernelFound { + /// The formatted context on why no valid kernel was found. + context: String, + }, + /// The autotune is skipped manually. + Skip { + /// The name of the skipped kernel. + name: String, + }, + + /// An error happened when launching a kernel. + Launch(LaunchError), +} + +impl From for AutotuneError { + fn from(value: LaunchError) -> Self { + Self::Launch(value) + } +} + +/// A successfully-queued benchmark: the profile futures for each sample, plus its metadata. +struct PendingBench { + index: usize, + name: String, + profiles: Vec, +} + +/// A queued tuning job: all data needed to resolve samples and commit the result. +/// Holds no references so it's trivially `Send + 'static` for the wasm spawn path. +struct TuneRequest { + key: K, + results: Vec, + #[cfg(std_io)] + checksum: String, + context_logs: Option, + pending: Vec, +} + +#[allow(clippy::new_without_default)] +impl Tuner { + /// Create a tuner. Its cache is seeded from the persistent on-disk cache when + /// `std_io` is enabled. + pub fn new(name: &str, device_id: &str) -> Self { + Self { + cache: Arc::new(spin::Mutex::new(TuneCache::new(name, device_id))), + logger: Arc::new(spin::Mutex::new(Logger::new())), + } + } + + /// Fetch the fastest autotune operation index for an autotune key. + pub fn fastest(&self, key: &K) -> TuneCacheResult { + self.cache.lock().fastest(key) + } + + /// Check the cache, validate checksums if needed, and kick off a tuning job if the + /// key is a miss. Returns the resolved cache state. + pub fn check_tune<'a, R: Runtime, F: TuneInputs, Out: AutotuneOutput>( + &self, + key: &K, + inputs: &F::At<'a>, + tunables: &TunableSet, + #[cfg_attr(not(std_io), allow(unused))] checksum: impl FnOnce() -> String + Send + Sync, + client: &ComputeClient, + ) -> TuneCacheResult + where + ::At<'a>: Clone + Send, + { + { + let mut cache = self.cache.lock(); + let cur = cache.fastest(key); + + #[cfg(std_io)] + let cur = if matches!(cur, TuneCacheResult::Unchecked) { + let mut log = self.logger.lock(); + let checksum = checksum(); + if let AutotuneLogLevel::Full = log.log_level_autotune() { + log.log_autotune(&format!("validate checksum key={key}, checksum={checksum}")); + } + cache.validate_checksum(key, &checksum) + } else { + cur + }; + + match cur { + TuneCacheResult::Hit { .. } | TuneCacheResult::Pending => return cur, + TuneCacheResult::Miss | TuneCacheResult::Unchecked => { + cache.mark_pending(key.clone()) + } + } + // Scope the guard: the rest of this function re-locks `self.cache` (fast + // path insert, `process_request`), and `spin::Mutex` is non-reentrant. + } + + log::info!("Tuning {key}"); + + let autotunables = tunables.autotunables().collect::>(); + let mut results: Vec = autotunables + .iter() + .map(|a| { + AutotuneResult::error(AutotuneError::Skip { + name: a.name.to_string(), + }) + }) + .collect(); + + #[cfg(std_io)] + let checksum = tunables.compute_checksum(); + + // Fast path: single tunable, no benchmarking needed. + if results.len() == 1 { + self.cache.lock().cache_insert(key.clone(), 0); + return TuneCacheResult::Hit { fastest_index: 0 }; + } + + let test_inputs = tunables.generate_inputs(key, inputs); + let mut plan = tunables.plan(key); + let mut context_logs = match self.logger.lock().log_level_autotune() { + AutotuneLogLevel::Full => Some(String::new()), + _ => None, + }; + + // Walk the plan batch by batch, launching each benchmark synchronously. A + // successful launch queues a `PendingBench` for the async resolver below; + // launch errors go straight into `results`. Retry the next batch if a whole + // batch failed to queue anything. + let mut pending = Vec::::new(); + loop { + let tunable_indices = plan.next(context_logs.as_mut()); + + if tunable_indices.is_empty() { + panic!( + "Can't execute the autotune plan for key: {key:?}\n - plan: {plan:?}\n - results: {results:?}" + ); + } + + for index in tunable_indices { + let op = autotunables[index]; + + match tune_benchmark(op, test_inputs.clone(), client.clone()) { + Ok(profiles) => pending.push(PendingBench { + index, + name: op.name.clone(), + profiles, + }), + Err(err) => { + results[index] = AutotuneResult::error(err); + } + } + } + + if !pending.is_empty() { + break; + } + } + + let request = TuneRequest { + key: key.clone(), + results, + #[cfg(std_io)] + checksum, + context_logs, + pending, + }; + + // Resolve samples and commit the result. On wasm this runs on the browser + // event loop; elsewhere it blocks inline. + #[cfg(target_family = "wasm")] + { + let cache = self.cache.clone(); + let logger = self.logger.clone(); + wasm_bindgen_futures::spawn_local(async move { + process_request(request, &cache, &logger).await; + }); + + return TuneCacheResult::Pending; + } + + #[cfg(not(target_family = "wasm"))] + cubecl_common::future::block_on(process_request(request, &self.cache, &self.logger)) + } +} + +/// Await every profile sample, pick the fastest tunable, commit to the cache. +async fn process_request( + request: TuneRequest, + cache: &spin::Mutex>, + logger: &spin::Mutex, +) -> TuneCacheResult { + let TuneRequest { + key, + mut results, + #[cfg(std_io)] + checksum, + context_logs, + pending, + } = request; + + for bench in pending { + let PendingBench { + index, + name, + profiles, + } = bench; + + if profiles.is_empty() { + results[index] = AutotuneResult::error(AutotuneError::Unknown { + name: name.to_string(), + err: "No profiling available".to_string(), + }); + continue; + } + + let timing_method = profiles.first().unwrap().timing_method(); + let mut durations = Vec::with_capacity(profiles.len()); + for profile in profiles { + durations.push(profile.resolve().await.duration()); + } + + results[index] = AutotuneResult::success(AutotuneOutcome::new( + name.to_string(), + index, + BenchmarkComputations::new(&BenchmarkDurations::from_durations( + timing_method, + durations, + )), + )); + } + + results.sort_by(|a, b| { + let a = a + .outcome + .as_ref() + .map(|r| r.computation.score()) + .unwrap_or(u64::MAX); + let b = b + .outcome + .as_ref() + .map(|r| r.computation.score()) + .unwrap_or(u64::MAX); + a.cmp(&b) + }); + + let fastest_index = results + .first() + .expect("At least one kernel needed.") + .outcome + .as_ref() + .expect("At least one kernel has to succeed.") + .index; + + { + log_result(&mut logger.lock(), &key, &results, context_logs.as_deref()); + cache.lock().cache_insert(key.clone(), fastest_index); + #[cfg(std_io)] + cache + .lock() + .persistent_cache_insert(key, checksum, fastest_index, results); + } + + TuneCacheResult::Hit { fastest_index } +} + +/// Emit the autotune result through the logger at the currently configured level. +fn log_result( + logger: &mut Logger, + key: &K, + results: &[AutotuneResult], + context_logs: Option<&str>, +) { + match logger.log_level_autotune() { + AutotuneLogLevel::Minimal => { + let top_times = results + .iter() + .map(|r| { + let time = r + .outcome + .as_ref() + .map(|r| r.computation.median) + .unwrap_or(Duration::MAX); + + let index = r.outcome.as_ref().map(|r| r.index).unwrap_or_default(); + (index, time) + }) + .take(3) + .collect::>(); + + let result = results + .first() + .expect("At least one kernel needed.") + .outcome + .as_ref() + .expect("At least one kernel has to succeed."); + + let context = context_logs.unwrap_or(""); + logger.log_autotune(&format!( + "Fastest result {}-{key}. \n Top 3 times: {top_times:?}, context: {context}", + result.name, + )); + } + AutotuneLogLevel::Full => { + let result = results + .first() + .expect("At least one kernel needed.") + .outcome + .as_ref() + .expect("At least one kernel has to succeed."); + + let context = context_logs.unwrap_or(""); + logger.log_autotune(&format!( + "Fastest result {}-{key}. Context: {context}", + result.name, + )); + + for result in results.iter() { + match &result.outcome { + Ok(val) => { + logger.log_autotune(&format!("{val}")); + } + Err(err) => logger.log_autotune(&format!("{err:?}")), + } + } + } + AutotuneLogLevel::Disabled => {} + } +} + +#[cfg(feature = "autotune-checks")] +pub(crate) fn check_autotune_outputs( + mut checks_outputs: Vec>, +) { + let reference = checks_outputs.remove(checks_outputs.len() - 1); + + if let Ok(reference) = reference { + for other in checks_outputs.into_iter().flatten() { + reference.check_equivalence(other); + } + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/tune/util.rs b/third_party/cubecl-runtime-0.10.0-patched/src/tune/util.rs new file mode 100644 index 00000000..62e4a35a --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/tune/util.rs @@ -0,0 +1,59 @@ +use core::sync::atomic::{AtomicI32, Ordering}; + +use crate::config::{CubeClRuntimeConfig, RuntimeConfig}; + +/// Autotune levels: +/// +/// '0' => Minimal autotune: scaled anchor of '1.25'. +/// '1' => Medium autotune: normal anchor. +/// '2' => More autotune: scaled anchor of '0.75'. +/// '3' => Autotune everything without anchor. +static AUTOTUNE_LEVEL: AtomicI32 = AtomicI32::new(-1); + +/// Anchor a number to a power of the provided base. +/// +/// Useful when creating autotune keys. +pub fn anchor(x: usize, max: Option, min: Option, base: Option) -> usize { + let autotune_level = load_autotune_level(); + let factor = match autotune_level { + 3 => return x, // Autotune everything, there is no anchor. + 2 => 0.75, + 1 => 1.0, + 0 => 1.25, + _ => panic!("Invalid autotune level {autotune_level:?}"), + }; + + let base = base.unwrap_or(2) as f64 * factor; + let base = f64::max(base, 1.1); // Minimum base. + let exp = (x as f64).log(base).ceil(); + let power = base.powf(exp).ceil() as usize; + + let result = if let Some(max) = max { + core::cmp::min(power, max) + } else { + power + }; + + if let Some(min) = min { + core::cmp::max(result, min) + } else { + result + } +} + +fn load_autotune_level() -> u32 { + let autotune_level = AUTOTUNE_LEVEL.load(Ordering::Relaxed); + if autotune_level == -1 { + let config = CubeClRuntimeConfig::get(); + let level = match config.autotune.level { + crate::config::autotune::AutotuneLevel::Minimal => 0, + crate::config::autotune::AutotuneLevel::Balanced => 1, + crate::config::autotune::AutotuneLevel::Extensive => 2, + crate::config::autotune::AutotuneLevel::Full => 3, + }; + AUTOTUNE_LEVEL.store(level, Ordering::Relaxed); + level as u32 + } else { + autotune_level as u32 + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/src/validation.rs b/third_party/cubecl-runtime-0.10.0-patched/src/validation.rs new file mode 100644 index 00000000..01ad7ba0 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/src/validation.rs @@ -0,0 +1,45 @@ +use cubecl_common::backtrace::BackTrace; +use cubecl_ir::DeviceProperties; + +use crate::{ + id::KernelId, + server::{CubeDim, LaunchError, ResourceLimitError}, +}; + +/// Validate the cube dim of a kernel fits within the hardware limits +pub fn validate_cube_dim( + properties: &DeviceProperties, + kernel_id: &KernelId, +) -> Result<(), LaunchError> { + let requested = kernel_id.cube_dim; + let max: CubeDim = properties.hardware.max_cube_dim.into(); + if !max.can_contain(requested) { + Err(ResourceLimitError::CubeDim { + requested: requested.into(), + max: max.into(), + backtrace: BackTrace::capture(), + } + .into()) + } else { + Ok(()) + } +} + +/// Validate the total units of a kernel fits within the hardware limits +pub fn validate_units( + properties: &DeviceProperties, + kernel_id: &KernelId, +) -> Result<(), LaunchError> { + let requested = kernel_id.cube_dim.num_elems(); + let max = properties.hardware.max_units_per_cube; + if requested > max { + Err(ResourceLimitError::Units { + requested, + max, + backtrace: BackTrace::capture(), + } + .into()) + } else { + Ok(()) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/compute.rs b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/compute.rs new file mode 100644 index 00000000..830b98f0 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/compute.rs @@ -0,0 +1,136 @@ +use super::DummyServer; +use crate::dummy::KernelTask; +use cubecl_common::device::{Device, DeviceService}; +use cubecl_ir::MemoryDeviceProperties; +use cubecl_ir::StorageType; +use cubecl_runtime::server::ComputeServer; +use cubecl_runtime::{ + client::ComputeClient, + compiler::{CompilationError, Compiler}, + logging::ServerLogger, + memory_management::{MemoryConfiguration, MemoryManagement, MemoryManagementOptions}, + runtime::Runtime, + server::ExecutionMode, + storage::BytesStorage, +}; +use cubecl_zspace::Shape; +use cubecl_zspace::Strides; +use std::sync::Arc; + +/// The dummy device. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Default)] +pub struct DummyDevice; + +impl Device for DummyDevice { + fn from_id(_device_id: cubecl_common::device::DeviceId) -> Self { + Self + } + + fn to_id(&self) -> cubecl_common::device::DeviceId { + cubecl_common::device::DeviceId { + type_id: 0, + index_id: 0, + } + } +} + +pub type DummyClient = ComputeClient; + +impl DeviceService for DummyServer { + fn init(_device_id: cubecl_common::device::DeviceId) -> Self { + init_server() + } + + fn utilities(&self) -> Arc { + ComputeServer::utilities(self) as Arc + } +} + +fn init_server() -> DummyServer { + let storage = BytesStorage::default(); + let mem_properties = MemoryDeviceProperties { + max_page_size: 1024 * 1024 * 512, + alignment: 32, + }; + + let memory_management = MemoryManagement::from_configuration( + storage, + &mem_properties, + MemoryConfiguration::default(), + Arc::new(ServerLogger::default()), + MemoryManagementOptions::new("Main CPU Memory"), + ); + DummyServer::new(memory_management, mem_properties) +} + +pub fn test_client(device: &DummyDevice) -> DummyClient { + ComputeClient::load(device) +} + +#[derive(Debug, Clone)] +pub struct DummyCompiler; + +impl Compiler for DummyCompiler { + type Representation = KernelTask; + + type CompilationOptions = (); + + fn compile( + &mut self, + _kernel: cubecl_runtime::kernel::KernelDefinition, + _compilation_options: &Self::CompilationOptions, + _mode: ExecutionMode, + _addr_type: StorageType, + ) -> Result { + unimplemented!() + } + + fn elem_size(&self, _elem: cubecl_ir::ElemType) -> usize { + unimplemented!() + } + + fn extension(&self) -> &'static str { + unimplemented!() + } +} + +#[derive(Debug, Clone)] +pub struct DummyRuntime; + +impl Runtime for DummyRuntime { + type Compiler = DummyCompiler; + + type Server = DummyServer; + + type Device = DummyDevice; + + fn client(device: &Self::Device) -> ComputeClient { + ComputeClient::load(device) + } + + fn name(_client: &ComputeClient) -> &'static str { + unimplemented!() + } + + fn max_cube_count() -> (u32, u32, u32) { + unimplemented!() + } + + fn can_read_tensor(_shape: &Shape, _strides: &Strides) -> bool { + unimplemented!() + } + + fn target_properties() -> cubecl_ir::TargetProperties { + unimplemented!() + } + + fn enumerate_devices( + _: u16, + _: &::Info, + ) -> Vec { + vec![cubecl_common::device::DeviceId { + type_id: 0, + index_id: 0, + }] + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/kernel.rs b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/kernel.rs new file mode 100644 index 00000000..b4c38fef --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/kernel.rs @@ -0,0 +1,36 @@ +use cubecl_runtime::{id::KernelId, storage::BytesResource}; + +/// The `DummyKernel` trait should be implemented for every supported operation +pub trait DummyKernel: Sync + Send + 'static + core::fmt::Debug { + fn compute(&self, resources: &mut [&mut BytesResource]); + + fn id(&self) -> KernelId; + + fn name(&self) -> &'static str { + core::any::type_name::() + } +} + +/// Contains the algorithm for element-wise addition +#[derive(Debug)] +pub struct DummyElementwiseAddition; + +impl DummyKernel for DummyElementwiseAddition { + fn compute(&self, inputs: &mut [&mut BytesResource]) { + // Notice how the kernel is responsible for determining which inputs + // are read-only and which are writable. + let lhs = &inputs[0].read(); + let rhs = &inputs[1].read(); + let out = &mut inputs[2].write(); + + let size = lhs.len(); + + for i in 0..size { + out[i] = lhs[i] + rhs[i]; + } + } + + fn id(&self) -> KernelId { + KernelId::new::() + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/mod.rs new file mode 100644 index 00000000..a5603681 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/mod.rs @@ -0,0 +1,9 @@ +mod compute; +mod kernel; +mod server; +mod tune; + +pub use compute::*; +pub use kernel::*; +pub use server::*; +pub use tune::*; diff --git a/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/server.rs b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/server.rs new file mode 100644 index 00000000..466a9c5c --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/server.rs @@ -0,0 +1,299 @@ +use super::DummyKernel; +use crate::dummy::DummyCompiler; +use cubecl_common::{bytes::Bytes, future::DynFut, profile::ProfileDuration, stream_id::StreamId}; +use cubecl_ir::{ + DeviceProperties, ElemType, HardwareProperties, MemoryDeviceProperties, StorageType, UIntKind, + VectorSize, features::Features, +}; +use cubecl_runtime::{ + allocator::ContiguousMemoryLayoutPolicy, + compiler::{CompilationError, CubeTask}, + id::KernelId, + kernel::{CompiledKernel, KernelMetadata}, + logging::ServerLogger, + memory_management::{ManagedMemoryHandle, MemoryAllocationMode, MemoryManagement, MemoryUsage}, + server::{ + Binding, ComputeServer, CopyDescriptor, CubeCount, CubeDim, ExecutionMode, Handle, + KernelArguments, ProfileError, ProfilingToken, ServerCommunication, ServerError, + ServerUtilities, + }, + storage::{BytesResource, BytesStorage, ComputeStorage, ManagedResource}, + timestamp_profiler::TimestampProfiler, +}; +use cubecl_zspace::{Shape, Strides}; +use std::sync::Arc; + +/// The dummy server is used to test the cubecl-runtime infrastructure. +/// It uses simple memory management with a bytes storage on CPU, without asynchronous tasks. +#[derive(Debug)] +pub struct DummyServer { + memory_management: MemoryManagement, + timestamps: TimestampProfiler, + utilities: Arc>, +} + +#[derive(Debug, Clone)] +pub struct KernelTask { + kernel: Arc, +} + +impl KernelMetadata for KernelTask { + fn name(&self) -> &'static str { + self.kernel.name() + } + + fn id(&self) -> KernelId { + self.kernel.id() + } + + fn address_type(&self) -> cubecl_ir::StorageType { + ElemType::UInt(UIntKind::U32).into() + } +} + +impl core::fmt::Display for KernelTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Dummy kernel") + } +} + +impl CubeTask for KernelTask { + fn compile( + &self, + _compiler: &mut DummyCompiler, + _compilation_options: &::CompilationOptions, + _mode: ExecutionMode, + _addr_type: StorageType, + ) -> Result, CompilationError> { + Ok(CompiledKernel { + entrypoint_name: self.kernel.name().to_string(), + debug_name: None, + source: String::new(), + repr: Some(self.clone()), + cube_dim: CubeDim::new_single(), + debug_info: None, + }) + } +} + +impl KernelTask { + pub fn new(kernel: impl DummyKernel) -> Self { + Self { + kernel: Arc::new(kernel), + } + } + + pub fn compute(&self, resources: &mut [&mut BytesResource]) { + self.kernel.compute(resources); + } +} + +impl ServerCommunication for DummyServer { + const SERVER_COMM_ENABLED: bool = false; +} + +impl ComputeServer for DummyServer { + type Kernel = Box>; + type Storage = BytesStorage; + type MemoryLayoutPolicy = ContiguousMemoryLayoutPolicy; + type Info = (); + + fn logger(&self) -> Arc { + self.utilities.logger.clone() + } + + fn utilities(&self) -> Arc> { + self.utilities.clone() + } + + fn initialize_memory(&mut self, memory: ManagedMemoryHandle, size: u64, _stream_id: StreamId) { + let reserved = self.memory_management.reserve(size).unwrap(); + self.memory_management + .bind(reserved, memory.clone(), 0) + .unwrap(); + } + + fn read( + &mut self, + descriptors: Vec, + _stream_id: StreamId, + ) -> DynFut, ServerError>> { + let bytes: Vec<_> = descriptors + .into_iter() + .map(|b| { + let size = b.handle.size_in_used(); + let resource = self + .memory_management + .get_resource( + b.handle.memory.clone(), + b.handle.offset_start, + b.handle.offset_end, + ) + .unwrap(); + // Keep the binding alive in the future so the memory pool + // doesn't reuse this storage while we still hold a pointer. + (resource, size, b.handle.memory) + }) + .collect(); + + Box::pin(async move { + Ok(bytes + .into_iter() + .map(|(b, size, _binding)| { + Bytes::from_bytes_vec(b.read()[0..size as usize].to_vec()) + }) + .collect()) + }) + } + + fn write(&mut self, descriptors: Vec<(CopyDescriptor, Bytes)>, _stream_id: StreamId) { + for (descriptor, data) in descriptors { + let storage_h = self + .memory_management + .get_storage(descriptor.handle.memory) + .unwrap(); + let mut bytes = self.memory_management.storage().get(&storage_h); + bytes.write()[..data.len()].copy_from_slice(&data); + } + } + + fn sync(&mut self, _stream_id: StreamId) -> DynFut> { + Box::pin(async move { Ok(()) }) + } + + fn get_resource( + &mut self, + binding: Binding, + _stream_id: StreamId, + ) -> Result, ServerError> { + let resource = self.memory_management.get_resource( + binding.memory.clone(), + binding.offset_start, + binding.offset_end, + )?; + + Ok(ManagedResource::new(binding.memory, resource)) + } + + unsafe fn launch( + &mut self, + kernel: Self::Kernel, + _count: CubeCount, + bindings: KernelArguments, + mode: ExecutionMode, + stream_id: StreamId, + ) { + let mut resources: Vec<_> = bindings + .buffers + .into_iter() + .map(|b| { + self.memory_management + .get_resource(b.memory, b.offset_start, b.offset_end) + .unwrap() + }) + .collect(); + let data = bytemuck::cast_slice(&bindings.info.data); + let metadata = Handle::new(stream_id, data.len() as u64); + self.bind_with_data(data, metadata.clone(), stream_id); + + resources.push({ + self.memory_management + .get_resource( + metadata.memory.binding(), + metadata.offset_start, + metadata.offset_end, + ) + .unwrap() + }); + + let mut resources: Vec<_> = resources.iter_mut().collect(); + let kernel = kernel + .compile(&mut DummyCompiler, &(), mode, kernel.address_type()) + .unwrap(); + kernel.repr.unwrap().compute(resources.as_mut_slice()); + } + + fn flush(&mut self, _stream_id: StreamId) -> Result<(), ServerError> { + // Nothing to do with dummy backend. + Ok(()) + } + + fn memory_usage(&mut self, _stream_id: StreamId) -> Result { + Ok(self.memory_management.memory_usage()) + } + + fn memory_cleanup(&mut self, _stream_id: StreamId) { + self.memory_management.cleanup(true); + } + + fn start_profile(&mut self, _stream_id: StreamId) -> Result { + Ok(self.timestamps.start()) + } + + fn end_profile( + &mut self, + _stream_id: StreamId, + token: ProfilingToken, + ) -> Result { + self.timestamps.stop(token) + } + + fn allocation_mode(&mut self, mode: MemoryAllocationMode, _stream_id: StreamId) { + self.memory_management.mode(mode) + } +} + +impl DummyServer { + pub fn new( + memory_management: MemoryManagement, + mem_props: MemoryDeviceProperties, + ) -> Self { + let hardware = HardwareProperties { + load_width: 128, + plane_size_min: 32, + plane_size_max: 32, + max_bindings: 32, + max_shared_memory_size: 48000, + max_cube_count: (u16::MAX as u32, u16::MAX as u32, u16::MAX as u32), + max_units_per_cube: 1024, + max_cube_dim: (1024, 1024, 64), + num_streaming_multiprocessors: None, + num_tensor_cores: None, + min_tensor_cores_dim: None, + num_cpu_cores: None, + max_vector_size: VectorSize::MAX, + }; + let features = Features::default(); + let timing_method = cubecl_common::profile::TimingMethod::System; + let props = DeviceProperties::new(features, mem_props, hardware, timing_method); + let logger = Arc::new(ServerLogger::default()); + + let utilities = Arc::new(ServerUtilities::new( + props, + logger, + (), + ContiguousMemoryLayoutPolicy::new(4), + )); + + Self { + memory_management, + utilities, + timestamps: TimestampProfiler::default(), + } + } + + /// Utility to create a new buffer and immediately copy contiguous data into it + fn bind_with_data(&mut self, data: &[u8], handle: Handle, stream_id: StreamId) { + let strides: Strides = [1].into(); + let shape: Shape = [data.len()].into(); + + self.initialize_memory(handle.memory.clone(), handle.size(), stream_id); + self.write( + vec![( + CopyDescriptor::new(handle.binding(), shape, strides, 1), + Bytes::from_bytes_vec(data.to_vec()), + )], + stream_id, + ); + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/autotune_operations.rs b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/autotune_operations.rs new file mode 100644 index 00000000..7dfedb8a --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/autotune_operations.rs @@ -0,0 +1,26 @@ +use cubecl_runtime::{ + client::ComputeClient, + server::{CubeCount, Handle, KernelArguments}, +}; +use derive_new::new; + +use crate::dummy::{DummyRuntime, KernelTask}; + +#[derive(new, Clone)] +/// Extended kernel that accounts for additional parameters, i.e. needed +/// information that does not count as an input/output. +pub struct OneKernelAutotuneOperation { + kernel: KernelTask, + client: ComputeClient, +} + +impl OneKernelAutotuneOperation { + pub fn run(&self, inputs: Vec) -> Result<(), String> { + self.client.launch( + Box::new(self.kernel.clone()), + CubeCount::Static(1, 1, 1), + KernelArguments::new().with_buffers(inputs.into_iter().map(|h| h.binding()).collect()), + ); + Ok(()) + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/kernels.rs b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/kernels.rs new file mode 100644 index 00000000..04bda3e0 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/kernels.rs @@ -0,0 +1,70 @@ +use std::{thread::sleep, time::Duration}; + +use cubecl_runtime::{id::KernelId, storage::BytesResource}; + +use crate::dummy::DummyKernel; + +const SLEEP_MS: u64 = 1; + +#[derive(Debug)] +pub struct DummyElementwiseAdditionSlowWrong; +#[derive(Debug)] +pub struct DummyElementwiseMultiplication; +#[derive(Debug)] +pub struct DummyElementwiseMultiplicationSlowWrong; + +impl DummyKernel for DummyElementwiseAdditionSlowWrong { + fn compute(&self, inputs: &mut [&mut BytesResource]) { + // Slow and wrong on purpose, for tests + let lhs = &inputs[0].read(); + let out = &mut inputs[2].write(); + + let size = lhs.len(); + + for i in 0..size { + sleep(Duration::from_millis(SLEEP_MS)); + out[i] = lhs[i] + } + } + + fn id(&self) -> KernelId { + KernelId::new::() + } +} + +impl DummyKernel for DummyElementwiseMultiplication { + fn compute(&self, inputs: &mut [&mut BytesResource]) { + let lhs = &inputs[0].read(); + let rhs = &inputs[1].read(); + let out = &mut inputs[2].write(); + + let size = lhs.len(); + + for i in 0..size { + out[i] = lhs[i] * rhs[i]; + } + } + + fn id(&self) -> KernelId { + KernelId::new::() + } +} + +impl DummyKernel for DummyElementwiseMultiplicationSlowWrong { + fn compute(&self, inputs: &mut [&mut BytesResource]) { + // Slow and wrong on purpose, for tests + let lhs = &inputs[0].read(); + let out = &mut inputs[2].write(); + + let size = lhs.len(); + + for i in 0..size { + sleep(Duration::from_millis(SLEEP_MS)); + out[i] = lhs[i]; + } + } + + fn id(&self) -> KernelId { + KernelId::new::() + } +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/mod.rs b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/mod.rs new file mode 100644 index 00000000..c72d787f --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/mod.rs @@ -0,0 +1,8 @@ +mod autotune_operations; +mod kernels; +mod operation_sets; + +pub use autotune_operations::*; +pub use kernels::*; +#[allow(unused)] +pub use operation_sets::*; diff --git a/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/operation_sets.rs b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/operation_sets.rs new file mode 100644 index 00000000..44f9f619 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/tests/dummy/tune/operation_sets.rs @@ -0,0 +1,63 @@ +use cubecl_runtime::{ + server::Handle, + tune::{CloneInputGenerator, Tunable, TunableSet}, +}; + +use crate::dummy::{ + DummyClient, DummyElementwiseAddition, DummyElementwiseMultiplication, + DummyElementwiseMultiplicationSlowWrong, KernelTask, OneKernelAutotuneOperation, +}; + +use super::DummyElementwiseAdditionSlowWrong; + +type TestSet = TunableSet, ()>; + +pub fn addition_set( + client: DummyClient, + shapes: Vec>, +) -> TunableSet, ()> { + let op_add = + OneKernelAutotuneOperation::new(KernelTask::new(DummyElementwiseAddition), client.clone()); + let op_add_slow = OneKernelAutotuneOperation::new( + KernelTask::new(DummyElementwiseAdditionSlowWrong), + client.clone(), + ); + TestSet::new( + move |_input: &Vec| format!("{}-{}", "add", log_shape_input_key(&shapes)), + CloneInputGenerator, + ) + .with(Tunable::new("add", move |inputs| op_add.run(inputs))) + .with(Tunable::new("add_slow_wrong", move |inputs| { + op_add_slow.run(inputs) + })) +} + +pub fn multiplication_set(client: DummyClient, shapes: Vec>) -> TestSet { + let op_mul_slow = OneKernelAutotuneOperation::new( + KernelTask::new(DummyElementwiseMultiplicationSlowWrong), + client.clone(), + ); + let op_mul = OneKernelAutotuneOperation::new( + KernelTask::new(DummyElementwiseMultiplication), + client.clone(), + ); + TestSet::new( + move |_input: &Vec| format!("{}-{}", "mul", log_shape_input_key(&shapes)), + CloneInputGenerator, + ) + .with(Tunable::new("mul_slow_wrong", move |inputs| { + op_mul_slow.run(inputs) + })) + .with(Tunable::new("mul", move |inputs| op_mul.run(inputs))) +} + +pub fn log_shape_input_key(shapes: &[Vec]) -> String { + let mut hash = String::new(); + let lhs = &shapes[0]; + for size in lhs { + let exp = f32::ceil(f32::log2(*size as f32)) as u32; + hash.push_str(2_u32.pow(exp).to_string().as_str()); + hash.push(','); + } + hash +} diff --git a/third_party/cubecl-runtime-0.10.0-patched/tests/integration_test.rs b/third_party/cubecl-runtime-0.10.0-patched/tests/integration_test.rs new file mode 100644 index 00000000..c3a54065 --- /dev/null +++ b/third_party/cubecl-runtime-0.10.0-patched/tests/integration_test.rs @@ -0,0 +1,102 @@ +mod dummy; + +use crate::dummy::{DummyDevice, DummyElementwiseAddition, test_client}; + +use cubecl_runtime::server::CubeCount; +use cubecl_runtime::server::KernelArguments; +use cubecl_runtime::{local_tuner, tune::LocalTuner}; +use dummy::*; + +#[test_log::test] +fn created_resource_is_the_same_when_read() { + let client = test_client(&DummyDevice); + let resource = Vec::from([0, 1, 2]); + let resource_description = client.create_from_slice(&resource); + + let obtained_resource = client.read_one(resource_description).unwrap().to_vec(); + + assert_eq!(resource, obtained_resource) +} + +#[test_log::test] +fn empty_allocates_memory() { + let client = test_client(&DummyDevice); + let size = 4; + let resource_description = client.empty(size); + let empty_resource = client.read_one(resource_description).unwrap(); + + assert_eq!(empty_resource.len(), 4); +} + +#[test_log::test] +fn execute_elementwise_addition() { + let client = test_client(&DummyDevice); + let lhs = client.create_from_slice(&[0, 1, 2]); + let rhs = client.create_from_slice(&[4, 4, 4]); + let out = client.empty(3); + + client.launch( + Box::new(KernelTask::new(DummyElementwiseAddition)), + CubeCount::Static(1, 1, 1), + KernelArguments::new().with_buffers(vec![ + lhs.binding(), + rhs.binding(), + out.clone().binding(), + ]), + ); + + let obtained_resource = client.read_one(out).unwrap().to_vec(); + + assert_eq!(obtained_resource, Vec::from([4, 5, 6])) +} + +#[test_log::test] +#[cfg(feature = "std")] +fn autotune_basic_addition_execution() { + static TUNER: LocalTuner = local_tuner!("autotune_basic_addition_execution"); + + let client = test_client(&DummyDevice); + + let lhs = client.create_from_slice(&[0, 1, 2]); + let rhs = client.create_from_slice(&[4, 4, 4]); + let out = client.empty(3); + let handles = vec![lhs, rhs, out.clone()]; + + let test_set = TUNER.init(|| { + let client = test_client(&DummyDevice); + let shapes = vec![vec![1, 3], vec![1, 3], vec![1, 3]]; + dummy::addition_set(client, shapes) + }); + TUNER.execute(&"test".to_string(), &client, test_set, handles); + + let obtained_resource = client.read_one(out).unwrap().to_vec(); + + // If slow kernel was selected it would output [0, 1, 2] + assert_eq!(obtained_resource, Vec::from([4, 5, 6])); +} + +#[test_log::test] +#[cfg(feature = "std")] +fn autotune_basic_multiplication_execution() { + static TUNER: LocalTuner = + local_tuner!("autotune_basic_multiplication_execution"); + + let client = test_client(&DummyDevice); + + let lhs = client.create_from_slice(&[0, 1, 2]); + let rhs = client.create_from_slice(&[4, 4, 4]); + let out = client.empty(3); + let handles = vec![lhs, rhs, out.clone()]; + + let test_set = TUNER.init(|| { + let client = test_client(&DummyDevice); + let shapes = vec![vec![1, 3], vec![1, 3], vec![1, 3]]; + dummy::multiplication_set(client, shapes) + }); + TUNER.execute(&"test".to_string(), &client, test_set, handles); + + let obtained_resource = client.read_one(out).unwrap().to_vec(); + + // If slow kernel was selected it would output [0, 1, 2] + assert_eq!(obtained_resource, Vec::from([0, 4, 8])); +} diff --git a/third_party/wgpu-29.0.4-patched/.cargo-ok b/third_party/wgpu-29.0.4-patched/.cargo-ok new file mode 100644 index 00000000..5f8b7958 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/third_party/wgpu-29.0.4-patched/.cargo_vcs_info.json b/third_party/wgpu-29.0.4-patched/.cargo_vcs_info.json new file mode 100644 index 00000000..ceac6fa3 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "e99f5305ded96ff7006f0714d043a7f735bd45c2" + }, + "path_in_vcs": "wgpu" +} \ No newline at end of file diff --git a/third_party/wgpu-29.0.4-patched/Cargo.lock b/third_party/wgpu-29.0.4-patched/Cargo.lock new file mode 100644 index 00000000..318731b6 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/Cargo.lock @@ -0,0 +1,1280 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +dependencies = [ + "serde", +] + +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bit-set" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ddef2995421ab6a5c779542c81ee77c115206f4ad9d5a8e05f4ff49716a3dd" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +dependencies = [ + "serde_core", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "objc2", +] + +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + +[[package]] +name = "glow" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29038e1c483364cc6bb3cf78feee1816002e127c331a1eec55a4d202b9e1adb5" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "gpu-allocator" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795" +dependencies = [ + "ash", + "hashbrown 0.16.1", + "log", + "presser", + "thiserror", + "windows", +] + +[[package]] +name = "gpu-descriptor" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" +dependencies = [ + "bitflags", + "gpu-descriptor-types", + "hashbrown 0.15.5", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91" +dependencies = [ + "bitflags", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "serde", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "mach-dxcompiler-rs" +version = "0.1.4+2024.11.22-df583a3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e3cd67e8ea2ba061339150970542cf1c60ba44c6d17e31279cbc133a4b018f8" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + +[[package]] +name = "naga" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2bf919621e7975acb27d881bae2fb993e0d45c8e0446e85e6272971e00dc8df" +dependencies = [ + "arrayvec", + "bit-set", + "bitflags", + "cfg-if", + "cfg_aliases", + "codespan-reporting", + "half", + "hashbrown 0.16.1", + "hexf-parse", + "indexmap", + "libm", + "log", + "num-traits", + "once_cell", + "petgraph", + "pp-rs", + "rustc-hash", + "serde", + "spirv", + "thiserror", + "unicode-ident", +] + +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags", + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" +dependencies = [ + "bitflags", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-metal", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ordered-float" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "pp-rs" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb458bb7f6e250e6eb79d5026badc10a3ebb8f9a15d1fff0f13d17c71f4d6dee" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "profiling" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "range-alloc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" + +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + +[[package]] +name = "raw-window-metal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d213455a5f1dc59214213c7330e074ddf8114c9a42411eb890c767357ce135" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-quartz-core", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + +[[package]] +name = "ron" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd490c5b18261893f14449cbd28cb9c0b637aebf161cd77900bfdedaff21ec32" +dependencies = [ + "bitflags", + "once_cell", + "serde", + "serde_derive", + "typeid", + "unicode-ident", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "spirv" +version = "0.4.0+sdk-1.4.341.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9571ea910ebd84c86af4b3ed27f9dbdc6ad06f17c5f96146b2b671e2976744f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wayland-sys" +version = "0.31.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374f6b70e8e0d6bf9461a32988fd553b59ff630964924dad6e4a4eb6bd538d17" +dependencies = [ + "dlib", + "log", + "once_cell", + "pkg-config", +] + +[[package]] +name = "web-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wgpu" +version = "29.0.4" +dependencies = [ + "arrayvec", + "bitflags", + "bytemuck", + "cfg-if", + "cfg_aliases", + "document-features", + "hashbrown 0.16.1", + "js-sys", + "log", + "naga", + "parking_lot", + "portable-atomic", + "profiling", + "raw-window-handle", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-core" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7" +dependencies = [ + "arrayvec", + "bit-set", + "bit-vec", + "bitflags", + "bytemuck", + "cfg_aliases", + "document-features", + "hashbrown 0.16.1", + "indexmap", + "log", + "macro_rules_attribute", + "naga", + "once_cell", + "parking_lot", + "portable-atomic", + "profiling", + "raw-window-handle", + "ron", + "rustc-hash", + "serde", + "smallvec", + "thiserror", + "wgpu-core-deps-apple", + "wgpu-core-deps-emscripten", + "wgpu-core-deps-wasm", + "wgpu-core-deps-windows-linux-android", + "wgpu-hal", + "wgpu-naga-bridge", + "wgpu-types", +] + +[[package]] +name = "wgpu-core-deps-apple" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e39e26c4c0e07589e67d18546cf79ff45383659fc72fca4dd293358a0347f3" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-emscripten" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e09be551dc939498bdd5f6b2c66e55ab275dad25825267a08605a80fc9f0af" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-wasm" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1fb1798be2a912497d4c224f72d39bb0cb34af50e8bcc29865bc339c943059" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-windows-linux-android" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e592c1bbef6ad047647ae6e666ebd8cee7a32bb4544d9700ec96cbf73230257" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-hal" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d" +dependencies = [ + "android_system_properties", + "arrayvec", + "ash", + "bit-set", + "bitflags", + "block2", + "bytemuck", + "cfg-if", + "cfg_aliases", + "glow", + "glutin_wgl_sys", + "gpu-allocator", + "gpu-descriptor", + "hashbrown 0.16.1", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "mach-dxcompiler-rs", + "naga", + "ndk-sys", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-metal", + "objc2-quartz-core", + "once_cell", + "ordered-float", + "parking_lot", + "portable-atomic", + "portable-atomic-util", + "profiling", + "range-alloc", + "raw-window-handle", + "raw-window-metal", + "renderdoc-sys", + "smallvec", + "thiserror", + "wasm-bindgen", + "wayland-sys", + "web-sys", + "wgpu-naga-bridge", + "wgpu-types", + "windows", + "windows-core", + "windows-result", +] + +[[package]] +name = "wgpu-naga-bridge" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95226013f547544b223281cd16a4fb549aa9dcb562adbda0faae4c73ffbbc161" +dependencies = [ + "naga", + "wgpu-types", +] + +[[package]] +name = "wgpu-types" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bf84cd9ca8ca45e2b223a3868f1adf9bfc0c66aeac212e76ee7e40fdadf8f5" +dependencies = [ + "bitflags", + "bytemuck", + "js-sys", + "log", + "raw-window-handle", + "serde", + "web-sys", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "zerocopy" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/third_party/wgpu-29.0.4-patched/Cargo.toml b/third_party/wgpu-29.0.4-patched/Cargo.toml new file mode 100644 index 00000000..07f0b46b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/Cargo.toml @@ -0,0 +1,268 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.87.0" +name = "wgpu" +version = "29.0.4" +authors = ["gfx-rs developers"] +build = "build.rs" +exclude = ["Cargo.lock"] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Cross-platform, safe, pure-rust graphics API" +homepage = "https://wgpu.rs/" +readme = "README.md" +keywords = ["graphics"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/gfx-rs/wgpu" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs", +] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "wasm32-unknown-unknown", +] + +[package.metadata.cargo-machete] +ignored = ["cfg_aliases"] + +[features] +angle = ["wgpu-core?/angle"] +counters = ["wgpu-core?/counters"] +custom = [] +default = [ + "std", + "parking_lot", + "dx12", + "metal", + "gles", + "vulkan", + "wgsl", + "webgpu", +] +dx12 = ["wgpu-core?/dx12"] +fragile-send-sync-non-atomic-wasm = [ + "wgpu-core?/fragile-send-sync-non-atomic-wasm", + "wgpu-types/fragile-send-sync-non-atomic-wasm", +] +gles = ["wgpu-core?/gles"] +glsl = [ + "naga/glsl-in", + "wgpu-core?/glsl", +] +metal = ["wgpu-core?/metal"] +naga-ir = ["dep:naga"] +noop = [ + "wgpu-core/noop", + "dep:wgpu-hal", + "dep:smallvec", +] +parking_lot = ["dep:parking_lot"] +serde = [ + "wgpu-core?/serde", + "wgpu-types/serde", +] +spirv = [ + "naga/spv-in", + "wgpu-core?/spirv", +] +static-dxc = ["wgpu-core?/static-dxc"] +std = [ + "raw-window-handle/std", + "wgpu-types/std", + "wgpu-core?/std", + "js-sys?/std", + "web-sys?/std", + "wasm-bindgen?/std", + "wasm-bindgen-futures?/std", +] +strict_asserts = [ + "wgpu-core?/strict_asserts", + "wgpu-types/strict_asserts", +] +trace = [ + "serde", + "wgpu-core?/trace", +] +vulkan = ["wgpu-core?/vulkan"] +vulkan-portability = ["wgpu-core?/vulkan-portability"] +web = [ + "dep:wasm-bindgen", + "dep:js-sys", + "dep:web-sys", + "wgpu-types/web", +] +webgl = [ + "web", + "wgpu-core/webgl", + "dep:wgpu-hal", + "dep:smallvec", +] +webgpu = [ + "web", + "naga?/wgsl-out", + "dep:wasm-bindgen-futures", + "web-sys/Document", + "web-sys/Event", + "web-sys/Navigator", + "web-sys/NodeList", + "web-sys/Window", + "web-sys/WorkerGlobalScope", + "web-sys/WorkerNavigator", +] +wgsl = ["wgpu-core?/wgsl"] + +[lib] +name = "wgpu" +path = "src/lib.rs" + +[dependencies.arrayvec] +version = "0.7.1" +default-features = false + +[dependencies.bitflags] +version = "2.9" + +[dependencies.bytemuck] +version = "1.22" +features = [ + "extern_crate_alloc", + "min_const_generics", +] + +[dependencies.cfg-if] +version = "1" + +[dependencies.document-features] +version = "0.2.10" + +[dependencies.hashbrown] +version = "0.16" +features = [ + "default-hasher", + "inline-more", +] +default-features = false + +[dependencies.log] +version = "0.4.29" + +[dependencies.naga] +version = "29.0.1" +features = ["termcolor"] +optional = true + +[dependencies.parking_lot] +version = "0.12.3" +optional = true + +[dependencies.profiling] +version = "1.0.1" +default-features = false + +[dependencies.raw-window-handle] +version = "0.6.2" +features = ["alloc"] +default-features = false + +[dependencies.static_assertions] +version = "1.1" + +[dependencies.wgpu-core] +version = "29.0.1" +optional = true + +[dependencies.wgpu-types] +version = "29.0.1" +default-features = false + +[dev-dependencies.bytemuck] +version = "1.22" +features = [ + "extern_crate_alloc", + "min_const_generics", +] + +[build-dependencies.cfg_aliases] +version = "0.2.1" + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.js-sys] +version = "0.3.85" +optional = true +default-features = false + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.smallvec] +version = "1.14" +optional = true + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.wasm-bindgen] +version = "0.2.108" +optional = true +default-features = false + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.wasm-bindgen-futures] +version = "0.4.58" +optional = true +default-features = false + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.web-sys] +version = "0.3.85" +features = [ + "HtmlCanvasElement", + "OffscreenCanvas", +] +optional = true +default-features = false + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.wgpu-core] +version = "29.0.1" +optional = true + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.wgpu-hal] +version = "29.0.1" +optional = true + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.smallvec] +version = "1.14" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.wgpu-core] +version = "29.0.1" +features = [ + "renderdoc", + "wgsl", + "portable-atomic", +] + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.wgpu-hal] +version = "29.0.1" + +[target.'cfg(not(target_has_atomic = "64"))'.dependencies.portable-atomic] +version = "1.10" + +[target.'cfg(target_os = "emscripten")'.dependencies.smallvec] +version = "1.14" + +[target.'cfg(target_os = "emscripten")'.dependencies.wgpu-core] +version = "29.0.1" + +[target.'cfg(target_os = "emscripten")'.dependencies.wgpu-hal] +version = "29.0.1" diff --git a/third_party/wgpu-29.0.4-patched/Cargo.toml.orig b/third_party/wgpu-29.0.4-patched/Cargo.toml.orig new file mode 100644 index 00000000..ccb10f83 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/Cargo.toml.orig @@ -0,0 +1,271 @@ +[package] +name = "wgpu" +version.workspace = true +authors.workspace = true +edition.workspace = true +description = "Cross-platform, safe, pure-rust graphics API" +homepage.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true +readme = "../README.md" +exclude = ["Cargo.lock"] + +# Override the workspace's `rust-version` key. `wgpu` and its dependencies +# have a less strict MSRV, to allow users more leeway in updating their Rust toolchain. +# +# See the repo README for more information on MSRV policy. +rust-version = "1.87.0" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "wasm32-unknown-unknown", +] + +[package.metadata.cargo-machete] +# Cargo machete can't check build.rs dependencies. See https://github.com/bnjbvr/cargo-machete/issues/100 +ignored = ["cfg_aliases"] + +[lib] + +[features] +default = [ + "std", + "parking_lot", + "dx12", + "metal", + "gles", + "vulkan", + "wgsl", + "webgpu", +] + +#! ### Backends +# -------------------------------------------------------------------- + +## Enables the DX12 backend on Windows. +dx12 = ["wgpu-core?/dx12"] + +## Enables the Metal backend on macOS & iOS. +metal = ["wgpu-core?/metal"] + +## Enables the Vulkan backend on Windows, Linux, and Android. +vulkan = ["wgpu-core?/vulkan"] + +## Enables the OpenGL/GLES backend on Windows, Linux, Android, and Emscripten. +gles = ["wgpu-core?/gles"] + +## Enables the WebGPU backend on WebAssembly. +webgpu = [ + "web", + "naga?/wgsl-out", + "dep:wasm-bindgen-futures", + "web-sys/Document", + "web-sys/Event", + "web-sys/Navigator", + "web-sys/NodeList", + "web-sys/Window", + "web-sys/WorkerGlobalScope", + "web-sys/WorkerNavigator", +] + +#! ### Conditional Backends + +## Enables the GLES backend on macOS only for use with [ANGLE](https://github.com/google/angle). +angle = ["wgpu-core?/angle"] + +## Enables the Vulkan backend on macOS & iOS only for use with [MoltenVK](https://github.com/KhronosGroup/MoltenVK). +vulkan-portability = ["wgpu-core?/vulkan-portability"] + +## Enables the GLES backend on WebAssembly only. +webgl = ["web", "wgpu-core/webgl", "dep:wgpu-hal", "dep:smallvec"] + +## Enables the noop backend for testing. +## +## This backend allows creating resources such as buffers and textures, +## but performs no computation. +## Because it lacks basic functionality, it is only actually used if explicitly enabled +## through `NoopBackendOptions`. +noop = ["wgpu-core/noop", "dep:wgpu-hal", "dep:smallvec"] + +#! **Note:** In the documentation, if you see that an item depends on a backend, +#! it means that the item is only available when that backend is enabled _and_ the backend +#! is supported on the current platform. + +custom = [] + +#! ### Shading language support +# -------------------------------------------------------------------- +#! These features enable support for that input language on all platforms. +#! We will translate the input language to whatever the backend requires. + +## Enable accepting SPIR-V shaders as input. +spirv = ["naga/spv-in", "wgpu-core?/spirv"] + +## Enable accepting GLSL shaders as input. +glsl = ["naga/glsl-in", "wgpu-core?/glsl"] + +## Enable accepting WGSL shaders as input. +wgsl = ["wgpu-core?/wgsl"] + +## Enable accepting naga IR shaders as input. +naga-ir = ["dep:naga"] + +#! ### Assertions and Serialization +# -------------------------------------------------------------------- +## Apply run-time checks, even in release builds. These are in addition +## to the validation carried out at public APIs in all builds. +strict_asserts = ["wgpu-core?/strict_asserts", "wgpu-types/strict_asserts"] + +## Enables serialization via `serde` on common wgpu types. +serde = ["wgpu-core?/serde", "wgpu-types/serde"] + +# ## Allow writing of trace capture files. See [`Adapter::request_device`]. +trace = ["serde", "wgpu-core?/trace"] + +#! ### External libraries +# -------------------------------------------------------------------- +#! The following features facilitate integration with third-party supporting libraries. + +## Enables statically linking DXC. +## +## Normally, to use the modern DXC shader compiler with wgpu, the final application +## must be shipped alongside `dxcompiler.dll` (min v1.8.2502) (which can be downloaded from [Microsoft's GitHub][dxc]). +## This feature statically links a version of DXC so that no external binaries are required +## to compile DX12 shaders. +## +## [dxc]: https://github.com/Microsoft/DirectXShaderCompiler +static-dxc = ["wgpu-core?/static-dxc"] + +#! ### Other +# -------------------------------------------------------------------- + +## Internally count resources and events for debugging purposes. If the counters +## feature is disabled, the counting infrastructure is removed from the build and +## the exposed counters always return 0. +counters = ["wgpu-core?/counters"] + +## Implement `Send` and `Sync` on Wasm, but only if atomics are not enabled. +## +## WebGL/WebGPU objects can not be shared between threads. +## However, it can be useful to artificially mark them as `Send` and `Sync` +## anyways to make it easier to write cross-platform code. +## This is technically *very* unsafe in a multithreaded environment, +## but on a wasm binary compiled without atomics is a definitionally single-threaded environment. +fragile-send-sync-non-atomic-wasm = [ + "wgpu-core?/fragile-send-sync-non-atomic-wasm", + "wgpu-types/fragile-send-sync-non-atomic-wasm", +] + +## Use web-specific libraries on WASM +## +## Those libraries (wasm-bindgen, web-sys, js-sys) can only be used when there is a JavaScript +## context around the WASM VM, e.g., when the WASM binary is used in a browser. +web = ["dep:wasm-bindgen", "dep:js-sys", "dep:web-sys", "wgpu-types/web"] + +## Enables use of the standard library within `wgpu` and its dependencies. +## +## This can allow for better error reporting and for improved multithreading +## support. +std = [ + "raw-window-handle/std", + "wgpu-types/std", + "wgpu-core?/std", + "js-sys?/std", + "web-sys?/std", + "wasm-bindgen?/std", + "wasm-bindgen-futures?/std", +] + +## Uses `parking_lot` as the implementation for locking primitives. +## +## This is a recommended feature for most users and should only be disabled when +## required, e.g., for `no_std` support. +## If disabled, either `std::sync::Mutex` or `core::cell::RefCell` will be used, +## based on whether `std` is enabled or not. +parking_lot = ["dep:parking_lot"] + +######################### +# Standard Dependencies # +######################### + +[dependencies] +naga = { workspace = true, optional = true, features = ["termcolor"] } +wgpu-core = { workspace = true, optional = true } +wgpu-types.workspace = true + +# Needed for both wgpu-core and webgpu +arrayvec.workspace = true +bitflags.workspace = true +bytemuck.workspace = true +cfg-if.workspace = true +document-features.workspace = true +log.workspace = true +hashbrown.workspace = true +parking_lot = { workspace = true, optional = true } +profiling.workspace = true +raw-window-handle = { workspace = true, features = ["alloc"] } +static_assertions.workspace = true + +######################################## +# Target Specific Feature Dependencies # +######################################## + +################### +# Not Webassembly # +################### +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# Needed for only wgpu-core backend. Not optional as only wgpu-core backend exists on native platforms. +wgpu-core = { workspace = true, features = [ + "renderdoc", + "wgsl", # needed for indirect draw/dispatch validation + "portable-atomic", +] } +wgpu-hal.workspace = true + +smallvec.workspace = true + +############### +# Webassembly # +############### +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies] +# Needed for all backends +js-sys = { workspace = true, optional = true } +wasm-bindgen = { workspace = true, optional = true } +web-sys = { workspace = true, optional = true, features = [ + "HtmlCanvasElement", + "OffscreenCanvas", +] } + +# Needed for only wgpu-core backend. Optional as webgl is optional on WebAssembly. +wgpu-core = { workspace = true, optional = true } +wgpu-hal = { workspace = true, optional = true } + +smallvec = { workspace = true, optional = true } + +# Needed for the webgpu backend. Optional as webgpu is optional on WebAssembly. +wasm-bindgen-futures = { workspace = true, optional = true } + +############## +# Emscripten # +############## +[target.'cfg(target_os = "emscripten")'.dependencies] +wgpu-core.workspace = true +wgpu-hal.workspace = true + +smallvec.workspace = true + +[target.'cfg(not(target_has_atomic = "64"))'.dependencies] +portable-atomic.workspace = true + +[dev-dependencies] +# Used in doc-tests +bytemuck.workspace = true + +[build-dependencies] +cfg_aliases.workspace = true diff --git a/third_party/wgpu-29.0.4-patched/LICENSE.APACHE b/third_party/wgpu-29.0.4-patched/LICENSE.APACHE new file mode 100644 index 00000000..d9a10c0d --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/LICENSE.APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/third_party/wgpu-29.0.4-patched/LICENSE.MIT b/third_party/wgpu-29.0.4-patched/LICENSE.MIT new file mode 100644 index 00000000..8d02e4db --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/LICENSE.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 The gfx-rs developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md b/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md new file mode 100644 index 00000000..01d9b006 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md @@ -0,0 +1,19 @@ +# Patch provenance + +Source: `wgpu` 29.0.4 from crates.io, checksum +`76e8840e1ba2881d4cbb18d2147627a56af426ff064c0401eb0c8410c6325d07`. + +Local change: add a hidden, unsafe, range-scoped +`Buffer::mark_external_write_initialized` hook and its private-construction +error type. The hook validates the requested buffer range and delegates only +to the patched `wgpu-core` lazy-initialization tracker; it does not submit, +synchronize, transition, or expose a backend handle. `j2k-ml` calls it only +while a fresh CubeCL allocation is uniquely owned, after validating that the +four-byte-rounded tracker range lies inside that exact suballocation and after +registering the Metal producer dependency on Burn's consumer queue. + +This patch exists only because wgpu otherwise clears externally written bytes +on the first tracked use. It must be removed when the targeted wgpu/Burn/CubeCL +versions provide an upstream safe external-write handoff, or an external +allocation API that registers initialization and queue dependencies without a +local tracker hook. diff --git a/third_party/wgpu-29.0.4-patched/README.md b/third_party/wgpu-29.0.4-patched/README.md new file mode 100644 index 00000000..5060ed39 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/README.md @@ -0,0 +1,168 @@ +# wgpu + + +[![Build Status](https://img.shields.io/github/actions/workflow/status/gfx-rs/wgpu/ci.yml?branch=trunk&logo=github&label=CI)](https://github.com/gfx-rs/wgpu/actions) +[![codecov.io](https://img.shields.io/codecov/c/github/gfx-rs/wgpu?logo=codecov&logoColor=fff&label=codecov&token=84qJTesmeS)](https://codecov.io/gh/gfx-rs/wgpu) + +`wgpu` is a cross-platform, safe, pure-Rust graphics API. It runs natively on Vulkan, Metal, D3D12, and OpenGL; and on top of WebGL2 and WebGPU on wasm. + +The API is based on the [WebGPU standard][webgpu], but is a fully native Rust library. It serves as the core of the WebGPU integration in Firefox, Servo, and Deno. + +## Getting Started + +See our examples online at . You can see the Rust sources at [examples](examples) and run them directly with `cargo run --bin wgpu-examples `. + +### Learning `wgpu` + +If you are new to `wgpu` and graphics programming, we recommend starting with [Learn Wgpu]. + + +Additionally, [WebGPU Fundamentals] is a tutorial for WebGPU which is very similar to our API, minus differences between Rust and Javascript. + +[Learn Wgpu]: https://sotrh.github.io/learn-wgpu/ +[WebGPU Fundamentals]: https://webgpufundamentals.org/ + +### Wiki + +We have a [wiki](https://github.com/gfx-rs/wgpu/wiki) which has information on useful architecture patterns, debugging tips, and more getting started information. + +### Need Help? Want to Contribute? + +The wgpu community uses Matrix and Discord to discuss. + +- [![`#wgpu:matrix.org`](https://img.shields.io/static/v1?label=wgpu-devs&message=%23wgpu&color=blueviolet&logo=matrix)](https://matrix.to/#/#wgpu:matrix.org) - discussion of wgpu's development. +- [![`#wgpu-users:matrix.org`](https://img.shields.io/static/v1?label=wgpu-users&message=%23wgpu-users&color=blueviolet&logo=matrix)](https://matrix.to/#/#wgpu-users:matrix.org) - discussion of using the library and the surrounding ecosystem. +- [![#wgpu on the Rust Gamedev Discord](https://img.shields.io/discord/676678179678715904?logo=discord&logoColor=E0E3FF&label=%23wgpu&color=5865F2) +](https://discord.gg/X3MYBNXUMJ) - Dedicated support channel on the Rust Gamedev Discord. + + +### Other Languages + +To use wgpu in C or dozens of other languages, look at [wgpu-native](https://github.com/gfx-rs/wgpu-native). These are C bindings to wgpu and has an up-to-date list of libraries bringing support to other languages. + +[Learn WebGPU (for C++)] is a good resource for learning how to use wgpu-native from C++. + +[Learn WebGPU (for C++)]: https://eliemichel.github.io/LearnWebGPU/ +[webgpu]: https://gpuweb.github.io/gpuweb/ + +## Quick Links + +| Docs | Examples | Changelog | +|:---------------------:|:-------------------------:|:-----------------------:| +| [v29][rel-docs] | [v29][rel-examples] | [v29][rel-change] | +| [`trunk`][trunk-docs] | [`trunk`][trunk-examples] | [`trunk`][trunk-change] | + +Contributors are welcome! See [CONTRIBUTING.md][contrib] for more information. + +[rel-docs]: https://docs.rs/wgpu/ +[rel-examples]: https://github.com/gfx-rs/wgpu/tree/v29/examples#readme +[rel-change]: https://github.com/gfx-rs/wgpu/releases +[trunk-docs]: https://wgpu.rs/doc/wgpu/ +[trunk-examples]: https://github.com/gfx-rs/wgpu/tree/trunk/examples#readme +[trunk-change]: https://github.com/gfx-rs/wgpu/blob/trunk/CHANGELOG.md#unreleased +[contrib]: CONTRIBUTING.md + +## Supported Platforms + +| API | Windows | Linux/Android | macOS/iOS | Web (wasm) | +| ------ | ------------------ | ------------------ | ------------------ | ------------------ | +| Vulkan | ✅ | ✅ | 🌋 | | +| Metal | | | ✅ | | +| DX12 | ✅ | | | | +| OpenGL | 🆗 (GL 3.3+) | 🆗 (GL ES 3.0+) | 📐 | 🆗 (WebGL2) | +| WebGPU | | | | ✅ | + +✅ = First Class Support +🆗 = Downlevel/Best Effort Support +📐 = Requires the [ANGLE](https://github.com/gfx-rs/wgpu/wiki/Running-on-ANGLE) translation layer (GL ES 3.0 only) +🌋 = Requires the [MoltenVK](https://vulkan.lunarg.com/sdk/home#mac) translation layer +🛠️ = Unsupported, though open to contributions + +## Environment Variables + +Testing, examples, and `::from_env()` methods use a standardized set of environment variables to control wgpu's behavior. + +- `WGPU_BACKEND` with a comma-separated list of the backends you want to use (`vulkan`, `metal`, `dx12`, or `gl`). +- `WGPU_ADAPTER_NAME` with a case-insensitive substring of the name of the adapter you want to use (ex. `1080` will match `NVIDIA GeForce 1080ti`). +- `WGPU_DX12_COMPILER` with the DX12 shader compiler you wish to use (`dxc`, `static-dxc`, or `fxc`). Note that `dxc` requires `dxcompiler.dll` (min v1.8.2502) to be in the working directory, and `static-dxc` requires the `static-dxc` crate feature to be enabled. Otherwise, it will fall back to `fxc`. + +See the [documentation](https://docs.rs/wgpu/latest/wgpu/index.html?search=env) for more environment variables. + +When running the CTS, use the variables `DENO_WEBGPU_ADAPTER_NAME`, `DENO_WEBGPU_BACKEND`, `DENO_WEBGPU_POWER_PREFERENCE`, and `DENO_WEBGPU_DX12_COMPILER`. + +## Repo Overview + +For an overview of all the components in the gfx-rs ecosystem, see [the big picture](./docs/big-picture.png). + +## MSRV policy + +TL;DR: If you're using `wgpu`, our MSRV is **1.87**. If you're running our tests or examples, our MSRV is **1.93**. + +We will avoid bumping the MSRV of `wgpu` without good reason, and such a change is considered breaking. + +
+ Specific Details + +Due to complex dependants, we have three MSRV policies: + +- `wgpu`'s MSRV is **1.87** +- `wgpu-core` (and hence `wgpu-hal`, `naga`, and `wgpu-types`)'s MSRV is **1.87**. +- The rest of the workspace has an MSRV of **1.93**. + +It is enforced on CI (in "/.github/workflows/ci.yml") with the `WGPU_MSRV`, `CORE_MSRV`, and `REPO_MSRV` variables, respectively. +This version can only be upgraded in breaking releases, though we release a breaking version every three months. + +The following rules apply: +- The `wgpu-core` crate should never require an MSRV ahead of Firefox's MSRV for nightly builds, as +determined by the value of `MINIMUM_RUST_VERSION` in [`python/mozboot/mozboot/util.py`][moz-msrv]. +- The `wgpu` crate should never require an MSRV ahead of Servo's MSRV, as determined by the value of +their rust-version declaration in [`Cargo.toml`][servo-msrv] +- The repository MSRV should never require an MSRV higher than `stable - 3`. For example, if stable is +at 1.97, the repository MSRV should be no higher than 1.94. This is to allow people who are using a decently-updated +OS-provided rust to be able to build our repository. Consider cross checking with [NixOS][nixos-msrv], though this +is not required. + +[moz-msrv]: https://searchfox.org/mozilla-central/source/python/mozboot/mozboot/util.py +[servo-msrv]: https://github.com/servo/servo/blob/main/Cargo.toml#L23 +[nixos-msrv]: https://search.nixos.org/packages?show=rustc + +
+ +## Testing and Environment Variables + +[Information about testing](./docs/testing.md), including where tests of various kinds live, and how to run the tests. + +## Tracking the WebGPU and WGSL draft specifications + +The `wgpu` crate is meant to be an idiomatic Rust translation of the [WebGPU API][webgpu spec]. +That specification, along with its shading language, [WGSL][wgsl spec], +are both still in the "Working Draft" phase, +and while the general outlines are stable, +details change frequently. +Until the specification is stabilized, the `wgpu` crate and the version of WGSL it implements +will likely differ from what is specified, +as the implementation catches up. + +Exactly which WGSL features `wgpu` supports depends on how you are using it: + +- When running as native code, `wgpu` uses [Naga][naga] + to translate WGSL code into the shading language of your platform's native GPU API. + Naga is working on catching up to the WGSL specification, + with [bugs][naga bugs] tracking various issues, + but there is no concise summary of differences from the specification. + +- When running in a web browser (by compilation to WebAssembly) + without the `"webgl"` feature enabled, + `wgpu` relies on the browser's own WebGPU implementation. + WGSL shaders are simply passed through to the browser, + so that determines which WGSL features you can use. + +- When running in a web browser with `wgpu`'s `"webgl"` feature enabled, + `wgpu` uses Naga to translate WGSL programs into GLSL. + This uses the same version of Naga as if you were running `wgpu` as native code. + +[webgpu spec]: https://www.w3.org/TR/webgpu/ +[wgsl spec]: https://gpuweb.github.io/gpuweb/wgsl/ +[naga]: https://github.com/gfx-rs/wgpu/tree/trunk/naga/ +[naga bugs]: https://github.com/gfx-rs/wgpu/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22naga%22 + diff --git a/third_party/wgpu-29.0.4-patched/build.rs b/third_party/wgpu-29.0.4-patched/build.rs new file mode 100644 index 00000000..fd5bf693 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/build.rs @@ -0,0 +1,62 @@ +fn main() { + cfg_aliases::cfg_aliases! { + native: { not(target_arch = "wasm32") }, + Emscripten: { all(target_arch = "wasm32", target_os = "emscripten") }, + web: { all(target_arch = "wasm32", not(Emscripten), feature = "web") }, + + send_sync: { any( + native, + all(feature = "fragile-send-sync-non-atomic-wasm", not(target_feature = "atomics")) + ) }, + + // Backends - keep this in sync with `wgpu-core/Cargo.toml` & docs in `wgpu/Cargo.toml` + webgpu: { all(not(native), not(Emscripten), feature = "webgpu") }, + webgl: { all(not(native), not(Emscripten), feature = "webgl") }, + dx12: { all(target_os = "windows", feature = "dx12") }, + metal: { all(target_vendor = "apple", feature = "metal") }, + vulkan: { any( + // The `vulkan` feature enables the Vulkan backend only on "native Vulkan" platforms, i.e. Windows/Linux/Android + all(any(windows, target_os = "linux", target_os = "android", target_os = "freebsd"), feature = "vulkan"), + // On Apple platforms, however, we require the `vulkan-portability` feature + // to explicitly opt-in to Vulkan since it's meant to be used with MoltenVK. + all(target_vendor = "apple", feature = "vulkan-portability") + ) }, + gles: { any( + // The `gles` feature enables the OpenGL/GLES backend only on "native OpenGL" platforms, i.e. Windows, Linux, Android, and Emscripten. + // (Note that WebGL is also not included here!) + all(any(windows, target_os = "linux", target_os = "android", target_os = "freebsd", Emscripten), feature = "gles"), + // On Apple platforms, however, we require the `angle` feature to explicitly opt-in to OpenGL + // since its meant to be used with ANGLE. + all(target_vendor = "apple", feature = "angle") + ) }, + noop: { feature = "noop" }, + + wgpu_core: { + any( + // On native, wgpu_core is currently always enabled, even if there's no backend enabled at all. + native, + // `wgpu_core` is implied if any backend other than WebGPU is enabled. + // (this is redundant except for `gles` and `noop`) + webgl, dx12, metal, vulkan, gles, noop + ) + }, + + // This alias is _only_ if _we_ need naga in the wrapper. wgpu-core provides + // its own re-export of naga, which can be used in other situations + naga: { any(feature = "naga-ir", feature = "spirv", feature = "glsl") }, + // ⚠️ Keep in sync with target.cfg() definition in wgpu-hal/Cargo.toml and cfg_alias in `wgpu-hal` crate ⚠️ + static_dxc: { all(target_os = "windows", feature = "static-dxc", not(target_arch = "aarch64"), target_env = "msvc") }, + supports_64bit_atomics: { target_has_atomic = "64" }, + custom: {any(feature = "custom")}, + std: { any( + feature = "std", + // TODO: Remove this when an alternative Mutex implementation is available for `no_std`. + // send_sync requires an appropriate Mutex implementation, which is only currently + // possible with `std` enabled. + send_sync, + // Unwinding panics necessitate access to `std` to determine if a thread is panicking + panic = "unwind" + ) }, + no_std: { not(std) } + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/adapter.rs b/third_party/wgpu-29.0.4-patched/src/api/adapter.rs new file mode 100644 index 00000000..dd019d1e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/adapter.rs @@ -0,0 +1,222 @@ +use alloc::vec::Vec; +use core::future::Future; +#[cfg(wgpu_core)] +use core::ops::Deref; + +use crate::*; + +/// Handle to a physical graphics and/or compute device. +/// +/// Adapters can be created using [`Instance::request_adapter`] +/// or other [`Instance`] methods. +/// +/// Adapters can be used to open a connection to the corresponding [`Device`] +/// on the host system by using [`Adapter::request_device`]. +/// +/// Does not have to be kept alive. +/// +/// Corresponds to [WebGPU `GPUAdapter`](https://gpuweb.github.io/gpuweb/#gpu-adapter). +#[derive(Debug, Clone)] +pub struct Adapter { + pub(crate) inner: dispatch::DispatchAdapter, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(Adapter: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(Adapter => .inner); + +pub use wgt::RequestAdapterOptions as RequestAdapterOptionsBase; +/// Additional information required when requesting an adapter. +/// +/// For use with [`Instance::request_adapter`]. +/// +/// Corresponds to [WebGPU `GPURequestAdapterOptions`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpurequestadapteroptions). +pub type RequestAdapterOptions<'a, 'b> = RequestAdapterOptionsBase<&'a Surface<'b>>; +#[cfg(send_sync)] +static_assertions::assert_impl_all!(RequestAdapterOptions<'_, '_>: Send, Sync); + +impl Adapter { + /// Requests a connection to a physical device, creating a logical device. + /// + /// Returns the [`Device`] together with a [`Queue`] that executes command buffers. + /// + /// [Per the WebGPU specification], an [`Adapter`] may only be used once to create a device. + /// If another device is wanted, call [`Instance::request_adapter()`] again to get a fresh + /// [`Adapter`]. + /// However, `wgpu` does not currently enforce this restriction. + /// + /// # Panics + /// + /// - `request_device()` was already called on this `Adapter`. + /// - Features specified by `desc` are not supported by this adapter. + /// - Unsafe features were requested but not enabled when requesting the adapter. + /// - Limits requested exceed the values provided by the adapter. + /// - Adapter does not support all features wgpu requires to safely operate. + /// + /// [Per the WebGPU specification]: https://www.w3.org/TR/webgpu/#dom-gpuadapter-requestdevice + pub fn request_device( + &self, + desc: &DeviceDescriptor<'_>, + ) -> impl Future> + WasmNotSend { + let device = self.inner.request_device(desc); + async move { + device + .await + .map(|(device, queue)| (Device { inner: device }, Queue { inner: queue })) + } + } + + /// Create a wgpu [`Device`] and [`Queue`] from a wgpu-hal [`hal::OpenDevice`]. + /// + /// # Safety + /// + /// - `hal_device` must be created from this adapter internal handle. + /// - `desc.features` must be a subset of `hal_device`'s supported features. + #[cfg(wgpu_core)] + pub unsafe fn create_device_from_hal( + &self, + hal_device: hal::OpenDevice, + desc: &DeviceDescriptor<'_>, + ) -> Result<(Device, Queue), RequestDeviceError> { + let core_adapter = self.inner.as_core(); + let (device, queue) = unsafe { + core_adapter + .context + .create_device_from_hal(core_adapter, hal_device, desc) + }?; + + Ok(( + Device { + inner: device.into(), + }, + Queue { + inner: queue.into(), + }, + )) + } + + /// Get the [`wgpu_hal`] adapter from this `Adapter`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::Adapter`]. + /// + /// # Types + /// + /// The returned type depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Adapter")] + #[doc = crate::macros::hal_type_metal!("Adapter")] + #[doc = crate::macros::hal_type_dx12!("Adapter")] + #[doc = crate::macros::hal_type_gles!("Adapter")] + /// + /// # Errors + /// + /// This method will return None if: + /// - The adapter is not from the backend specified by `A`. + /// - The adapter is from the `webgpu` or `custom` backend. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::Adapter`]: hal::Api::Adapter + #[cfg(wgpu_core)] + pub unsafe fn as_hal( + &self, + ) -> Option + WasmNotSendSync> { + let adapter = self.inner.as_core_opt()?; + + unsafe { adapter.context.adapter_as_hal::(adapter) } + } + + #[cfg(custom)] + /// Returns custom implementation of adapter (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } + + #[cfg(custom)] + /// Creates Adapter from custom implementation + pub fn from_custom(adapter: T) -> Self { + Self { + inner: dispatch::DispatchAdapter::custom(adapter), + } + } + + /// Returns whether this adapter may present to the passed surface. + pub fn is_surface_supported(&self, surface: &Surface<'_>) -> bool { + self.inner.is_surface_supported(&surface.inner) + } + + /// The features which can be used to create devices on this adapter. + pub fn features(&self) -> Features { + self.inner.features() + } + + /// The best limits which can be used to create devices on this adapter. + pub fn limits(&self) -> Limits { + self.inner.limits() + } + + /// Get info about the adapter itself. + pub fn get_info(&self) -> AdapterInfo { + self.inner.get_info() + } + + /// Get info about the adapter itself. + pub fn get_downlevel_capabilities(&self) -> DownlevelCapabilities { + self.inner.downlevel_capabilities() + } + + /// Returns the features supported for a given texture format by this adapter. + /// + /// Note that the WebGPU spec further restricts the available usages/features. + /// To disable these restrictions on a device, request the [`Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES`] feature. + pub fn get_texture_format_features(&self, format: TextureFormat) -> TextureFormatFeatures { + self.inner.get_texture_format_features(format) + } + + /// Generates a timestamp using the clock used by the presentation engine. + /// + /// When comparing completely opaque timestamp systems, we need a way of generating timestamps that signal + /// the exact same time. You can do this by calling your own timestamp function immediately after a call to + /// this function. This should result in timestamps that are 0.5 to 5 microseconds apart. There are locks + /// that must be taken during the call, so don't call your function before. + /// + /// ```no_run + /// # let adapter: wgpu::Adapter = panic!(); + /// # let some_code = || wgpu::PresentationTimestamp::INVALID_TIMESTAMP; + /// use std::time::{Duration, Instant}; + /// let presentation = adapter.get_presentation_timestamp(); + /// let instant = Instant::now(); + /// + /// // We can now turn a new presentation timestamp into an Instant. + /// let some_pres_timestamp = some_code(); + /// let duration = Duration::from_nanos((some_pres_timestamp.0 - presentation.0) as u64); + /// let new_instant: Instant = instant + duration; + /// ``` + // + /// [Instant]: std::time::Instant + pub fn get_presentation_timestamp(&self) -> PresentationTimestamp { + self.inner.get_presentation_timestamp() + } + + /// Returns the supported cooperative matrix configurations for this adapter. + /// + /// Cooperative matrices enable hardware-accelerated matrix multiply-accumulate + /// operations where threads in a subgroup collectively process matrix tiles. + /// + /// Returns an empty vector if cooperative matrices are not supported. + /// + /// Requires [`Features::EXPERIMENTAL_COOPERATIVE_MATRIX`] to be meaningful. + pub fn cooperative_matrix_properties(&self) -> Vec { + self.inner.cooperative_matrix_properties() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/bind_group.rs b/third_party/wgpu-29.0.4-patched/src/api/bind_group.rs new file mode 100644 index 00000000..93e9a8f1 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/bind_group.rs @@ -0,0 +1,167 @@ +use crate::*; + +/// Handle to a binding group. +/// +/// A `BindGroup` represents the set of resources bound to the bindings described by a +/// [`BindGroupLayout`]. It can be created with [`Device::create_bind_group`]. A `BindGroup` can +/// be bound to a particular [`RenderPass`] with [`RenderPass::set_bind_group`], or to a +/// [`ComputePass`] with [`ComputePass::set_bind_group`]. +/// +/// Corresponds to [WebGPU `GPUBindGroup`](https://gpuweb.github.io/gpuweb/#gpubindgroup). +#[derive(Debug, Clone)] +pub struct BindGroup { + pub(crate) inner: dispatch::DispatchBindGroup, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(BindGroup: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(BindGroup => .inner); + +impl BindGroup { + #[cfg(custom)] + /// Returns custom implementation of BindGroup (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Resource to be bound by a [`BindGroup`] for use with a pipeline. +/// +/// The pipeline’s [`BindGroupLayout`] must contain a matching [`BindingType`]. +/// +/// Corresponds to [WebGPU `GPUBindingResource`]( +/// https://gpuweb.github.io/gpuweb/#typedefdef-gpubindingresource). +#[non_exhaustive] +#[derive(Clone, Debug)] +pub enum BindingResource<'a> { + /// Binding is backed by a buffer. + /// + /// Corresponds to [`wgt::BufferBindingType::Uniform`] and [`wgt::BufferBindingType::Storage`] + /// with [`BindGroupLayoutEntry::count`] set to None. + Buffer(BufferBinding<'a>), + /// Binding is backed by an array of buffers. + /// + /// [`Features::BUFFER_BINDING_ARRAY`] must be supported to use this feature. + /// + /// Corresponds to [`wgt::BufferBindingType::Uniform`] and [`wgt::BufferBindingType::Storage`] + /// with [`BindGroupLayoutEntry::count`] set to Some. + BufferArray(&'a [BufferBinding<'a>]), + /// Binding is a sampler. + /// + /// Corresponds to [`wgt::BindingType::Sampler`] with [`BindGroupLayoutEntry::count`] set to None. + Sampler(&'a Sampler), + /// Binding is backed by an array of samplers. + /// + /// [`Features::TEXTURE_BINDING_ARRAY`] must be supported to use this feature. + /// + /// Corresponds to [`wgt::BindingType::Sampler`] with [`BindGroupLayoutEntry::count`] set + /// to Some. + SamplerArray(&'a [&'a Sampler]), + /// Binding is backed by a texture. + /// + /// Corresponds to [`wgt::BindingType::Texture`] and [`wgt::BindingType::StorageTexture`] with + /// [`BindGroupLayoutEntry::count`] set to None. + TextureView(&'a TextureView), + /// Binding is backed by an array of textures. + /// + /// [`Features::TEXTURE_BINDING_ARRAY`] must be supported to use this feature. + /// + /// Corresponds to [`wgt::BindingType::Texture`] and [`wgt::BindingType::StorageTexture`] with + /// [`BindGroupLayoutEntry::count`] set to Some. + TextureViewArray(&'a [&'a TextureView]), + /// Binding is backed by a top level acceleration structure + /// + /// Corresponds to [`wgt::BindingType::AccelerationStructure`] with [`BindGroupLayoutEntry::count`] set to None. + /// + /// # Validation + /// When using (e.g. with `set_bind_group`) a bind group that has been created with one or more of this binding + /// resource certain checks take place. + /// - TLAS must have been built, if not a validation error is generated + /// - All BLASes that were built into the TLAS must be built before the TLAS, if this was not satisfied and TLAS was + /// built using `build_acceleration_structures` a validation error is generated otherwise this is a part of the + /// safety section of `build_acceleration_structures_unsafe_tlas` and so undefined behavior occurs. + AccelerationStructure(&'a Tlas), + /// Binding is backed by an array of top level acceleration structures. + /// + /// Corresponds to [`wgt::BindingType::AccelerationStructure`] with [`BindGroupLayoutEntry::count`] set to Some. + /// + /// # Validation + /// The same validation rules apply as for [`BindingResource::AccelerationStructure`], for each element. + /// + /// Note: backend support may vary; this is primarily intended for native backends. + AccelerationStructureArray(&'a [&'a Tlas]), + /// Binding is backed by an external texture. + /// + /// [`Features::EXTERNAL_TEXTURE`] must be supported to use this feature. + /// + /// Corresponds to [`wgt::BindingType::ExternalTexture`]. + ExternalTexture(&'a ExternalTexture), +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(BindingResource<'_>: Send, Sync); + +/// Describes the segment of a buffer to bind. +/// +/// Corresponds to [WebGPU `GPUBufferBinding`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpubufferbinding). +#[derive(Clone, Debug)] +pub struct BufferBinding<'a> { + /// The buffer to bind. + pub buffer: &'a Buffer, + + /// Base offset of the buffer, in bytes. + /// + /// If the [`has_dynamic_offset`] field of this buffer's layout entry is + /// `true`, the offset here will be added to the dynamic offset passed to + /// [`RenderPass::set_bind_group`] or [`ComputePass::set_bind_group`]. + /// + /// If the buffer was created with [`BufferUsages::UNIFORM`], then this + /// offset must be a multiple of + /// [`Limits::min_uniform_buffer_offset_alignment`]. + /// + /// If the buffer was created with [`BufferUsages::STORAGE`], then this + /// offset must be a multiple of + /// [`Limits::min_storage_buffer_offset_alignment`]. + /// + /// [`has_dynamic_offset`]: BindingType::Buffer::has_dynamic_offset + pub offset: BufferAddress, + + /// Size of the binding in bytes, or `None` for using the rest of the buffer. + pub size: Option, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(BufferBinding<'_>: Send, Sync); + +/// An element of a [`BindGroupDescriptor`], consisting of a bindable resource +/// and the slot to bind it to. +/// +/// Corresponds to [WebGPU `GPUBindGroupEntry`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpubindgroupentry). +#[derive(Clone, Debug)] +pub struct BindGroupEntry<'a> { + /// Slot for which binding provides resource. Corresponds to an entry of the same + /// binding index in the [`BindGroupLayoutDescriptor`]. + pub binding: u32, + /// Resource to attach to the binding + pub resource: BindingResource<'a>, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(BindGroupEntry<'_>: Send, Sync); + +/// Describes a group of bindings and the resources to be bound. +/// +/// For use with [`Device::create_bind_group`]. +/// +/// Corresponds to [WebGPU `GPUBindGroupDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpubindgroupdescriptor). +#[derive(Clone, Debug)] +pub struct BindGroupDescriptor<'a> { + /// Debug label of the bind group. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The [`BindGroupLayout`] that corresponds to this bind group. + pub layout: &'a BindGroupLayout, + /// The resources to bind to this bind group. + pub entries: &'a [BindGroupEntry<'a>], +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(BindGroupDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/bind_group_layout.rs b/third_party/wgpu-29.0.4-patched/src/api/bind_group_layout.rs new file mode 100644 index 00000000..62eb7dd4 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/bind_group_layout.rs @@ -0,0 +1,45 @@ +use crate::*; + +/// Handle to a binding group layout. +/// +/// A `BindGroupLayout` is a handle to the GPU-side layout of a binding group. It can be used to +/// create a [`BindGroupDescriptor`] object, which in turn can be used to create a [`BindGroup`] +/// object with [`Device::create_bind_group`]. A series of `BindGroupLayout`s can also be used to +/// create a [`PipelineLayoutDescriptor`], which can be used to create a [`PipelineLayout`]. +/// +/// It can be created with [`Device::create_bind_group_layout`]. +/// +/// Corresponds to [WebGPU `GPUBindGroupLayout`]( +/// https://gpuweb.github.io/gpuweb/#gpubindgrouplayout). +#[derive(Debug, Clone)] +pub struct BindGroupLayout { + pub(crate) inner: dispatch::DispatchBindGroupLayout, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(BindGroupLayout: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(BindGroupLayout => .inner); + +impl BindGroupLayout { + #[cfg(custom)] + /// Returns custom implementation of BindGroupLayout (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Describes a [`BindGroupLayout`]. +/// +/// For use with [`Device::create_bind_group_layout`]. +/// +/// Corresponds to [WebGPU `GPUBindGroupLayoutDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpubindgrouplayoutdescriptor). +#[derive(Clone, Debug)] +pub struct BindGroupLayoutDescriptor<'a> { + /// Debug label of the bind group layout. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + + /// Array of entries in this BindGroupLayout + pub entries: &'a [BindGroupLayoutEntry], +} +static_assertions::assert_impl_all!(BindGroupLayoutDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/blas.rs b/third_party/wgpu-29.0.4-patched/src/api/blas.rs new file mode 100644 index 00000000..879081b0 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/blas.rs @@ -0,0 +1,286 @@ +#[cfg(wgpu_core)] +use core::ops::Deref; + +use alloc::{boxed::Box, vec::Vec}; + +use wgt::{WasmNotSend, WasmNotSendSync}; + +use crate::dispatch; +use crate::{Buffer, Label}; + +/// Descriptor for the size defining attributes of a triangle geometry, for a bottom level acceleration structure. +pub type BlasTriangleGeometrySizeDescriptor = wgt::BlasTriangleGeometrySizeDescriptor; +static_assertions::assert_impl_all!(BlasTriangleGeometrySizeDescriptor: Send, Sync); + +/// Descriptor for the size defining attributes, for a bottom level acceleration structure. +pub type BlasGeometrySizeDescriptors = wgt::BlasGeometrySizeDescriptors; +static_assertions::assert_impl_all!(BlasGeometrySizeDescriptors: Send, Sync); + +/// Flags for an acceleration structure. +pub type AccelerationStructureFlags = wgt::AccelerationStructureFlags; +static_assertions::assert_impl_all!(AccelerationStructureFlags: Send, Sync); + +/// Flags for a geometry inside a bottom level acceleration structure. +pub type AccelerationStructureGeometryFlags = wgt::AccelerationStructureGeometryFlags; +static_assertions::assert_impl_all!(AccelerationStructureGeometryFlags: Send, Sync); + +/// Update mode for acceleration structure builds. +pub type AccelerationStructureUpdateMode = wgt::AccelerationStructureUpdateMode; +static_assertions::assert_impl_all!(AccelerationStructureUpdateMode: Send, Sync); + +/// Descriptor to create bottom level acceleration structures. +pub type CreateBlasDescriptor<'a> = wgt::CreateBlasDescriptor>; +static_assertions::assert_impl_all!(CreateBlasDescriptor<'_>: Send, Sync); + +/// Safe instance for a [Tlas]. +/// +/// A TlasInstance may be made invalid, if a TlasInstance is invalid, any attempt to build a [Tlas] containing an +/// invalid TlasInstance will generate a validation error +/// +/// Each one contains: +/// - A reference to a BLAS, this ***must*** be interacted with using [TlasInstance::new] or [TlasInstance::set_blas], a +/// TlasInstance that references a BLAS keeps that BLAS from being dropped +/// - A user accessible transformation matrix +/// - A user accessible mask +/// - A user accessible custom index +/// +/// [Tlas]: crate::Tlas +#[derive(Debug, Clone)] +pub struct TlasInstance { + pub(crate) blas: dispatch::DispatchBlas, + /// Affine transform matrix 3x4 (rows x columns, row major order). + pub transform: [f32; 12], + /// Custom index for the instance used inside the shader. + /// + /// This must only use the lower 24 bits, if any bits are outside that range (byte 4 does not equal 0) the TlasInstance becomes + /// invalid and generates a validation error when built + pub custom_data: u32, + /// Mask for the instance used inside the shader to filter instances. + /// Reports hit only if `(shader_cull_mask & tlas_instance.mask) != 0u`. + pub mask: u8, +} + +impl TlasInstance { + /// Construct TlasInstance. + /// - blas: Reference to the bottom level acceleration structure + /// - transform: Transform buffer offset in bytes (optional, required if transform buffer is present) + /// - custom_data: Custom index for the instance used inside the shader (max 24 bits) + /// - mask: Mask for the instance used inside the shader to filter instances + /// + /// Note: while one of these contains a reference to a BLAS that BLAS will not be dropped, + /// but it can still be destroyed. Destroying a BLAS that is referenced by one or more + /// TlasInstance(s) will immediately make them invalid. If one or more of those invalid + /// TlasInstances is inside a TlasPackage that is attempted to be built, the build will + /// generate a validation error. + pub fn new(blas: &Blas, transform: [f32; 12], custom_data: u32, mask: u8) -> Self { + Self { + blas: blas.inner.clone(), + transform, + custom_data, + mask, + } + } + + /// Set the bottom level acceleration structure. + /// + /// See the note on [TlasInstance] about the + /// guarantees of keeping a BLAS alive. + pub fn set_blas(&mut self, blas: &Blas) { + self.blas = blas.inner.clone(); + } +} + +#[derive(Debug)] +/// Definition for a triangle geometry for a Bottom Level Acceleration Structure (BLAS). +/// +/// The size must match the rest of the structures fields, otherwise the build will fail. +/// (e.g. if index count is present in the size, the index buffer must be present as well.) +pub struct BlasTriangleGeometry<'a> { + /// Sub descriptor for the size defining attributes of a triangle geometry. + pub size: &'a BlasTriangleGeometrySizeDescriptor, + /// Vertex buffer. + pub vertex_buffer: &'a Buffer, + /// Offset into the vertex buffer as a factor of the vertex stride. + pub first_vertex: u32, + /// Vertex stride, must be greater than [`wgpu_types::VertexFormat::min_acceleration_structure_vertex_stride`] + /// of the format and must be a multiple of [`wgpu_types::VertexFormat::acceleration_structure_stride_alignment`]. + pub vertex_stride: wgt::BufferAddress, + /// Index buffer (optional). + pub index_buffer: Option<&'a Buffer>, + /// Number of indexes to skip in the index buffer (optional, required if index buffer is present). + pub first_index: Option, + /// Transform buffer containing 3x4 (rows x columns, row major) affine transform matrices `[f32; 12]` (optional). + pub transform_buffer: Option<&'a Buffer>, + /// Transform buffer offset in bytes (optional, required if transform buffer is present). + pub transform_buffer_offset: Option, +} +static_assertions::assert_impl_all!(BlasTriangleGeometry<'_>: WasmNotSendSync); + +/// Contains the sets of geometry that go into a [Blas]. +pub enum BlasGeometries<'a> { + /// Triangle geometry variant. + TriangleGeometries(Vec>), +} +static_assertions::assert_impl_all!(BlasGeometries<'_>: WasmNotSendSync); + +/// Builds the given sets of geometry into the given [Blas]. +pub struct BlasBuildEntry<'a> { + /// Reference to the acceleration structure. + pub blas: &'a Blas, + /// Geometries. + pub geometry: BlasGeometries<'a>, +} +static_assertions::assert_impl_all!(BlasBuildEntry<'_>: WasmNotSendSync); + +#[derive(Debug, Clone)] +/// Bottom Level Acceleration Structure (BLAS). +/// +/// A BLAS is a device-specific raytracing acceleration structure that contains geometry data. +/// +/// These BLASes are combined with transform in a [TlasInstance] to create a [Tlas]. +/// +/// [Tlas]: crate::Tlas +pub struct Blas { + pub(crate) handle: Option, + pub(crate) inner: dispatch::DispatchBlas, +} +static_assertions::assert_impl_all!(Blas: WasmNotSendSync); + +crate::cmp::impl_eq_ord_hash_proxy!(Blas => .inner); + +impl Blas { + /// Raw handle to the acceleration structure, used inside raw instance buffers. + pub fn handle(&self) -> Option { + self.handle + } + + /// Get the [`wgpu_hal`] acceleration structure from this `Blas`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::AccelerationStructure`]. + /// + /// # Types + /// + /// The returned type depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("AccelerationStructure")] + #[doc = crate::macros::hal_type_metal!("AccelerationStructure")] + #[doc = crate::macros::hal_type_dx12!("AccelerationStructure")] + #[doc = crate::macros::hal_type_gles!("AccelerationStructure")] + /// + /// # Deadlocks + /// + /// - The returned guard holds a read-lock on a device-local "destruction" + /// lock, which will cause all calls to `destroy` to block until the + /// guard is released. + /// + /// # Errors + /// + /// This method will return None if: + /// - The acceleration structure is not from the backend specified by `A`. + /// - The acceleration structure is from the `webgpu` or `custom` backend. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::AccelerationStructure`]: hal::Api::AccelerationStructure + #[cfg(wgpu_core)] + pub unsafe fn as_hal( + &mut self, + ) -> Option + WasmNotSendSync> { + let blas = self.inner.as_core_opt()?; + unsafe { blas.context.blas_as_hal::(blas) } + } + + #[cfg(custom)] + /// Returns custom implementation of Blas (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Context version of [BlasTriangleGeometry]. +pub struct ContextBlasTriangleGeometry<'a> { + #[expect(dead_code)] + pub(crate) size: &'a BlasTriangleGeometrySizeDescriptor, + #[expect(dead_code)] + pub(crate) vertex_buffer: &'a dispatch::DispatchBuffer, + #[expect(dead_code)] + pub(crate) index_buffer: Option<&'a dispatch::DispatchBuffer>, + #[expect(dead_code)] + pub(crate) transform_buffer: Option<&'a dispatch::DispatchBuffer>, + #[expect(dead_code)] + pub(crate) first_vertex: u32, + #[expect(dead_code)] + pub(crate) vertex_stride: wgt::BufferAddress, + #[expect(dead_code)] + pub(crate) index_buffer_offset: Option, + #[expect(dead_code)] + pub(crate) transform_buffer_offset: Option, +} + +/// Context version of [BlasGeometries]. +pub enum ContextBlasGeometries<'a> { + /// Triangle geometries. + TriangleGeometries(Box> + 'a>), +} + +/// Context version see [BlasBuildEntry]. +pub struct ContextBlasBuildEntry<'a> { + #[expect(dead_code)] + pub(crate) blas: &'a dispatch::DispatchBlas, + #[expect(dead_code)] + pub(crate) geometries: ContextBlasGeometries<'a>, +} + +/// Error occurred when trying to asynchronously prepare a blas for compaction. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct BlasAsyncError; +static_assertions::assert_impl_all!(BlasAsyncError: Send, Sync); + +impl core::fmt::Display for BlasAsyncError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "Error occurred when trying to asynchronously prepare a blas for compaction" + ) + } +} + +impl core::error::Error for BlasAsyncError {} + +impl Blas { + /// Asynchronously prepares this BLAS for compaction. The callback is called once all builds + /// using this BLAS are finished and the BLAS is compactable. This can be checked using + /// [`Blas::ready_for_compaction`]. Rebuilding this BLAS will reset its compacted state, and it + /// will need to be prepared again. + /// + /// ### Interaction with other functions + /// On native, `queue.submit(..)` and polling devices (that is calling `instance.poll_all` or + /// `device.poll`) with [`PollType::Poll`] may call the callback. On native, polling devices with + /// [`PollType::Wait`] (optionally with a submission index greater + /// than the last submit the BLAS was used in) will guarantee callback is called. + /// + /// [`PollType::Poll`]: wgpu_types::PollType::Poll + /// [`PollType::Wait`]: wgpu_types::PollType::Wait + pub fn prepare_compaction_async( + &self, + callback: impl FnOnce(Result<(), BlasAsyncError>) + WasmNotSend + 'static, + ) { + self.inner.prepare_compact_async(Box::new(callback)); + } + + /// Checks whether this BLAS is ready for compaction. The returned value is `true` if + /// [`Blas::prepare_compaction_async`]'s callback was called with a non-error value, otherwise + /// this is `false`. + pub fn ready_for_compaction(&self) -> bool { + self.inner.ready_for_compaction() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/buffer.rs b/third_party/wgpu-29.0.4-patched/src/api/buffer.rs new file mode 100644 index 00000000..32ade16b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/buffer.rs @@ -0,0 +1,1177 @@ +use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use core::{ + error, fmt, + ops::{Bound, Deref, Range, RangeBounds}, +}; + +use crate::util::Mutex; +use crate::*; + +/// Handle to a GPU-accessible buffer. +/// +/// A `Buffer` is a memory allocation for use by the GPU, somewhat analogous to +/// [Box]<[\[u8\]][primitive@slice]> in Rust. +/// The contents of buffers are untyped bytes; it is up to the application to +/// specify the interpretation of the bytes when the buffer is used, in ways +/// such as [`VertexBufferLayout`]. +/// A single buffer can be used to hold multiple independent pieces of data at +/// different offsets (e.g. both vertices and indices for one or more meshes). +/// +/// A `Buffer`'s bytes have "interior mutability": functions like +/// [`Queue::write_buffer`] or [mapping] a buffer for writing only require a +/// `&Buffer`, not a `&mut Buffer`, even though they modify its contents. `wgpu` +/// prevents simultaneous reads and writes of buffer contents using run-time +/// checks. +/// +/// Created with [`Device::create_buffer()`] or +/// [`DeviceExt::create_buffer_init()`]. +/// +/// Corresponds to [WebGPU `GPUBuffer`](https://gpuweb.github.io/gpuweb/#buffer-interface). +/// +/// [mapping]: Buffer#mapping-buffers +/// +/// # How to get your data into a buffer +/// +/// Every `Buffer` starts with all bytes zeroed. +/// There are many ways to load data into a `Buffer`: +/// +/// - When creating a buffer, you may set the [`mapped_at_creation`][mac] flag, +/// then write to its [`get_mapped_range_mut()`][Buffer::get_mapped_range_mut]. +/// This only works when the buffer is created and has not yet been used by +/// the GPU, but it is all you need for buffers whose contents do not change +/// after creation. +/// - You may use [`DeviceExt::create_buffer_init()`] as a convenient way to +/// do that and copy data from a `&[u8]` you provide. +/// - After creation, you may use [`Buffer::map_async()`] to map it again; +/// however, you then need to wait until the GPU is no longer using the buffer +/// before you begin writing. +/// - You may use [`CommandEncoder::copy_buffer_to_buffer()`] to copy data into +/// this buffer from another buffer. +/// - You may use [`Queue::write_buffer()`] to copy data into the buffer from a +/// `&[u8]`. This uses a temporary “staging” buffer managed by `wgpu` to hold +/// the data. +/// - [`Queue::write_buffer_with()`] allows you to write directly into temporary +/// storage instead of providing a slice you already prepared, which may +/// allow *your* code to save the allocation of a [`Vec`] or such. +/// - You may use [`util::StagingBelt`] to manage a set of temporary buffers. +/// This may be more efficient than [`Queue::write_buffer_with()`] when you +/// have many small copies to perform, but requires more steps to use, and +/// tuning of the belt buffer size. +/// - You may write your own staging buffer management customized to your +/// application, based on mapped buffers and +/// [`CommandEncoder::copy_buffer_to_buffer()`]. +/// - A GPU computation’s results can be stored in a buffer: +/// - A [compute shader][ComputePipeline] may write to a buffer bound as a +/// [storage buffer][BufferBindingType::Storage]. +/// - A render pass may render to a texture which is then copied to a buffer +/// using [`CommandEncoder::copy_texture_to_buffer()`]. +/// +/// # Mapping buffers +/// +/// If a `Buffer` is created with the appropriate [`usage`], it can be *mapped*: +/// you can make its contents accessible to the CPU as an ordinary `&[u8]` or +/// `&mut [u8]` slice of bytes. Buffers created with the +/// [`mapped_at_creation`][mac] flag set are also mapped initially. +/// +/// Depending on the hardware, the buffer could be memory shared between CPU and +/// GPU, so that the CPU has direct access to the same bytes the GPU will +/// consult; or it may be ordinary CPU memory, whose contents the system must +/// copy to/from the GPU as needed. This crate's API is designed to work the +/// same way in either case: at any given time, a buffer is either mapped and +/// available to the CPU, or unmapped and ready for use by the GPU, but never +/// both. This makes it impossible for either side to observe changes by the +/// other immediately, and any necessary transfers can be carried out when the +/// buffer transitions from one state to the other. +/// +/// There are two ways to map a buffer: +/// +/// - If [`BufferDescriptor::mapped_at_creation`] is `true`, then the entire +/// buffer is mapped when it is created. This is the easiest way to initialize +/// a new buffer. You can set `mapped_at_creation` on any kind of buffer, +/// regardless of its [`usage`] flags. +/// +/// - If the buffer's [`usage`] includes the [`MAP_READ`] or [`MAP_WRITE`] +/// flags, then you can call `buffer.slice(range).map_async(mode, callback)` +/// to map the portion of `buffer` given by `range`. This waits for the GPU to +/// finish using the buffer, and invokes `callback` as soon as the buffer is +/// safe for the CPU to access. +/// +/// Once a buffer is mapped: +/// +/// - You can call `buffer.slice(range).get_mapped_range()` to obtain a +/// [`BufferView`], which dereferences to a `&[u8]` that you can use to read +/// the buffer's contents. +/// +/// - Or, you can call `buffer.slice(range).get_mapped_range_mut()` to obtain a +/// [`BufferViewMut`], which dereferences to a `&mut [u8]` that you can use to +/// read and write the buffer's contents. +/// +/// The given `range` must fall within the mapped portion of the buffer. If you +/// attempt to access overlapping ranges, even for shared access only, these +/// methods panic. +/// +/// While a buffer is mapped, you may not submit any commands to the GPU that +/// access it. You may record command buffers that use the buffer, but if you +/// submit them while the buffer is mapped, submission will panic. +/// +/// When you are done using the buffer on the CPU, you must call +/// [`Buffer::unmap`] to make it available for use by the GPU again. All +/// [`BufferView`] and [`BufferViewMut`] views referring to the buffer must be +/// dropped before you unmap it; otherwise, [`Buffer::unmap`] will panic. +/// +/// # Example +/// +/// If `buffer` was created with [`BufferUsages::MAP_WRITE`], we could fill it +/// with `f32` values like this: +/// +/// ``` +/// # #[cfg(feature = "noop")] +/// # let (device, _queue) = wgpu::Device::noop(&wgpu::DeviceDescriptor::default()); +/// # #[cfg(not(feature = "noop"))] +/// # let device: wgpu::Device = { return; }; +/// # +/// # let buffer = device.create_buffer(&wgpu::BufferDescriptor { +/// # label: None, +/// # size: 400, +/// # usage: wgpu::BufferUsages::MAP_WRITE, +/// # mapped_at_creation: false, +/// # }); +/// let capturable = buffer.clone(); +/// buffer.map_async(wgpu::MapMode::Write, .., move |result| { +/// if result.is_ok() { +/// let mut view = capturable.get_mapped_range_mut(..); +/// let mut floats: wgpu::WriteOnly<[[u8; 4]]> = view.slice(..).into_chunks::<4>().0; +/// floats.fill(42.0f32.to_ne_bytes()); +/// drop(view); +/// capturable.unmap(); +/// } +/// }); +/// ``` +/// +/// This code takes the following steps: +/// +/// - First, it makes a cloned handle to the buffer for capture by +/// the callback passed to [`map_async`]. Since a [`map_async`] callback may be +/// invoked from another thread, interaction between the callback and the +/// thread calling [`map_async`] generally requires some sort of shared heap +/// data like this. In real code, there might be an [`Arc`] to some larger +/// structure that itself owns `buffer`. +/// +/// - Then, it calls [`Buffer::slice`] to make a [`BufferSlice`] referring to +/// the buffer's entire contents. +/// +/// - Next, it calls [`BufferSlice::map_async`] to request that the bytes to +/// which the slice refers be made accessible to the CPU ("mapped"). This may +/// entail waiting for previously enqueued operations on `buffer` to finish. +/// Although [`map_async`] itself always returns immediately, it saves the +/// callback function to be invoked later. +/// +/// - When some later call to [`Device::poll`] or [`Instance::poll_all`] (not +/// shown in this example) determines that the buffer is mapped and ready for +/// the CPU to use, it invokes the callback function. +/// +/// - The callback function calls [`Buffer::slice`] and then +/// [`BufferSlice::get_mapped_range_mut`] to obtain a [`BufferViewMut`], which +/// dereferences to a `&mut [u8]` slice referring to the buffer's bytes. +/// +/// - It then uses the [`bytemuck`] crate to turn the `&mut [u8]` into a `&mut +/// [f32]`, and calls the slice [`fill`] method to fill the buffer with a +/// useful value. +/// +/// - Finally, the callback drops the view and calls [`Buffer::unmap`] to unmap +/// the buffer. In real code, the callback would also need to do some sort of +/// synchronization to let the rest of the program know that it has completed +/// its work. +/// +/// If using [`map_async`] directly is awkward, you may find it more convenient to +/// use [`Queue::write_buffer`] and [`util::DownloadBuffer::read_buffer`]. +/// However, those each have their own tradeoffs; the asynchronous nature of GPU +/// execution makes it hard to avoid friction altogether. +/// +/// [`Arc`]: std::sync::Arc +/// [`map_async`]: BufferSlice::map_async +/// [`bytemuck`]: https://crates.io/crates/bytemuck +/// [`fill`]: slice::fill +/// +/// ## Mapping buffers on the web +/// +/// When compiled to WebAssembly and running in a browser content process, +/// `wgpu` implements its API in terms of the browser's WebGPU implementation. +/// In this context, `wgpu` is further isolated from the GPU: +/// +/// - Depending on the browser's WebGPU implementation, mapping and unmapping +/// buffers probably entails copies between WebAssembly linear memory and the +/// graphics driver's buffers. +/// +/// - All modern web browsers isolate web content in its own sandboxed process, +/// which can only interact with the GPU via interprocess communication (IPC). +/// Although most browsers' IPC systems use shared memory for large data +/// transfers, there will still probably need to be copies into and out of the +/// shared memory buffers. +/// +/// All of these copies contribute to the cost of buffer mapping in this +/// configuration. +/// +/// [`usage`]: BufferDescriptor::usage +/// [mac]: BufferDescriptor::mapped_at_creation +/// [`MAP_READ`]: BufferUsages::MAP_READ +/// [`MAP_WRITE`]: BufferUsages::MAP_WRITE +/// [`DeviceExt::create_buffer_init()`]: util::DeviceExt::create_buffer_init +#[derive(Debug, Clone)] +pub struct Buffer { + pub(crate) inner: dispatch::DispatchBuffer, + pub(crate) map_context: Arc>, + pub(crate) size: wgt::BufferAddress, + pub(crate) usage: BufferUsages, + // Todo: missing map_state https://www.w3.org/TR/webgpu/#dom-gpubuffer-mapstate +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(Buffer: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(Buffer => .inner); + +impl Buffer { + /// Return the binding view of the entire buffer. + pub fn as_entire_binding(&self) -> BindingResource<'_> { + BindingResource::Buffer(self.as_entire_buffer_binding()) + } + + /// Return the binding view of the entire buffer. + pub fn as_entire_buffer_binding(&self) -> BufferBinding<'_> { + BufferBinding { + buffer: self, + offset: 0, + size: None, + } + } + + /// Get the [`wgpu_hal`] buffer from this `Buffer`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::Buffer`]. + /// + /// # Types + /// + /// The returned type depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Buffer")] + #[doc = crate::macros::hal_type_metal!("Buffer")] + #[doc = crate::macros::hal_type_dx12!("Buffer")] + #[doc = crate::macros::hal_type_gles!("Buffer")] + /// + /// # Deadlocks + /// + /// - The returned guard holds a read-lock on a device-local "destruction" + /// lock, which will cause all calls to `destroy` to block until the + /// guard is released. + /// + /// # Errors + /// + /// This method will return None if: + /// - The buffer is not from the backend specified by `A`. + /// - The buffer is from the `webgpu` or `custom` backend. + /// - The buffer has had [`Self::destroy()`] called on it. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::Buffer`]: hal::Api::Buffer + #[cfg(wgpu_core)] + pub unsafe fn as_hal( + &self, + ) -> Option + WasmNotSendSync> { + let buffer = self.inner.as_core_opt()?; + unsafe { buffer.context.buffer_as_hal::(buffer) } + } + + /// Tell wgpu that a completed external GPU producer fully initialized a + /// byte range of this buffer. + /// + /// This narrow hook is for audited native-runtime interop. It changes only + /// wgpu's lazy initialization tracker; it does not submit work, establish a + /// resource transition, or synchronize either producer. + /// + /// # Safety + /// + /// - The caller must hold exclusive allocation ownership of `bounds` for + /// the external write. Every semantically accessible byte must have been + /// initialized. Backend-required binding padding may be included only + /// when runtime bounds checks make it unobservable. + /// - Every prior wgpu use must be resolved before the raw write begins. + /// - The external write must be complete, or an equivalent GPU dependency + /// must be registered, before any later wgpu use can observe the range. + /// - The raw resource must not have been destroyed or replaced. + /// + /// # Panics + /// + /// Panics when `bounds` is empty or outside this buffer. + #[cfg(wgpu_core)] + #[doc(hidden)] + pub unsafe fn mark_external_write_initialized>( + &self, + bounds: S, + ) -> Result<(), ExternalWriteInitializationError> { + let (offset, size) = range_to_offset_size(bounds, self.size); + check_buffer_bounds(self.size, offset, size); + let buffer = self + .inner + .as_core_opt() + .ok_or(ExternalWriteInitializationError { _private: () })?; + unsafe { + buffer + .context + .buffer_mark_external_write_initialized(buffer, offset..offset + size.get()) + } + } + + /// Returns a [`BufferSlice`] referring to the portion of `self`'s contents + /// indicated by `bounds`. Regardless of what sort of data `self` stores, + /// `bounds` start and end are given in bytes. + /// + /// A [`BufferSlice`] can be used to supply vertex and index data, or to map + /// buffer contents for access from the CPU. See the [`BufferSlice`] + /// documentation for details. + /// + /// The `range` argument can be half or fully unbounded: for example, + /// `buffer.slice(..)` refers to the entire buffer, and `buffer.slice(n..)` + /// refers to the portion starting at the `n`th byte and extending to the + /// end of the buffer. + /// + /// # Panics + /// + /// - If `bounds` is outside of the bounds of `self`. + /// - If `bounds` has a length less than 1. + #[track_caller] + pub fn slice>(&self, bounds: S) -> BufferSlice<'_> { + let (offset, size) = range_to_offset_size(bounds, self.size); + check_buffer_bounds(self.size, offset, size); + BufferSlice { + buffer: self, + offset, + size, + } + } + + /// Unmaps the buffer from host memory. + /// + /// This terminates the effect of all previous [`map_async()`](Self::map_async) operations and + /// makes the buffer available for use by the GPU again. + pub fn unmap(&self) { + self.map_context.lock().reset(); + self.inner.unmap(); + } + + /// Destroy the associated native resources as soon as possible. + pub fn destroy(&self) { + self.inner.destroy(); + } + + /// Returns the length of the buffer allocation in bytes. + /// + /// This is always equal to the `size` that was specified when creating the buffer. + pub fn size(&self) -> BufferAddress { + self.size + } + + /// Returns the allowed usages for this `Buffer`. + /// + /// This is always equal to the `usage` that was specified when creating the buffer. + pub fn usage(&self) -> BufferUsages { + self.usage + } + + /// Map the buffer to host (CPU) memory, making it available for reading or writing via + /// [`get_mapped_range()`](Self::get_mapped_range). The buffer becomes accessible once the + /// `callback` is invoked with [`Ok`]. + /// + /// Use this when you want to map the buffer immediately. If you need to submit GPU work that + /// uses the buffer before mapping it, use `map_buffer_on_submit` on + /// [`CommandEncoder`][CEmbos], [`CommandBuffer`][CBmbos], [`RenderPass`][RPmbos], or + /// [`ComputePass`][CPmbos] to schedule the mapping after submission. This avoids extra calls to + /// [`Buffer::map_async()`] or [`BufferSlice::map_async()`] and lets you initiate mapping from a + /// more convenient place. + /// + /// For the callback to run, either [`queue.submit(..)`][q::s], [`instance.poll_all(..)`][i::p_a], + /// or [`device.poll(..)`][d::p] must be called elsewhere in the runtime, possibly integrated into + /// an event loop or run on a separate thread. + /// + /// The callback runs on the thread that first calls one of the above functions after the GPU work + /// completes. There are no restrictions on the code you can run in the callback; however, on native + /// the polling call will not return until the callback finishes, so keep callbacks short (set flags, + /// send messages, etc.). + /// + /// While a buffer is mapped, it cannot be used by other commands; at any time, either the GPU or + /// the CPU has exclusive access to the buffer’s contents. + /// + /// This can also be performed using [`BufferSlice::map_async()`]. + /// + /// # Panics + /// + /// - If the buffer is already mapped. + /// - If the buffer’s [`BufferUsages`] do not allow the requested [`MapMode`]. + /// - If `bounds` is outside of the bounds of `self`. + /// - If `bounds` does not start at a multiple of [`MAP_ALIGNMENT`]. + /// - If `bounds` has a length that is not a multiple of 4 greater than 0. + /// + /// [CEmbos]: CommandEncoder::map_buffer_on_submit + /// [CBmbos]: CommandBuffer::map_buffer_on_submit + /// [RPmbos]: RenderPass::map_buffer_on_submit + /// [CPmbos]: ComputePass::map_buffer_on_submit + /// [q::s]: Queue::submit + /// [i::p_a]: Instance::poll_all + /// [d::p]: Device::poll + pub fn map_async>( + &self, + mode: MapMode, + bounds: S, + callback: impl FnOnce(Result<(), BufferAsyncError>) + WasmNotSend + 'static, + ) { + self.slice(bounds).map_async(mode, callback) + } + + /// Gain read-only access to the bytes of a [mapped] [`Buffer`]. + /// + /// Returns a [`BufferView`] referring to the buffer range represented by + /// `self`. See the documentation for [`BufferView`] for details. + /// + /// `bounds` may be less than the bounds passed to [`Self::map_async()`], + /// and multiple views may be obtained and used simultaneously as long as they do not overlap. + /// + /// This can also be performed using [`BufferSlice::get_mapped_range()`]. + /// + /// # Panics + /// + /// - If `bounds` is outside of the bounds of `self`. + /// - If `bounds` does not start at a multiple of [`MAP_ALIGNMENT`]. + /// - If `bounds` has a length that is not a multiple of 4 greater than 0. + /// - If the buffer to which `self` refers is not currently [mapped]. + /// - If you try to create a view which overlaps an existing [`BufferViewMut`]. + /// + /// [mapped]: Buffer#mapping-buffers + #[track_caller] + pub fn get_mapped_range>(&self, bounds: S) -> BufferView { + self.slice(bounds).get_mapped_range() + } + + /// Gain write access to the bytes of a [mapped] [`Buffer`]. + /// + /// Returns a [`BufferViewMut`] referring to the buffer range represented by + /// `self`. See the documentation for [`BufferViewMut`] for more details. + /// + /// `bounds` may be less than the bounds passed to [`Self::map_async()`], + /// and multiple views may be obtained and used simultaneously as long as they do not overlap. + /// + /// This can also be performed using [`BufferSlice::get_mapped_range_mut()`]. + /// + /// # Panics + /// + /// - If `bounds` is outside of the bounds of `self`. + /// - If `bounds` does not start at a multiple of [`MAP_ALIGNMENT`]. + /// - If `bounds` has a length that is not a multiple of 4 greater than 0. + /// - If the buffer to which `self` refers is not currently [mapped]. + /// - If you try to create a view which overlaps an existing [`BufferView`] or [`BufferViewMut`]. + /// + /// [mapped]: Buffer#mapping-buffers + #[track_caller] + pub fn get_mapped_range_mut>(&self, bounds: S) -> BufferViewMut { + self.slice(bounds).get_mapped_range_mut() + } + + #[cfg(custom)] + /// Returns custom implementation of Buffer (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// A slice of a [`Buffer`], to be mapped, used for vertex or index data, or the like. +/// +/// You can create a `BufferSlice` by calling [`Buffer::slice`]: +/// +/// ```no_run +/// # let buffer: wgpu::Buffer = todo!(); +/// let slice = buffer.slice(10..20); +/// ``` +/// +/// This returns a slice referring to the second ten bytes of `buffer`. To get a +/// slice of the entire `Buffer`: +/// +/// ```no_run +/// # let buffer: wgpu::Buffer = todo!(); +/// let whole_buffer_slice = buffer.slice(..); +/// ``` +/// +/// You can pass buffer slices to methods like [`RenderPass::set_vertex_buffer`] +/// and [`RenderPass::set_index_buffer`] to indicate which portion of the buffer +/// a draw call should consult. You can also convert it to a [`BufferBinding`] +/// with `.into()`. +/// +/// To access the slice's contents on the CPU, you must first [map] the buffer, +/// and then call [`BufferSlice::get_mapped_range`] or +/// [`BufferSlice::get_mapped_range_mut`] to obtain a view of the slice's +/// contents. See the documentation on [mapping][map] for more details, +/// including example code. +/// +/// Unlike a Rust shared slice `&[T]`, whose existence guarantees that +/// nobody else is modifying the `T` values to which it refers, a +/// [`BufferSlice`] doesn't guarantee that the buffer's contents aren't +/// changing. You can still record and submit commands operating on the +/// buffer while holding a [`BufferSlice`]. A [`BufferSlice`] simply +/// represents a certain range of the buffer's bytes. +/// +/// The `BufferSlice` type is unique to the Rust API of `wgpu`. In the WebGPU +/// specification, an offset and size are specified as arguments to each call +/// working with the [`Buffer`], instead. +/// +/// [map]: Buffer#mapping-buffers +#[derive(Copy, Clone, Debug, PartialEq)] +pub struct BufferSlice<'a> { + pub(crate) buffer: &'a Buffer, + pub(crate) offset: BufferAddress, + pub(crate) size: BufferSize, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(BufferSlice<'_>: Send, Sync); + +impl<'a> BufferSlice<'a> { + /// Return another [`BufferSlice`] referring to the portion of `self`'s contents + /// indicated by `bounds`. + /// + /// The `range` argument can be half or fully unbounded: for example, + /// `buffer.slice(..)` refers to the entire buffer, and `buffer.slice(n..)` + /// refers to the portion starting at the `n`th byte and extending to the + /// end of the buffer. + /// + /// # Panics + /// + /// - If `bounds` is outside of the bounds of `self`. + /// - If `bounds` has a length less than 1. + #[track_caller] + pub fn slice>(&self, bounds: S) -> BufferSlice<'a> { + let (offset, size) = range_to_offset_size(bounds, self.size.get()); + check_buffer_bounds(self.size.get(), offset, size); + BufferSlice { + buffer: self.buffer, + offset: self.offset + offset, // check_buffer_bounds ensures this does not overflow + size, // check_buffer_bounds ensures this is essentially min() + } + } + + /// Map the buffer to host (CPU) memory, making it available for reading or writing via + /// [`get_mapped_range()`](Self::get_mapped_range). The buffer becomes accessible once the + /// `callback` is invoked with [`Ok`]. + /// + /// Use this when you want to map the buffer immediately. If you need to submit GPU work that + /// uses the buffer before mapping it, use `map_buffer_on_submit` on + /// [`CommandEncoder`][CEmbos], [`CommandBuffer`][CBmbos], [`RenderPass`][RPmbos], or + /// [`ComputePass`][CPmbos] to schedule the mapping after submission. This avoids extra calls to + /// [`Buffer::map_async()`] or [`BufferSlice::map_async()`] and lets you initiate mapping from a + /// more convenient place. + /// + /// For the callback to run, either [`queue.submit(..)`][q::s], [`instance.poll_all(..)`][i::p_a], + /// or [`device.poll(..)`][d::p] must be called elsewhere in the runtime, possibly integrated into + /// an event loop or run on a separate thread. + /// + /// The callback runs on the thread that first calls one of the above functions after the GPU work + /// completes. There are no restrictions on the code you can run in the callback; however, on native + /// the polling call will not return until the callback finishes, so keep callbacks short (set flags, + /// send messages, etc.). + /// + /// While a buffer is mapped, it cannot be used by other commands; at any time, either the GPU or + /// the CPU has exclusive access to the buffer’s contents. + /// + /// This can also be performed using [`Buffer::map_async()`]. + /// + /// # Panics + /// + /// - If the buffer is already mapped. + /// - If the buffer’s [`BufferUsages`] do not allow the requested [`MapMode`]. + /// - If the beginning of this slice is not aligned to [`MAP_ALIGNMENT`] within the buffer. + /// - If the length of this slice is not a multiple of 4. + /// + /// [CEmbos]: CommandEncoder::map_buffer_on_submit + /// [CBmbos]: CommandBuffer::map_buffer_on_submit + /// [RPmbos]: RenderPass::map_buffer_on_submit + /// [CPmbos]: ComputePass::map_buffer_on_submit + /// [q::s]: Queue::submit + /// [i::p_a]: Instance::poll_all + /// [d::p]: Device::poll + pub fn map_async( + &self, + mode: MapMode, + callback: impl FnOnce(Result<(), BufferAsyncError>) + WasmNotSend + 'static, + ) { + let mut mc = self.buffer.map_context.lock(); + assert_eq!(mc.mapped_range, 0..0, "Buffer is already mapped"); + let end = self.offset + self.size.get(); + mc.mapped_range = self.offset..end; + drop(mc); // release the lock of map_context as callback can call lock it again + + self.buffer + .inner + .map_async(mode, self.offset..end, Box::new(callback)); + } + + /// Gain read-only access to the bytes of a [mapped] [`Buffer`]. + /// + /// Returns a [`BufferView`] referring to the buffer range represented by + /// `self`. See the documentation for [`BufferView`] for details. + /// + /// Multiple views may be obtained and used simultaneously as long as they are from + /// non-overlapping slices. + /// + /// This can also be performed using [`Buffer::get_mapped_range()`]. + /// + /// # Panics + /// + /// - If the beginning of this slice is not aligned to [`MAP_ALIGNMENT`] within the buffer. + /// - If the length of this slice is not a multiple of 4. + /// - If the buffer to which `self` refers is not currently [mapped]. + /// - If you try to create a view which overlaps an existing [`BufferViewMut`]. + /// + /// [mapped]: Buffer#mapping-buffers + #[track_caller] + pub fn get_mapped_range(&self) -> BufferView { + let subrange = Subrange::new(self.offset, self.size, RangeMappingKind::Immutable); + self.buffer + .map_context + .lock() + .validate_and_add(subrange.clone()); + let range = self.buffer.inner.get_mapped_range(subrange.index); + BufferView { + buffer: self.buffer.clone(), + size: self.size, + offset: self.offset, + inner: range, + } + } + + /// Gain write-only access to the bytes of a [mapped] [`Buffer`]. + /// + /// Returns a [`BufferViewMut`] referring to the buffer range represented by + /// `self`. See the documentation for [`BufferViewMut`] for more details. + /// + /// Multiple views may be obtained and used simultaneously as long as they are from + /// non-overlapping slices. + /// + /// This can also be performed using [`Buffer::get_mapped_range_mut()`]. + /// + /// # Panics + /// + /// - If the beginning of this slice is not aligned to [`MAP_ALIGNMENT`] within the buffer. + /// - If the length of this slice is not a multiple of 4. + /// - If the buffer to which `self` refers is not currently [mapped]. + /// - If you try to create a view which overlaps an existing [`BufferView`] or [`BufferViewMut`]. + /// + /// [mapped]: Buffer#mapping-buffers + #[track_caller] + pub fn get_mapped_range_mut(&self) -> BufferViewMut { + let subrange = Subrange::new(self.offset, self.size, RangeMappingKind::Mutable); + self.buffer + .map_context + .lock() + .validate_and_add(subrange.clone()); + let range = self.buffer.inner.get_mapped_range(subrange.index); + BufferViewMut { + buffer: self.buffer.clone(), + size: self.size, + offset: self.offset, + inner: range, + } + } + + /// Returns the buffer this is a slice of. + /// + /// You should usually not need to call this, and if you received the buffer from code you + /// do not control, you should refrain from accessing the buffer outside the bounds of the + /// slice. Nevertheless, it’s possible to get this access, so this method makes it simple. + pub fn buffer(&self) -> &'a Buffer { + self.buffer + } + + /// Returns the offset in [`Self::buffer()`] this slice starts at. + pub fn offset(&self) -> BufferAddress { + self.offset + } + + /// Returns the size of this slice. + pub fn size(&self) -> BufferSize { + self.size + } +} + +impl<'a> From> for crate::BufferBinding<'a> { + /// Convert a [`BufferSlice`] to an equivalent [`BufferBinding`], + /// provided that it will be used without a dynamic offset. + fn from(value: BufferSlice<'a>) -> Self { + BufferBinding { + buffer: value.buffer, + offset: value.offset, + size: Some(value.size), + } + } +} + +impl<'a> From> for crate::BindingResource<'a> { + /// Convert a [`BufferSlice`] to an equivalent [`BindingResource::Buffer`], + /// provided that it will be used without a dynamic offset. + fn from(value: BufferSlice<'a>) -> Self { + crate::BindingResource::Buffer(crate::BufferBinding::from(value)) + } +} + +fn range_overlaps(a: &Range, b: &Range) -> bool { + a.start < b.end && b.start < a.end +} + +fn range_contains(a: &Range, b: &Range) -> bool { + a.start <= b.start && a.end >= b.end +} + +#[derive(Debug, Copy, Clone)] +enum RangeMappingKind { + Mutable, + Immutable, +} + +impl RangeMappingKind { + /// Returns true if a range of this kind can touch the same bytes as a range of the other kind. + /// + /// This is Rust's Mutable XOR Shared rule. + fn allowed_concurrently_with(self, other: Self) -> bool { + matches!( + (self, other), + (RangeMappingKind::Immutable, RangeMappingKind::Immutable) + ) + } +} + +#[derive(Debug, Clone)] +struct Subrange { + index: Range, + kind: RangeMappingKind, +} + +impl Subrange { + fn new(offset: BufferAddress, size: BufferSize, kind: RangeMappingKind) -> Self { + Self { + index: offset..(offset + size.get()), + kind, + } + } +} + +impl fmt::Display for Subrange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}..{} ({:?})", + self.index.start, self.index.end, self.kind + ) + } +} + +/// The mapped portion of a buffer, if any, and its outstanding views. +/// +/// This ensures that views fall within the mapped range and don't overlap. +#[derive(Debug)] +pub(crate) struct MapContext { + /// The range of the buffer that is mapped. + /// + /// This is `0..0` if the buffer is not mapped. This becomes non-empty when + /// the buffer is mapped at creation time, and when you call `map_async` on + /// some [`BufferSlice`] (so technically, it indicates the portion that is + /// *or has been requested to be* mapped.) + /// + /// All [`BufferView`]s and [`BufferViewMut`]s must fall within this range. + mapped_range: Range, + + /// The ranges covered by all outstanding [`BufferView`]s and + /// [`BufferViewMut`]s. These are non-overlapping, and are all contained + /// within `mapped_range`. + sub_ranges: Vec, +} + +impl MapContext { + /// Creates a new `MapContext`. + /// + /// For [`mapped_at_creation`] buffers, pass the full buffer range in the + /// `mapped_range` argument. For other buffers, pass `None`. + /// + /// [`mapped_at_creation`]: BufferDescriptor::mapped_at_creation + pub(crate) fn new(mapped_range: Option>) -> Self { + Self { + mapped_range: mapped_range.unwrap_or(0..0), + sub_ranges: Vec::new(), + } + } + + /// Record that the buffer is no longer mapped. + fn reset(&mut self) { + self.mapped_range = 0..0; + + assert!( + self.sub_ranges.is_empty(), + "You cannot unmap a buffer that still has accessible mapped views" + ); + } + + /// Record that the `size` bytes of the buffer at `offset` are now viewed. + /// + /// # Panics + /// + /// This panics if the given range is invalid. + #[track_caller] + fn validate_and_add(&mut self, new_sub: Subrange) { + if self.mapped_range.is_empty() { + panic!("tried to call get_mapped_range(_mut) on an unmapped buffer"); + } + if !range_contains(&self.mapped_range, &new_sub.index) { + panic!( + "tried to call get_mapped_range(_mut) on a range that is not entirely mapped. \ + Attempted to get range {}, but the mapped range is {}..{}", + new_sub, self.mapped_range.start, self.mapped_range.end + ); + } + + // This check is essential for avoiding undefined behavior: it is the + // only thing that ensures that `&mut` references to the buffer's + // contents don't alias anything else. + for sub in self.sub_ranges.iter() { + if range_overlaps(&sub.index, &new_sub.index) + && !sub.kind.allowed_concurrently_with(new_sub.kind) + { + panic!( + "tried to call get_mapped_range(_mut) on a range that has already \ + been mapped and would break Rust memory aliasing rules. Attempted \ + to get range {}, and the conflicting range is {}", + new_sub, sub + ); + } + } + self.sub_ranges.push(new_sub); + } + + /// Record that the `size` bytes of the buffer at `offset` are no longer viewed. + /// + /// # Panics + /// + /// This panics if the given range does not exactly match one previously + /// passed to [`MapContext::validate_and_add`]. + pub(crate) fn remove(&mut self, offset: BufferAddress, size: BufferSize) { + let end = offset + size.get(); + + let index = self + .sub_ranges + .iter() + .position(|r| r.index == (offset..end)) + .expect("unable to remove range from map context"); + self.sub_ranges.swap_remove(index); + } +} + +/// Describes a [`Buffer`]. +/// +/// For use with [`Device::create_buffer`]. +/// +/// Corresponds to [WebGPU `GPUBufferDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpubufferdescriptor). +pub type BufferDescriptor<'a> = wgt::BufferDescriptor>; +static_assertions::assert_impl_all!(BufferDescriptor<'_>: Send, Sync); + +/// Error occurred when trying to async map a buffer. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct BufferAsyncError; +static_assertions::assert_impl_all!(BufferAsyncError: Send, Sync); + +impl fmt::Display for BufferAsyncError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Error occurred when trying to async map a buffer") + } +} + +impl error::Error for BufferAsyncError {} + +/// Wgpu could not accept a guarded external-write initialization handoff. +#[doc(hidden)] +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct ExternalWriteInitializationError { + pub(crate) _private: (), +} +static_assertions::assert_impl_all!(ExternalWriteInitializationError: Send, Sync); + +impl fmt::Display for ExternalWriteInitializationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("wgpu rejected an external-write initialization handoff") + } +} + +impl error::Error for ExternalWriteInitializationError {} + +/// Type of buffer mapping. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum MapMode { + /// Map only for reading + Read, + /// Map only for writing + Write, +} +static_assertions::assert_impl_all!(MapMode: Send, Sync); + +/// A read-only view of a mapped buffer's bytes. +/// +/// To get a `BufferView`, first [map] the buffer, and then +/// call `buffer.slice(range).get_mapped_range()`. +/// +/// `BufferView` dereferences to `&[u8]`, so you can use all the usual Rust +/// slice methods to access the buffer's contents. It also implements +/// `AsRef<[u8]>`, if that's more convenient. +/// +/// Before the buffer can be unmapped, all `BufferView`s observing it +/// must be dropped. Otherwise, the call to [`Buffer::unmap`] will panic. +/// +/// For example code, see the documentation on [mapping buffers][map]. +/// +/// [map]: Buffer#mapping-buffers +/// [`map_async`]: BufferSlice::map_async +#[derive(Debug)] +pub struct BufferView { + // `buffer, offset, size` are similar to `BufferSlice`, except that they own the buffer. + buffer: Buffer, + offset: BufferAddress, + size: BufferSize, + inner: dispatch::DispatchBufferMappedRange, +} + +/// A write-only view of a mapped buffer's bytes. +/// +/// To get a `BufferViewMut`, first [map] the buffer, and then +/// call `buffer.slice(range).get_mapped_range_mut()`. +/// +/// Because Rust has no write-only reference type +/// (`&[u8]` is read-only and `&mut [u8]` is read-write), +/// this type does not dereference to a slice in the way that [`BufferView`] does. +/// Instead, [`.slice()`][BufferViewMut::slice] returns a special [`WriteOnly`] pointer type, +/// and there are also a few convenience methods such as [`BufferViewMut::copy_from_slice()`]. +/// +/// Before the buffer can be unmapped, all `BufferViewMut`s observing it +/// must be dropped. Otherwise, the call to [`Buffer::unmap`] will panic. +/// +/// For example code, see the documentation on [mapping buffers][map]. +/// +/// [map]: Buffer#mapping-buffers +#[derive(Debug)] +pub struct BufferViewMut { + // `buffer, offset, size` are similar to `BufferSlice`, except that they own the buffer. + buffer: Buffer, + offset: BufferAddress, + size: BufferSize, + inner: dispatch::DispatchBufferMappedRange, +} + +// `BufferView` simply dereferences. `BufferViewMut` cannot, because mapped memory may be +// write-combining memory , +// and not support the expected behavior of atomic accesses. +// Further context: + +impl core::ops::Deref for BufferView { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &[u8] { + // SAFETY: this is a read mapping + unsafe { self.inner.read_slice() } + } +} + +impl AsRef<[u8]> for BufferView { + #[inline] + fn as_ref(&self) -> &[u8] { + self + } +} + +impl Drop for BufferView { + fn drop(&mut self) { + self.buffer + .map_context + .lock() + .remove(self.offset, self.size); + } +} + +impl Drop for BufferViewMut { + fn drop(&mut self) { + self.buffer + .map_context + .lock() + .remove(self.offset, self.size); + } +} + +#[cfg(webgpu)] +impl BufferView { + /// Provides the same data as dereferencing the view, but as a `Uint8Array` in js. + /// This can be MUCH faster than dereferencing the view which copies the data into + /// the Rust / wasm heap. + pub fn as_uint8array(&self) -> &js_sys::Uint8Array { + self.inner.as_uint8array() + } +} + +/// These methods are equivalent to the methods of the same names on [`WriteOnly`]. +impl BufferViewMut { + /// Returns the length of this view; the number of bytes to be written. + pub fn len(&self) -> usize { + // cannot fail because we can't actually map more than isize::MAX bytes + usize::try_from(self.size.get()).unwrap() + } + + /// Returns `true` if the view has a length of 0. + /// + /// Note that this is currently impossible. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns a [`WriteOnly`] reference to a portion of this. + /// + /// `.slice(..)` can be used to access the whole data. + pub fn slice<'a, S: RangeBounds>(&'a mut self, bounds: S) -> WriteOnly<'a, [u8]> { + // SAFETY: this is a write mapping + unsafe { self.inner.write_slice() }.into_slice(bounds) + } + + /// Copies all elements from src into `self`. + /// + /// The length of `src` must be the same as `self`. + /// + /// This method is equivalent to + /// [`self.slice(..).copy_from_slice(src)`][WriteOnly::copy_from_slice]. + pub fn copy_from_slice(&mut self, src: &[u8]) { + self.slice(..).copy_from_slice(src) + } +} + +#[track_caller] +fn check_buffer_bounds( + buffer_size: BufferAddress, + slice_offset: BufferAddress, + slice_size: BufferSize, +) { + // A slice of length 0 is invalid, so the offset must not be equal to or greater than the buffer size. + if slice_offset >= buffer_size { + panic!( + "slice offset {} is out of range for buffer of size {}", + slice_offset, buffer_size + ); + } + + // Detect integer overflow. + let end = slice_offset.checked_add(slice_size.get()); + if end.is_none_or(|end| end > buffer_size) { + panic!( + "slice offset {} size {} is out of range for buffer of size {}", + slice_offset, slice_size, buffer_size + ); + } +} + +#[track_caller] +pub(crate) fn range_to_offset_size>( + bounds: S, + whole_size: BufferAddress, +) -> (BufferAddress, BufferSize) { + let offset = match bounds.start_bound() { + Bound::Included(&bound) => bound, + Bound::Excluded(&bound) => bound + 1, + Bound::Unbounded => 0, + }; + let size = BufferSize::new(match bounds.end_bound() { + Bound::Included(&bound) => bound + 1 - offset, + Bound::Excluded(&bound) => bound - offset, + Bound::Unbounded => whole_size - offset, + }) + .expect("buffer slices can not be empty"); + + (offset, size) +} + +#[cfg(test)] +mod tests { + use super::{ + check_buffer_bounds, range_overlaps, range_to_offset_size, BufferAddress, BufferSize, + }; + + fn bs(value: BufferAddress) -> BufferSize { + BufferSize::new(value).unwrap() + } + + #[test] + fn range_to_offset_size_works() { + let whole = 100; + + assert_eq!(range_to_offset_size(0..2, whole), (0, bs(2))); + assert_eq!(range_to_offset_size(2..5, whole), (2, bs(3))); + assert_eq!(range_to_offset_size(.., whole), (0, bs(whole))); + assert_eq!(range_to_offset_size(21.., whole), (21, bs(whole - 21))); + assert_eq!(range_to_offset_size(0.., whole), (0, bs(whole))); + assert_eq!(range_to_offset_size(..21, whole), (0, bs(21))); + } + + #[test] + #[should_panic = "buffer slices can not be empty"] + fn range_to_offset_size_panics_for_empty_range() { + range_to_offset_size(123..123, 200); + } + + #[test] + #[should_panic = "buffer slices can not be empty"] + fn range_to_offset_size_panics_for_unbounded_empty_range() { + range_to_offset_size(..0, 100); + } + + #[test] + fn check_buffer_bounds_works_for_end_in_range() { + check_buffer_bounds(200, 100, bs(50)); + check_buffer_bounds(200, 100, bs(100)); + check_buffer_bounds(u64::MAX, u64::MAX - 100, bs(100)); + check_buffer_bounds(u64::MAX, 0, bs(u64::MAX)); + check_buffer_bounds(u64::MAX, 1, bs(u64::MAX - 1)); + } + + #[test] + #[should_panic] + fn check_buffer_bounds_panics_for_end_over_size() { + check_buffer_bounds(200, 100, bs(101)); + } + + #[test] + #[should_panic] + fn check_buffer_bounds_panics_for_end_wraparound() { + check_buffer_bounds(u64::MAX, 1, bs(u64::MAX)); + } + + #[test] + fn range_overlapping() { + // First range to the left + assert_eq!(range_overlaps(&(0..1), &(1..3)), false); + // First range overlaps left edge + assert_eq!(range_overlaps(&(0..2), &(1..3)), true); + // First range completely inside second + assert_eq!(range_overlaps(&(1..2), &(0..3)), true); + // First range completely surrounds second + assert_eq!(range_overlaps(&(0..3), &(1..2)), true); + // First range overlaps right edge + assert_eq!(range_overlaps(&(1..3), &(0..2)), true); + // First range entirely to the right + assert_eq!(range_overlaps(&(2..3), &(0..2)), false); + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/command_buffer.rs b/third_party/wgpu-29.0.4-patched/src/api/command_buffer.rs new file mode 100644 index 00000000..7685e240 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/command_buffer.rs @@ -0,0 +1,32 @@ +use crate::{ + api::{impl_deferred_command_buffer_actions, SharedDeferredCommandBufferActions}, + *, +}; + +/// Handle to a command buffer on the GPU. +/// +/// A `CommandBuffer` represents a complete sequence of commands that may be submitted to a command +/// queue with [`Queue::submit`]. A `CommandBuffer` is obtained by recording a series of commands to +/// a [`CommandEncoder`] and then calling [`CommandEncoder::finish`]. +/// +/// Corresponds to [WebGPU `GPUCommandBuffer`](https://gpuweb.github.io/gpuweb/#command-buffer). +#[derive(Debug)] +pub struct CommandBuffer { + pub(crate) buffer: dispatch::DispatchCommandBuffer, + /// Deferred actions recorded at encode time, to run at Queue::submit. + pub(crate) actions: SharedDeferredCommandBufferActions, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(CommandBuffer: Send, Sync); + +impl CommandBuffer { + #[cfg(custom)] + /// Returns custom implementation of CommandBuffer (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.buffer.as_custom() + } + + // Expose map_buffer_on_submit/on_submitted_work_done on CommandBuffer as well, + // so callers can schedule after finishing encoding. + impl_deferred_command_buffer_actions!(); +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/command_buffer_actions.rs b/third_party/wgpu-29.0.4-patched/src/api/command_buffer_actions.rs new file mode 100644 index 00000000..1c79b16f --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/command_buffer_actions.rs @@ -0,0 +1,149 @@ +use alloc::{sync::Arc, vec::Vec}; +use core::num::NonZeroU64; + +use crate::{util::Mutex, *}; + +/// A deferred buffer mapping request captured during encoding (or a pass) +/// and executed later when the command buffer is submitted. +pub(crate) struct DeferredBufferMapping { + pub buffer: api::Buffer, + pub mode: MapMode, + pub offset: u64, + pub size: NonZeroU64, + pub callback: dispatch::BufferMapCallback, +} + +pub(super) type SharedDeferredCommandBufferActions = Arc>; + +/// Set of actions to take when the command buffer is submitted. +#[derive(Default)] +pub(crate) struct DeferredCommandBufferActions { + pub buffer_mappings: Vec, + pub on_submitted_work_done_callbacks: Vec, +} + +impl DeferredCommandBufferActions { + pub fn append(&mut self, other: &mut Self) { + self.buffer_mappings.append(&mut other.buffer_mappings); + self.on_submitted_work_done_callbacks + .append(&mut other.on_submitted_work_done_callbacks); + } + + pub fn execute(self, queue: &dispatch::DispatchQueue) { + for mapping in self.buffer_mappings { + mapping.buffer.map_async( + mapping.mode, + mapping.offset..mapping.offset + mapping.size.get(), + mapping.callback, + ); + } + for callback in self.on_submitted_work_done_callbacks { + queue.on_submitted_work_done(callback); + } + } +} + +impl core::fmt::Debug for DeferredCommandBufferActions { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("DeferredCommandBufferActions") + .field("buffer_mappings.len()", &self.buffer_mappings.len()) + .field( + "on_submitted_work_done_callbacks.len()", + &self.on_submitted_work_done_callbacks.len(), + ) + .finish() + } +} + +// We can't just implement this on CommandEncoders as by default passes make it so that +// you can't call any commands on the encoder while this is happening. As such, we need +// to implement these methods on the passes too. Use a macro to avoid massive code duplication +macro_rules! impl_deferred_command_buffer_actions { + () => { + /// On submission, maps the buffer to host (CPU) memory, making it available + /// for reading or writing via [`get_mapped_range()`](Buffer::get_mapped_range). + /// The buffer becomes accessible once the `callback` is invoked with [`Ok`]. + /// + /// Use this when you need to submit work that uses the buffer before mapping it. + /// Because that submission must happen before calling `map_async`, this method + /// schedules the mapping for after submission, avoiding extra calls to + /// [`Buffer::map_async()`] or [`BufferSlice::map_async()`] and letting you start + /// the mapping from a more convenient place. + /// + /// For the callback to run, either [`queue.submit(..)`][q::s], [`instance.poll_all(..)`][i::p_a], + /// or [`device.poll(..)`][d::p] must be called elsewhere in the runtime, possibly integrated + /// into an event loop or run on a separate thread. + /// + /// The callback runs on the thread that first calls one of the above functions + /// after the GPU work completes. There are no restrictions on the code you can run + /// in the callback; however, on native the polling call will not return until the + /// callback finishes, so keep callbacks short (set flags, send messages, etc.). + /// + /// While a buffer is mapped, it cannot be used by other commands; at any time, + /// either the GPU or the CPU has exclusive access to the buffer’s contents. + /// + /// # Panics + /// + /// - If `bounds` is outside the bounds of `buffer`. + /// - If `bounds` has a length less than 1. + /// + /// # Panics During Submit + /// + /// - If the buffer is already mapped. + /// - If the buffer’s [`BufferUsages`] do not allow the requested [`MapMode`]. + /// - If `bounds` is outside of the bounds of `buffer`. + /// - If `bounds` does not start at a multiple of [`MAP_ALIGNMENT`]. + /// - If `bounds` has a length that is not a multiple of 4 greater than 0. + /// + /// [q::s]: Queue::submit + /// [i::p_a]: Instance::poll_all + /// [d::p]: Device::poll + /// [CEmbos]: CommandEncoder::map_buffer_on_submit + /// [CBmbos]: CommandBuffer::map_buffer_on_submit + /// [RPmbos]: RenderPass::map_buffer_on_submit + /// [CPmbos]: ComputePass::map_buffer_on_submit + pub fn map_buffer_on_submit>( + &self, + buffer: &api::Buffer, + mode: MapMode, + bounds: S, + callback: impl FnOnce(Result<(), BufferAsyncError>) + WasmNotSend + 'static, + ) { + let (offset, size) = range_to_offset_size(bounds, buffer.size); + self.actions.lock().buffer_mappings.push( + crate::api::command_buffer_actions::DeferredBufferMapping { + buffer: buffer.clone(), + mode, + offset, + size, + callback: alloc::boxed::Box::new(callback), + }, + ); + } + + /// Registers a callback that is invoked when this command buffer’s work finishes + /// executing on the GPU. When this callback runs, all mapped-buffer callbacks + /// registered for the same submission are guaranteed to have been called. + /// + /// For the callback to run, either [`queue.submit(..)`][q::s], [`instance.poll_all(..)`][i::p_a], + /// or [`device.poll(..)`][d::p] must be called elsewhere in the runtime, possibly integrated + /// into an event loop or run on a separate thread. + /// + /// The callback runs on the thread that first calls one of the above functions + /// after the GPU work completes. There are no restrictions on the code you can run + /// in the callback; however, on native the polling call will not return until the + /// callback finishes, so keep callbacks short (set flags, send messages, etc.). + /// + /// [q::s]: Queue::submit + /// [i::p_a]: Instance::poll_all + /// [d::p]: Device::poll + pub fn on_submitted_work_done(&self, callback: impl FnOnce() + Send + 'static) { + self.actions + .lock() + .on_submitted_work_done_callbacks + .push(alloc::boxed::Box::new(callback)); + } + }; +} + +pub(crate) use impl_deferred_command_buffer_actions; diff --git a/third_party/wgpu-29.0.4-patched/src/api/command_encoder.rs b/third_party/wgpu-29.0.4-patched/src/api/command_encoder.rs new file mode 100644 index 00000000..77648d40 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/command_encoder.rs @@ -0,0 +1,448 @@ +use alloc::sync::Arc; +use core::ops::Range; + +use crate::{ + api::{ + blas::BlasBuildEntry, impl_deferred_command_buffer_actions, tlas::Tlas, + SharedDeferredCommandBufferActions, + }, + *, +}; + +/// Encodes a series of GPU operations. +/// +/// A command encoder can record [`RenderPass`]es, [`ComputePass`]es, +/// and transfer operations between driver-managed resources like [`Buffer`]s and [`Texture`]s. +/// +/// When finished recording, call [`CommandEncoder::finish`] to obtain a [`CommandBuffer`] which may +/// be submitted for execution. +/// +/// Corresponds to [WebGPU `GPUCommandEncoder`](https://gpuweb.github.io/gpuweb/#command-encoder). +#[derive(Debug)] +pub struct CommandEncoder { + pub(crate) inner: dispatch::DispatchCommandEncoder, + pub(crate) actions: SharedDeferredCommandBufferActions, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(CommandEncoder: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(CommandEncoder => .inner); + +/// Describes a [`CommandEncoder`]. +/// +/// For use with [`Device::create_command_encoder`]. +/// +/// Corresponds to [WebGPU `GPUCommandEncoderDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpucommandencoderdescriptor). +pub type CommandEncoderDescriptor<'a> = wgt::CommandEncoderDescriptor>; +static_assertions::assert_impl_all!(CommandEncoderDescriptor<'_>: Send, Sync); + +pub use wgt::TexelCopyBufferInfo as TexelCopyBufferInfoBase; +/// View of a buffer which can be used to copy to/from a texture. +/// +/// Corresponds to [WebGPU `GPUTexelCopyBufferInfo`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpuimagecopybuffer). +pub type TexelCopyBufferInfo<'a> = TexelCopyBufferInfoBase<&'a Buffer>; +#[cfg(send_sync)] +static_assertions::assert_impl_all!(TexelCopyBufferInfo<'_>: Send, Sync); + +pub use wgt::TexelCopyTextureInfo as TexelCopyTextureInfoBase; +/// View of a texture which can be used to copy to/from a buffer/texture. +/// +/// Corresponds to [WebGPU `GPUTexelCopyTextureInfo`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpuimagecopytexture). +pub type TexelCopyTextureInfo<'a> = TexelCopyTextureInfoBase<&'a Texture>; +#[cfg(send_sync)] +static_assertions::assert_impl_all!(TexelCopyTextureInfo<'_>: Send, Sync); + +impl CommandEncoder { + /// Finishes recording and returns a [`CommandBuffer`] that can be submitted for execution. + pub fn finish(self) -> CommandBuffer { + let Self { mut inner, actions } = self; + let buffer = inner.finish(); + CommandBuffer { buffer, actions } + } + + /// Begins recording of a render pass. + /// + /// This function returns a [`RenderPass`] object which records a single render pass. + /// + /// As long as the returned [`RenderPass`] has not ended, + /// any mutating operation on this command encoder causes an error and invalidates it. + /// Note that the `'encoder` lifetime relationship protects against this, + /// but it is possible to opt out of it by calling [`RenderPass::forget_lifetime`]. + /// This can be useful for runtime handling of the encoder->pass + /// dependency e.g. when pass and encoder are stored in the same data structure. + pub fn begin_render_pass<'encoder>( + &'encoder mut self, + desc: &RenderPassDescriptor<'_>, + ) -> RenderPass<'encoder> { + let rpass = self.inner.begin_render_pass(desc); + RenderPass { + inner: rpass, + actions: Arc::clone(&self.actions), + _encoder_guard: api::PhantomDrop::default(), + } + } + + /// Begins recording of a compute pass. + /// + /// This function returns a [`ComputePass`] object which records a single compute pass. + /// + /// As long as the returned [`ComputePass`] has not ended, + /// any mutating operation on this command encoder causes an error and invalidates it. + /// Note that the `'encoder` lifetime relationship protects against this, + /// but it is possible to opt out of it by calling [`ComputePass::forget_lifetime`]. + /// This can be useful for runtime handling of the encoder->pass + /// dependency e.g. when pass and encoder are stored in the same data structure. + pub fn begin_compute_pass<'encoder>( + &'encoder mut self, + desc: &ComputePassDescriptor<'_>, + ) -> ComputePass<'encoder> { + let cpass = self.inner.begin_compute_pass(desc); + ComputePass { + inner: cpass, + actions: Arc::clone(&self.actions), + _encoder_guard: api::PhantomDrop::default(), + } + } + + /// Copy data from one buffer to another. + /// + /// # Panics + /// + /// - Buffer offsets or copy size not a multiple of [`COPY_BUFFER_ALIGNMENT`]. + /// - Copy would overrun buffer. + /// - Copy within the same buffer. + pub fn copy_buffer_to_buffer( + &mut self, + source: &Buffer, + source_offset: BufferAddress, + destination: &Buffer, + destination_offset: BufferAddress, + copy_size: impl Into>, + ) { + self.inner.copy_buffer_to_buffer( + &source.inner, + source_offset, + &destination.inner, + destination_offset, + copy_size.into(), + ); + } + + /// Copy data from a buffer to a texture. + pub fn copy_buffer_to_texture( + &mut self, + source: TexelCopyBufferInfo<'_>, + destination: TexelCopyTextureInfo<'_>, + copy_size: Extent3d, + ) { + self.inner + .copy_buffer_to_texture(source, destination, copy_size); + } + + /// Copy data from a texture to a buffer. + pub fn copy_texture_to_buffer( + &mut self, + source: TexelCopyTextureInfo<'_>, + destination: TexelCopyBufferInfo<'_>, + copy_size: Extent3d, + ) { + self.inner + .copy_texture_to_buffer(source, destination, copy_size); + } + + /// Copy data from one texture to another. + /// + /// # Panics + /// + /// - Textures are not the same type + /// - If a depth texture, or a multisampled texture, the entire texture must be copied + /// - Copy would overrun either texture + pub fn copy_texture_to_texture( + &mut self, + source: TexelCopyTextureInfo<'_>, + destination: TexelCopyTextureInfo<'_>, + copy_size: Extent3d, + ) { + self.inner + .copy_texture_to_texture(source, destination, copy_size); + } + + /// Clears texture to zero. + /// + /// Note that unlike with clear_buffer, `COPY_DST` usage is not required. + /// + /// # Implementation notes + /// + /// - implemented either via buffer copies and render/depth target clear, path depends on texture usages + /// - behaves like texture zero init, but is performed immediately (clearing is *not* delayed via marking it as uninitialized) + /// + /// # Panics + /// + /// - `CLEAR_TEXTURE` extension not enabled + /// - Range is out of bounds + pub fn clear_texture(&mut self, texture: &Texture, subresource_range: &ImageSubresourceRange) { + self.inner.clear_texture(&texture.inner, subresource_range); + } + + /// Clears buffer to zero. + /// + /// # Panics + /// + /// - Buffer does not have `COPY_DST` usage. + /// - Range is out of bounds + pub fn clear_buffer( + &mut self, + buffer: &Buffer, + offset: BufferAddress, + size: Option, + ) { + self.inner.clear_buffer(&buffer.inner, offset, size); + } + + /// Inserts debug marker. + pub fn insert_debug_marker(&mut self, label: &str) { + self.inner.insert_debug_marker(label); + } + + /// Start record commands and group it into debug marker group. + pub fn push_debug_group(&mut self, label: &str) { + self.inner.push_debug_group(label); + } + + /// Stops command recording and creates debug group. + pub fn pop_debug_group(&mut self) { + self.inner.pop_debug_group(); + } + + /// Copies query results stored in `query_set` into `destination` so that they can be read + /// by compute shaders or buffer operations. + /// + /// * `query_range` is the range of query result indices to copy from `query_set`. + /// Occlusion and timestamp queries occupy 1 result index each; + /// for pipeline statistics queries, see [`PipelineStatisticsTypes`]. + /// * `destination_offset` is the offset within `destination` to start writing at. + /// It must be a multiple of [`QUERY_RESOLVE_BUFFER_ALIGNMENT`]. + /// + /// The length of the data written to `destination` will be 8 bytes ([`QUERY_SIZE`]) + /// times the number of elements in `query_range`. + /// + /// For further information about using queries, see [`QuerySet`]. + pub fn resolve_query_set( + &mut self, + query_set: &QuerySet, + query_range: Range, + destination: &Buffer, + destination_offset: BufferAddress, + ) { + self.inner.resolve_query_set( + &query_set.inner, + query_range.start, + query_range.end - query_range.start, + &destination.inner, + destination_offset, + ); + } + + impl_deferred_command_buffer_actions!(); + + /// Get the [`wgpu_hal`] command encoder from this `CommandEncoder`. + /// + /// The returned command encoder will be ready to record onto. + /// + /// # Errors + /// + /// This method will pass in [`None`] if: + /// - The encoder is not from the backend specified by `A`. + /// - The encoder is from the `webgpu` or `custom` backend. + /// + /// # Types + /// + /// The callback argument depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("CommandEncoder")] + #[doc = crate::macros::hal_type_metal!("CommandEncoder")] + #[doc = crate::macros::hal_type_dx12!("CommandEncoder")] + #[doc = crate::macros::hal_type_gles!("CommandEncoder")] + /// + /// # Safety + /// + /// - The raw handle obtained from the `A::CommandEncoder` must not be manually destroyed. + /// - You must not end the command buffer; wgpu will do it when you call finish. + /// - The wgpu command encoder must not be interacted with in any way while recording is + /// happening to the wgpu_hal or backend command encoder. + #[cfg(wgpu_core)] + pub unsafe fn as_hal_mut) -> R, R>( + &mut self, + hal_command_encoder_callback: F, + ) -> R { + if let Some(encoder) = self.inner.as_core_mut_opt() { + unsafe { + encoder + .context + .command_encoder_as_hal_mut::(encoder, hal_command_encoder_callback) + } + } else { + hal_command_encoder_callback(None) + } + } + + #[cfg(custom)] + /// Returns custom implementation of CommandEncoder (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// [`Features::TIMESTAMP_QUERY_INSIDE_ENCODERS`] must be enabled on the device in order to call these functions. +impl CommandEncoder { + /// Issue a timestamp command at this point in the queue. + /// The timestamp will be written to the specified query set, at the specified index. + /// + /// Must be multiplied by [`Queue::get_timestamp_period`] to get + /// the value in nanoseconds. Absolute values have no meaning, + /// but timestamps can be subtracted to get the time it takes + /// for a string of operations to complete. + /// + /// Attention: Since commands within a command recorder may be reordered, + /// there is no strict guarantee that timestamps are taken after all commands + /// recorded so far and all before all commands recorded after. + /// This may depend both on the backend and the driver. + pub fn write_timestamp(&mut self, query_set: &QuerySet, query_index: u32) { + self.inner.write_timestamp(&query_set.inner, query_index); + } +} + +/// [`Features::EXPERIMENTAL_RAY_QUERY`] must be enabled on the device in order to call these functions. +impl CommandEncoder { + /// When encoding the acceleration structure build with the raw Hal encoder + /// (obtained from [`CommandEncoder::as_hal_mut`]), this function marks the + /// acceleration structures as having been built. + /// + /// This function must only be used with the raw encoder API. When using the + /// wgpu encoding API, acceleration structure build is tracked automatically. + /// + /// # Panics + /// + /// - If the encoder is being used with the wgpu encoding API. + /// + /// # Safety + /// + /// - All acceleration structures must have been build in this command encoder. + /// - All BLASes inputted must have been built before all TLASes that were inputted here and + /// which use them. + pub unsafe fn mark_acceleration_structures_built<'a>( + &self, + blas: impl IntoIterator, + tlas: impl IntoIterator, + ) { + self.inner + .mark_acceleration_structures_built(&mut blas.into_iter(), &mut tlas.into_iter()) + } + /// Build bottom and top level acceleration structures. + /// + /// Builds the BLASes then the TLASes, but does ***not*** build the BLASes into the TLASes, + /// that must be done by setting a TLAS instance in the TLAS package to one that contains the BLAS (and with an appropriate transform) + /// + /// # Validation + /// + /// - blas: Iterator of bottom level acceleration structure entries to build. + /// For each entry, the provided size descriptor must be strictly smaller or equal to the descriptor given at BLAS creation, this means: + /// - Less or equal number of geometries + /// - Same kind of geometry (with index buffer or without) (same vertex/index format) + /// - Same flags + /// - Less or equal number of vertices + /// - Less or equal number of indices (if applicable) + /// - tlas: iterator of top level acceleration structure packages to build + /// For each entry: + /// - Each BLAS in each TLAS instance must have been being built in the current call or in a previous call to `build_acceleration_structures` or `build_acceleration_structures_unsafe_tlas` + /// - The number of TLAS instances must be less than or equal to the max number of tlas instances when creating (if creating a package with `TlasPackage::new()` this is already satisfied) + /// + /// If the device the command encoder is created from does not have [Features::EXPERIMENTAL_RAY_QUERY] enabled then a validation error is generated + /// + /// A bottom level acceleration structure may be build and used as a reference in a top level acceleration structure in the same invocation of this function. + /// + /// # Bind group usage + /// + /// When a top level acceleration structure is used in a bind group, some validation takes place: + /// - The top level acceleration structure is valid and has been built. + /// - All the bottom level acceleration structures referenced by the top level acceleration structure are valid and have been built prior, + /// or at same time as the containing top level acceleration structure. + /// + /// [Features::EXPERIMENTAL_RAY_QUERY]: wgt::Features::EXPERIMENTAL_RAY_QUERY + pub fn build_acceleration_structures<'a>( + &mut self, + blas: impl IntoIterator>, + tlas: impl IntoIterator, + ) { + self.inner + .build_acceleration_structures(&mut blas.into_iter(), &mut tlas.into_iter()); + } + + /// Transition resources to an underlying hal resource state. + /// + /// This is an advanced, native-only API (no-op on web) that has two main use cases: + /// + /// # Batching Barriers + /// + /// Wgpu does not have a global view of the frame when recording command buffers. When you submit multiple command buffers in a single queue submission, wgpu may need to record and + /// insert new command buffers (holding 1 or more barrier commands) in between the user-supplied command buffers in order to ensure that resources are transitioned to the correct state + /// for the start of the next user-supplied command buffer. + /// + /// Wgpu does not currently attempt to batch multiple of these generated command buffers/barriers together, which may lead to suboptimal barrier placement. + /// + /// Consider the following scenario, where the user does `queue.submit(&[a, b, c])`: + /// * CommandBuffer A: Use resource X as a render pass attachment + /// * CommandBuffer B: Use resource Y as a render pass attachment + /// * CommandBuffer C: Use resources X and Y in a bind group + /// + /// At submission time, wgpu will record and insert some new command buffers, resulting in a submission that looks like `queue.submit(&[0, a, 1, b, 2, c])`: + /// * CommandBuffer 0: Barrier to transition resource X from TextureUses::RESOURCE (from last frame) to TextureUses::COLOR_TARGET + /// * CommandBuffer A: Use resource X as a render pass attachment + /// * CommandBuffer 1: Barrier to transition resource Y from TextureUses::RESOURCE (from last frame) to TextureUses::COLOR_TARGET + /// * CommandBuffer B: Use resource Y as a render pass attachment + /// * CommandBuffer 2: Barrier to transition resources X and Y from TextureUses::COLOR_TARGET to TextureUses::RESOURCE + /// * CommandBuffer C: Use resources X and Y in a bind group + /// + /// To prevent this, after profiling their app, an advanced user might choose to instead do `queue.submit(&[a, b, c])`: + /// * CommandBuffer A: + /// * Use [`CommandEncoder::transition_resources`] to transition resources X and Y from TextureUses::RESOURCE (from last frame) to TextureUses::COLOR_TARGET + /// * Use resource X as a render pass attachment + /// * CommandBuffer B: Use resource Y as a render pass attachment + /// * CommandBuffer C: + /// * Use [`CommandEncoder::transition_resources`] to transition resources X and Y from TextureUses::COLOR_TARGET to TextureUses::RESOURCE + /// * Use resources X and Y in a bind group + /// + /// At submission time, wgpu will record and insert some new command buffers, resulting in a submission that looks like `queue.submit(&[0, a, b, 1, c])`: + /// * CommandBuffer 0: Barrier to transition resources X and Y from TextureUses::RESOURCE (from last frame) to TextureUses::COLOR_TARGET + /// * CommandBuffer A: Use resource X as a render pass attachment + /// * CommandBuffer B: Use resource Y as a render pass attachment + /// * CommandBuffer 1: Barrier to transition resources X and Y from TextureUses::COLOR_TARGET to TextureUses::RESOURCE + /// * CommandBuffer C: Use resources X and Y in a bind group + /// + /// Which eliminates the extra command buffer and barrier between command buffers A and B. + /// + /// # Native Interoperability + /// + /// A user wanting to interoperate with the underlying native graphics APIs (Vulkan, DirectX12, Metal, etc) can use this API to generate barriers between wgpu commands and + /// the native API commands, for synchronization and resource state transition purposes. + pub fn transition_resources<'a>( + &mut self, + buffer_transitions: impl Iterator>, + texture_transitions: impl Iterator>, + ) { + self.inner.transition_resources( + &mut buffer_transitions.map(|t| wgt::BufferTransition { + buffer: &t.buffer.inner, + state: t.state, + }), + &mut texture_transitions.map(|t| wgt::TextureTransition { + texture: &t.texture.inner, + selector: t.selector, + state: t.state, + }), + ); + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/common_pipeline.rs b/third_party/wgpu-29.0.4-patched/src/api/common_pipeline.rs new file mode 100644 index 00000000..79ac7121 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/common_pipeline.rs @@ -0,0 +1,58 @@ +use crate::*; + +#[derive(Clone, Debug)] +/// Advanced options for use when a pipeline is compiled +/// +/// This implements `Default`, and for most users can be set to `Default::default()` +pub struct PipelineCompilationOptions<'a> { + /// Specifies the values of pipeline-overridable constants in the shader module. + /// + /// If an `@id` attribute was specified on the declaration, + /// the key must be the pipeline constant ID as a decimal ASCII number; if not, + /// the key must be the constant's identifier name. + /// + /// If the given constant is specified more than once, the last value specified is used. + /// + /// The value may represent any of WGSL's concrete scalar types. + pub constants: &'a [(&'a str, f64)], + /// Whether workgroup scoped memory will be initialized with zero values for this stage. + /// + /// This is required by the WebGPU spec, but may have overhead which can be avoided + /// for cross-platform applications + pub zero_initialize_workgroup_memory: bool, +} + +impl Default for PipelineCompilationOptions<'_> { + fn default() -> Self { + Self { + constants: Default::default(), + zero_initialize_workgroup_memory: true, + } + } +} + +/// Describes a pipeline cache, which allows reusing compilation work +/// between program runs. +/// +/// For use with [`Device::create_pipeline_cache`]. +/// +/// This type is unique to the Rust API of `wgpu`. +#[derive(Clone, Debug)] +pub struct PipelineCacheDescriptor<'a> { + /// Debug label of the pipeline cache. This might show up in some logs from `wgpu` + pub label: Label<'a>, + /// The data used to initialise the cache initialise + /// + /// # Safety + /// + /// This data must have been provided from a previous call to + /// [`PipelineCache::get_data`], if not `None` + pub data: Option<&'a [u8]>, + /// Whether to create a cache without data when the provided data + /// is invalid. + /// + /// Recommended to set to true + pub fallback: bool, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(PipelineCacheDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/compute_pass.rs b/third_party/wgpu-29.0.4-patched/src/api/compute_pass.rs new file mode 100644 index 00000000..cbd970be --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/compute_pass.rs @@ -0,0 +1,198 @@ +use crate::{ + api::{impl_deferred_command_buffer_actions, SharedDeferredCommandBufferActions}, + *, +}; + +/// In-progress recording of a compute pass. +/// +/// It can be created with [`CommandEncoder::begin_compute_pass`]. +/// +/// Corresponds to [WebGPU `GPUComputePassEncoder`]( +/// https://gpuweb.github.io/gpuweb/#compute-pass-encoder). +#[derive(Debug)] +pub struct ComputePass<'encoder> { + pub(crate) inner: dispatch::DispatchComputePass, + + /// Shared with CommandEncoder to enqueue deferred actions from within a pass. + pub(crate) actions: SharedDeferredCommandBufferActions, + + /// This lifetime is used to protect the [`CommandEncoder`] from being used + /// while the pass is alive. This needs to be PhantomDrop to prevent the lifetime + /// from being shortened. + pub(crate) _encoder_guard: crate::api::PhantomDrop<&'encoder ()>, +} + +#[cfg(send_sync)] +static_assertions::assert_impl_all!(ComputePass<'_>: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(ComputePass<'_> => .inner); + +impl ComputePass<'_> { + /// Drops the lifetime relationship to the parent command encoder, making usage of + /// the encoder while this pass is recorded a run-time error instead. + /// + /// Attention: As long as the compute pass has not been ended, any mutating operation on the parent + /// command encoder will cause a run-time error and invalidate it! + /// By default, the lifetime constraint prevents this, but it can be useful + /// to handle this at run time, such as when storing the pass and encoder in the same + /// data structure. + /// + /// This operation has no effect on pass recording. + /// It's a safe operation, since [`CommandEncoder`] is in a locked state as long as the pass is active + /// regardless of the lifetime constraint or its absence. + pub fn forget_lifetime(self) -> ComputePass<'static> { + ComputePass { + inner: self.inner, + actions: self.actions, + _encoder_guard: crate::api::PhantomDrop::default(), + } + } + + /// Sets the active bind group for a given bind group index. The bind group layout + /// in the active pipeline when the `dispatch()` function is called must match the layout of this bind group. + /// + /// If the bind group have dynamic offsets, provide them in the binding order. + /// These offsets have to be aligned to [`Limits::min_uniform_buffer_offset_alignment`] + /// or [`Limits::min_storage_buffer_offset_alignment`] appropriately. + pub fn set_bind_group<'a, BG>(&mut self, index: u32, bind_group: BG, offsets: &[DynamicOffset]) + where + Option<&'a BindGroup>: From, + { + let bg: Option<&BindGroup> = bind_group.into(); + let bg = bg.map(|bg| &bg.inner); + self.inner.set_bind_group(index, bg, offsets); + } + + /// Sets the active compute pipeline. + pub fn set_pipeline(&mut self, pipeline: &ComputePipeline) { + self.inner.set_pipeline(&pipeline.inner); + } + + /// Inserts debug marker. + pub fn insert_debug_marker(&mut self, label: &str) { + self.inner.insert_debug_marker(label); + } + + /// Start record commands and group it into debug marker group. + pub fn push_debug_group(&mut self, label: &str) { + self.inner.push_debug_group(label); + } + + /// Stops command recording and creates debug group. + pub fn pop_debug_group(&mut self) { + self.inner.pop_debug_group(); + } + + /// Dispatches compute work operations. + /// + /// `x`, `y` and `z` denote the number of work groups to dispatch in each dimension. + pub fn dispatch_workgroups(&mut self, x: u32, y: u32, z: u32) { + self.inner.dispatch_workgroups(x, y, z); + } + + /// Dispatches compute work operations, based on the contents of the `indirect_buffer`. + /// + /// The structure expected in `indirect_buffer` must conform to [`DispatchIndirectArgs`](crate::util::DispatchIndirectArgs). + pub fn dispatch_workgroups_indirect( + &mut self, + indirect_buffer: &Buffer, + indirect_offset: BufferAddress, + ) { + self.inner + .dispatch_workgroups_indirect(&indirect_buffer.inner, indirect_offset); + } + + impl_deferred_command_buffer_actions!(); + + #[cfg(custom)] + /// Returns custom implementation of ComputePass (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// [`Features::IMMEDIATES`] must be enabled on the device in order to call these functions. +impl ComputePass<'_> { + /// Set immediate data for subsequent dispatch calls. + /// + /// Write the bytes in `data` at offset `offset` within immediate data + /// storage. Both `offset` and the length of `data` must be + /// multiples of [`crate::IMMEDIATE_DATA_ALIGNMENT`], which is always 4. + /// + /// For example, if `offset` is `4` and `data` is eight bytes long, this + /// call will write `data` to bytes `4..12` of immediate data storage. + pub fn set_immediates(&mut self, offset: u32, data: &[u8]) { + self.inner.set_immediates(offset, data); + } +} + +/// [`Features::TIMESTAMP_QUERY_INSIDE_PASSES`] must be enabled on the device in order to call these functions. +impl ComputePass<'_> { + /// Issue a timestamp command at this point in the queue. The timestamp will be written to the specified query set, at the specified index. + /// + /// Must be multiplied by [`Queue::get_timestamp_period`] to get + /// the value in nanoseconds. Absolute values have no meaning, + /// but timestamps can be subtracted to get the time it takes + /// for a string of operations to complete. + pub fn write_timestamp(&mut self, query_set: &QuerySet, query_index: u32) { + self.inner.write_timestamp(&query_set.inner, query_index); + } +} + +/// [`Features::PIPELINE_STATISTICS_QUERY`] must be enabled on the device in order to call these functions. +impl ComputePass<'_> { + /// Start a pipeline statistics query on this compute pass. It can be ended with + /// `end_pipeline_statistics_query`. Pipeline statistics queries may not be nested. + /// + /// The amount of information collected by this query, and the space occupied in the query set, + /// is determined by the [`PipelineStatisticsTypes`] the query set was created with. + /// `query_index` is the index of the first query result slot that will be written to, and + /// `query_set` must have sufficient size to hold all results written starting at that slot. + pub fn begin_pipeline_statistics_query(&mut self, query_set: &QuerySet, query_index: u32) { + self.inner + .begin_pipeline_statistics_query(&query_set.inner, query_index); + } + + /// End the pipeline statistics query on this compute pass. It can be started with + /// `begin_pipeline_statistics_query`. Pipeline statistics queries may not be nested. + pub fn end_pipeline_statistics_query(&mut self) { + self.inner.end_pipeline_statistics_query(); + } +} + +/// Describes the timestamp writes of a compute pass. +/// +/// For use with [`ComputePassDescriptor`]. +/// At least one of `beginning_of_pass_write_index` and `end_of_pass_write_index` must be `Some`. +/// +/// Corresponds to [WebGPU `GPUComputePassTimestampWrites`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpucomputepasstimestampwrites). +#[derive(Clone, Debug)] +pub struct ComputePassTimestampWrites<'a> { + /// The query set to write to. + pub query_set: &'a QuerySet, + /// The index of the query set at which a start timestamp of this pass is written, if any. + pub beginning_of_pass_write_index: Option, + /// The index of the query set at which an end timestamp of this pass is written, if any. + pub end_of_pass_write_index: Option, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(ComputePassTimestampWrites<'_>: Send, Sync); + +/// Describes the attachments of a compute pass. +/// +/// For use with [`CommandEncoder::begin_compute_pass`]. +/// +/// Corresponds to [WebGPU `GPUComputePassDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpucomputepassdescriptor). +#[derive(Clone, Default, Debug)] +pub struct ComputePassDescriptor<'a> { + /// Debug label of the compute pass. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// Defines which timestamp values will be written for this pass, and where to write them to. + /// + /// Requires [`Features::TIMESTAMP_QUERY`] to be enabled. + pub timestamp_writes: Option>, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(ComputePassDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/compute_pipeline.rs b/third_party/wgpu-29.0.4-patched/src/api/compute_pipeline.rs new file mode 100644 index 00000000..499f967b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/compute_pipeline.rs @@ -0,0 +1,87 @@ +use crate::*; + +/// Handle to a compute pipeline. +/// +/// A `ComputePipeline` object represents a compute pipeline and its single shader stage. +/// It can be created with [`Device::create_compute_pipeline`]. +/// +/// Corresponds to [WebGPU `GPUComputePipeline`](https://gpuweb.github.io/gpuweb/#compute-pipeline). +#[derive(Debug, Clone)] +pub struct ComputePipeline { + pub(crate) inner: dispatch::DispatchComputePipeline, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(ComputePipeline: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(ComputePipeline => .inner); + +impl ComputePipeline { + /// Get an object representing the bind group layout at a given index. + /// + /// If this pipeline was created with a [default layout][ComputePipelineDescriptor::layout], + /// then bind groups created with the returned `BindGroupLayout` can only be used with this + /// pipeline. + /// + /// This method will raise a validation error if there is no bind group layout at `index`. + pub fn get_bind_group_layout(&self, index: u32) -> BindGroupLayout { + let bind_group = self.inner.get_bind_group_layout(index); + BindGroupLayout { inner: bind_group } + } + + #[cfg(custom)] + /// Returns custom implementation of ComputePipeline (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Describes a compute pipeline. +/// +/// For use with [`Device::create_compute_pipeline`]. +/// +/// Corresponds to [WebGPU `GPUComputePipelineDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpucomputepipelinedescriptor). +#[derive(Clone, Debug)] +pub struct ComputePipelineDescriptor<'a> { + /// Debug label of the pipeline. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The layout of bind groups for this pipeline. + /// + /// If this is set, then [`Device::create_compute_pipeline`] will raise a validation error if + /// the layout doesn't match what the shader module(s) expect. + /// + /// Using the same [`PipelineLayout`] for many [`RenderPipeline`] or [`ComputePipeline`] + /// pipelines guarantees that you don't have to rebind any resources when switching between + /// those pipelines. + /// + /// ## Default pipeline layout + /// + /// If `layout` is `None`, then the pipeline has a [default layout] created and used instead. + /// The default layout is deduced from the shader modules. + /// + /// You can use [`ComputePipeline::get_bind_group_layout`] to create bind groups for use with + /// the default layout. However, these bind groups cannot be used with any other pipelines. This + /// is convenient for simple pipelines, but using an explicit layout is recommended in most + /// cases. + /// + /// [default layout]: https://www.w3.org/TR/webgpu/#default-pipeline-layout + pub layout: Option<&'a PipelineLayout>, + /// The compiled shader module for this stage. + pub module: &'a ShaderModule, + /// The name of the entry point in the compiled shader to use. + /// + /// If [`Some`], there must be a compute shader entry point with this name in `module`. + /// Otherwise, expect exactly one compute shader entry point in `module`, which will be + /// selected. + // NOTE: keep phrasing in sync. with `FragmentState::entry_point` + // NOTE: keep phrasing in sync. with `VertexState::entry_point` + pub entry_point: Option<&'a str>, + /// Advanced options for when this pipeline is compiled + /// + /// This implements `Default`, and for most users can be set to `Default::default()` + pub compilation_options: PipelineCompilationOptions<'a>, + /// The pipeline cache to use when creating this pipeline. + pub cache: Option<&'a PipelineCache>, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(ComputePipelineDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/device.rs b/third_party/wgpu-29.0.4-patched/src/api/device.rs new file mode 100644 index 00000000..56136d0c --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/device.rs @@ -0,0 +1,877 @@ +use alloc::{boxed::Box, string::String, sync::Arc, vec}; +#[cfg(wgpu_core)] +use core::ops::Deref; +use core::{error, fmt, future::Future, marker::PhantomData}; + +use crate::api::blas::{Blas, BlasGeometrySizeDescriptors, CreateBlasDescriptor}; +use crate::api::tlas::{CreateTlasDescriptor, Tlas}; +use crate::util::Mutex; +use crate::*; + +/// Open connection to a graphics and/or compute device. +/// +/// Responsible for the creation of most rendering and compute resources. +/// These are then used in commands, which are submitted to a [`Queue`]. +/// +/// A device may be requested from an adapter with [`Adapter::request_device`]. +/// +/// Corresponds to [WebGPU `GPUDevice`](https://gpuweb.github.io/gpuweb/#gpu-device). +#[derive(Debug, Clone)] +pub struct Device { + pub(crate) inner: dispatch::DispatchDevice, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(Device: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(Device => .inner); + +/// Describes a [`Device`]. +/// +/// For use with [`Adapter::request_device`]. +/// +/// Corresponds to [WebGPU `GPUDeviceDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpudevicedescriptor). +pub type DeviceDescriptor<'a> = wgt::DeviceDescriptor>; +static_assertions::assert_impl_all!(DeviceDescriptor<'_>: Send, Sync); + +impl Device { + #[cfg(custom)] + /// Returns custom implementation of Device (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } + + #[cfg(custom)] + /// Creates Device from custom implementation + pub fn from_custom(device: T) -> Self { + Self { + inner: dispatch::DispatchDevice::custom(device), + } + } + + /// Constructs a stub device for testing using [`Backend::Noop`]. + /// + /// This is a convenience function which avoids the configuration, `async`, and fallibility + /// aspects of constructing a device through `Instance`. + #[cfg(feature = "noop")] + pub fn noop(desc: &DeviceDescriptor<'_>) -> (Device, Queue) { + use core::future::Future as _; + use core::pin::pin; + use core::task; + let ctx = &mut task::Context::from_waker(task::Waker::noop()); + + let instance = Instance::new(InstanceDescriptor { + backends: Backends::NOOP, + backend_options: BackendOptions { + noop: NoopBackendOptions { enable: true }, + ..Default::default() + }, + ..InstanceDescriptor::new_without_display_handle() + }); + + // Both of these futures are trivial and should complete instantaneously, + // so we do not need an executor and can just poll them once. + let task::Poll::Ready(Ok(adapter)) = + pin!(instance.request_adapter(&RequestAdapterOptions::default())).poll(ctx) + else { + unreachable!() + }; + let task::Poll::Ready(Ok(device_and_queue)) = pin!(adapter.request_device(desc)).poll(ctx) + else { + unreachable!() + }; + device_and_queue + } + + /// Check for resource cleanups and mapping callbacks. Will block if [`PollType::Wait`] is passed. + /// + /// Return `true` if the queue is empty, or `false` if there are more queue + /// submissions still in flight. (Note that, unless access to the [`Queue`] is + /// coordinated somehow, this information could be out of date by the time + /// the caller receives it. `Queue`s can be shared between threads, so + /// other threads could submit new work at any time.) + /// + /// When running on WebGPU, this is a no-op. `Device`s are automatically polled. + pub fn poll(&self, poll_type: PollType) -> Result { + self.inner.poll(poll_type.map_index(|s| s.index)) + } + + /// The [features][Features] which can be used on this device. + /// + /// This will be equal to the [`required_features`][DeviceDescriptor::required_features] + /// specified when creating the device. + /// No additional features can be used, even if the underlying adapter can support them. + #[must_use] + pub fn features(&self) -> Features { + self.inner.features() + } + + /// The limits which can be used on this device. + /// + /// This will be equal to the [`required_limits`][DeviceDescriptor::required_limits] + /// specified when creating the device. + /// No better limits can be used, even if the underlying adapter can support them. + #[must_use] + pub fn limits(&self) -> Limits { + self.inner.limits() + } + + /// Get info about the adapter that this device was created from. + pub fn adapter_info(&self) -> AdapterInfo { + self.inner.adapter_info() + } + + /// Creates a shader module. + /// + ///
+ // NOTE: Keep this in sync with `naga::front::wgsl::parse_str`! + // NOTE: Keep this in sync with `wgpu_core::Global::device_create_shader_module`! + /// + /// This function may consume a lot of stack space. Compiler-enforced limits for parsing + /// recursion exist; if shader compilation runs into them, it will return an error gracefully. + /// However, on some build profiles and platforms, the default stack size for a thread may be + /// exceeded before this limit is reached during parsing. Callers should ensure that there is + /// enough stack space for this, particularly if calls to this method are exposed to user + /// input. + /// + ///
+ #[must_use] + pub fn create_shader_module(&self, desc: ShaderModuleDescriptor<'_>) -> ShaderModule { + let module = self + .inner + .create_shader_module(desc, wgt::ShaderRuntimeChecks::checked()); + ShaderModule { inner: module } + } + + /// Deprecated: Use [`create_shader_module_trusted`][csmt] instead. + /// + /// # Safety + /// + /// See [`create_shader_module_trusted`][csmt]. + /// + /// [csmt]: Self::create_shader_module_trusted + #[deprecated( + since = "24.0.0", + note = "Use `Device::create_shader_module_trusted(desc, wgpu::ShaderRuntimeChecks::unchecked())` instead." + )] + #[must_use] + pub unsafe fn create_shader_module_unchecked( + &self, + desc: ShaderModuleDescriptor<'_>, + ) -> ShaderModule { + unsafe { self.create_shader_module_trusted(desc, crate::ShaderRuntimeChecks::unchecked()) } + } + + /// Creates a shader module with flags to dictate runtime checks. + /// + /// When running on WebGPU, this will merely call [`create_shader_module`][csm]. + /// + /// # Safety + /// + /// In contrast with [`create_shader_module`][csm] this function + /// creates a shader module with user-customizable runtime checks which allows shaders to + /// perform operations which can lead to undefined behavior like indexing out of bounds, + /// thus it's the caller responsibility to pass a shader which doesn't perform any of this + /// operations. + /// + /// See the documentation for [`ShaderRuntimeChecks`] for more information about specific checks. + /// + /// [csm]: Self::create_shader_module + #[must_use] + pub unsafe fn create_shader_module_trusted( + &self, + desc: ShaderModuleDescriptor<'_>, + runtime_checks: crate::ShaderRuntimeChecks, + ) -> ShaderModule { + let module = self.inner.create_shader_module(desc, runtime_checks); + ShaderModule { inner: module } + } + + /// Creates a shader module which will bypass wgpu's shader tooling and validation and be used directly by the backend. + /// + /// # Safety + /// + /// This function passes data to the backend as-is and can potentially result in a + /// driver crash or bogus behaviour. No attempt is made to ensure that data is valid. + #[must_use] + pub unsafe fn create_shader_module_passthrough( + &self, + desc: ShaderModuleDescriptorPassthrough<'_>, + ) -> ShaderModule { + let module = unsafe { self.inner.create_shader_module_passthrough(&desc) }; + ShaderModule { inner: module } + } + + /// Creates an empty [`CommandEncoder`]. + #[must_use] + pub fn create_command_encoder(&self, desc: &CommandEncoderDescriptor<'_>) -> CommandEncoder { + let encoder = self.inner.create_command_encoder(desc); + // Each encoder starts with its own deferred-action store that travels + // with the CommandBuffer produced by finish(). + CommandEncoder { + inner: encoder, + actions: Default::default(), + } + } + + /// Creates an empty [`RenderBundleEncoder`]. + #[must_use] + pub fn create_render_bundle_encoder<'a>( + &self, + desc: &RenderBundleEncoderDescriptor<'_>, + ) -> RenderBundleEncoder<'a> { + let encoder = self.inner.create_render_bundle_encoder(desc); + RenderBundleEncoder { + inner: encoder, + _p: PhantomData, + } + } + + /// Creates a new [`BindGroup`]. + #[must_use] + pub fn create_bind_group(&self, desc: &BindGroupDescriptor<'_>) -> BindGroup { + let group = self.inner.create_bind_group(desc); + BindGroup { inner: group } + } + + /// Creates a [`BindGroupLayout`]. + #[must_use] + pub fn create_bind_group_layout( + &self, + desc: &BindGroupLayoutDescriptor<'_>, + ) -> BindGroupLayout { + let layout = self.inner.create_bind_group_layout(desc); + BindGroupLayout { inner: layout } + } + + /// Creates a [`PipelineLayout`]. + #[must_use] + pub fn create_pipeline_layout(&self, desc: &PipelineLayoutDescriptor<'_>) -> PipelineLayout { + let layout = self.inner.create_pipeline_layout(desc); + PipelineLayout { inner: layout } + } + + /// Creates a [`RenderPipeline`]. + #[must_use] + pub fn create_render_pipeline(&self, desc: &RenderPipelineDescriptor<'_>) -> RenderPipeline { + let pipeline = self.inner.create_render_pipeline(desc); + RenderPipeline { inner: pipeline } + } + + /// Creates a mesh shader based [`RenderPipeline`]. + #[must_use] + pub fn create_mesh_pipeline(&self, desc: &MeshPipelineDescriptor<'_>) -> RenderPipeline { + let pipeline = self.inner.create_mesh_pipeline(desc); + RenderPipeline { inner: pipeline } + } + + /// Creates a [`ComputePipeline`]. + #[must_use] + pub fn create_compute_pipeline(&self, desc: &ComputePipelineDescriptor<'_>) -> ComputePipeline { + let pipeline = self.inner.create_compute_pipeline(desc); + ComputePipeline { inner: pipeline } + } + + /// Creates a [`Buffer`]. + #[must_use] + pub fn create_buffer(&self, desc: &BufferDescriptor<'_>) -> Buffer { + let map_context = MapContext::new(desc.mapped_at_creation.then_some(0..desc.size)); + + let buffer = self.inner.create_buffer(desc); + + Buffer { + inner: buffer, + map_context: Arc::new(Mutex::new(map_context)), + size: desc.size, + usage: desc.usage, + } + } + + /// Creates a new [`Texture`]. + /// + /// `desc` specifies the general format of the texture. + #[must_use] + pub fn create_texture(&self, desc: &TextureDescriptor<'_>) -> Texture { + let texture = self.inner.create_texture(desc); + + Texture { + inner: texture, + descriptor: TextureDescriptor { + label: None, + view_formats: &[], + ..desc.clone() + }, + } + } + + /// Creates a [`Texture`] from a wgpu-hal Texture. + /// + /// # Types + /// + /// The type of `A::Texture` depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Texture")] + #[doc = crate::macros::hal_type_metal!("Texture")] + #[doc = crate::macros::hal_type_dx12!("Texture")] + #[doc = crate::macros::hal_type_gles!("Texture")] + /// + /// # Safety + /// + /// - `hal_texture` must be created from this device internal handle + /// - `hal_texture` must be created respecting `desc` + /// - `hal_texture` must be initialized + #[cfg(wgpu_core)] + #[must_use] + pub unsafe fn create_texture_from_hal( + &self, + hal_texture: A::Texture, + desc: &TextureDescriptor<'_>, + ) -> Texture { + let texture = unsafe { + let core_device = self.inner.as_core(); + core_device + .context + .create_texture_from_hal::(hal_texture, core_device, desc) + }; + Texture { + inner: texture.into(), + descriptor: TextureDescriptor { + label: None, + view_formats: &[], + ..desc.clone() + }, + } + } + + /// Creates a new [`ExternalTexture`]. + #[must_use] + pub fn create_external_texture( + &self, + desc: &ExternalTextureDescriptor<'_>, + planes: &[&TextureView], + ) -> ExternalTexture { + let external_texture = self.inner.create_external_texture(desc, planes); + + ExternalTexture { + inner: external_texture, + } + } + + /// Creates a [`Buffer`] from a wgpu-hal Buffer. + /// + /// # Types + /// + /// The type of `A::Buffer` depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Buffer")] + #[doc = crate::macros::hal_type_metal!("Buffer")] + #[doc = crate::macros::hal_type_dx12!("Buffer")] + #[doc = crate::macros::hal_type_gles!("Buffer")] + /// + /// # Safety + /// + /// - `hal_buffer` must be created from this device internal handle + /// - `hal_buffer` must be created respecting `desc` + /// - `hal_buffer` must be initialized + /// - `hal_buffer` must not have zero size + #[cfg(wgpu_core)] + #[must_use] + pub unsafe fn create_buffer_from_hal( + &self, + hal_buffer: A::Buffer, + desc: &BufferDescriptor<'_>, + ) -> Buffer { + let map_context = MapContext::new(desc.mapped_at_creation.then_some(0..desc.size)); + + let buffer = unsafe { + let core_device = self.inner.as_core(); + core_device + .context + .create_buffer_from_hal::(hal_buffer, core_device, desc) + }; + + Buffer { + inner: buffer.into(), + map_context: Arc::new(Mutex::new(map_context)), + size: desc.size, + usage: desc.usage, + } + } + + /// Creates a new [`Sampler`]. + /// + /// `desc` specifies the behavior of the sampler. + #[must_use] + pub fn create_sampler(&self, desc: &SamplerDescriptor<'_>) -> Sampler { + let sampler = self.inner.create_sampler(desc); + Sampler { inner: sampler } + } + + /// Creates a new [`QuerySet`]. + #[must_use] + pub fn create_query_set(&self, desc: &QuerySetDescriptor<'_>) -> QuerySet { + let query_set = self.inner.create_query_set(desc); + QuerySet { inner: query_set } + } + + /// Set a callback which will be called for all errors that are not handled in error scopes. + pub fn on_uncaptured_error(&self, handler: Arc) { + self.inner.on_uncaptured_error(handler) + } + + /// Push an error scope on this device's thread-local error scope + /// stack. All operations on this device, or on resources created + /// from this device, will have their errors captured by this scope + /// until the scope is popped. + /// + /// Scopes must be popped in reverse order to their creation. If + /// a guard is dropped without being `pop()`ped, the scope will be + /// popped, and the captured errors will be dropped. + /// + /// Multiple error scopes may be active at one time, forming a stack. + /// Each error will be reported to the inner-most scope that matches + /// its filter. + /// + /// With the `std` feature enabled, this stack is **thread-local**. + /// Without, this is **global** to all threads. + /// + /// ```rust + /// # async move { + /// # let device: wgpu::Device = unreachable!(); + /// let error_scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + /// + /// // ... + /// // do work that may produce validation errors + /// // ... + /// + /// // pop the error scope and get a future for the result + /// let error_future = error_scope.pop(); + /// + /// // await the future to get the error, if any + /// let error = error_future.await; + /// # }; + /// ``` + pub fn push_error_scope(&self, filter: ErrorFilter) -> ErrorScopeGuard { + let index = self.inner.push_error_scope(filter); + ErrorScopeGuard { + device: self.inner.clone(), + index, + popped: false, + _phantom: PhantomData, + } + } + + /// Starts a capture in the attached graphics debugger. + /// + /// This behaves differently depending on which graphics debugger is attached: + /// + /// - Renderdoc: Calls [`StartFrameCapture(device, NULL)`][rd]. + /// - Xcode: Creates a capture with [`MTLCaptureManager`][xcode]. + /// - None: No action is taken. + /// + /// # Safety + /// + /// - There should not be any other captures currently active. + /// - All other safety rules are defined by the graphics debugger, see the + /// documentation for the specific debugger. + /// - In general, graphics debuggers can easily cause crashes, so this isn't + /// ever guaranteed to be sound. + /// + /// # Tips + /// + /// - Debuggers need to capture both the recording of the commands and the + /// submission of the commands to the GPU. Try to wrap all of your + /// gpu work in a capture. + /// - If you encounter issues, try waiting for the GPU to finish all work + /// before stopping the capture. + /// + /// [rd]: https://renderdoc.org/docs/in_application_api.html#_CPPv417StartFrameCapture23RENDERDOC_DevicePointer22RENDERDOC_WindowHandle + /// [xcode]: https://developer.apple.com/documentation/metal/mtlcapturemanager + #[doc(alias = "start_renderdoc_capture")] + #[doc(alias = "start_xcode_capture")] + pub unsafe fn start_graphics_debugger_capture(&self) { + unsafe { self.inner.start_graphics_debugger_capture() } + } + + /// Stops the current capture in the attached graphics debugger. + /// + /// This behaves differently depending on which graphics debugger is attached: + /// + /// - Renderdoc: Calls [`EndFrameCapture(device, NULL)`][rd]. + /// - Xcode: Stops the capture with [`MTLCaptureManager`][xcode]. + /// - None: No action is taken. + /// + /// # Safety + /// + /// - There should be a capture currently active. + /// - All other safety rules are defined by the graphics debugger, see the + /// documentation for the specific debugger. + /// - In general, graphics debuggers can easily cause crashes, so this isn't + /// ever guaranteed to be sound. + /// + /// # Tips + /// + /// - If you encounter issues, try to submit all work to the GPU, and waiting + /// for that work to finish before stopping the capture. + /// + /// [rd]: https://renderdoc.org/docs/in_application_api.html#_CPPv415EndFrameCapture23RENDERDOC_DevicePointer22RENDERDOC_WindowHandle + /// [xcode]: https://developer.apple.com/documentation/metal/mtlcapturemanager + #[doc(alias = "stop_renderdoc_capture")] + #[doc(alias = "stop_xcode_capture")] + pub unsafe fn stop_graphics_debugger_capture(&self) { + unsafe { self.inner.stop_graphics_debugger_capture() } + } + + /// Query internal counters from the native backend for debugging purposes. + /// + /// Some backends may not set all counters, or may not set any counter at all. + /// The `counters` cargo feature must be enabled for any counter to be set. + /// + /// If a counter is not set, its contains its default value (zero). + #[must_use] + pub fn get_internal_counters(&self) -> wgt::InternalCounters { + self.inner.get_internal_counters() + } + + /// Generate an GPU memory allocation report if the underlying backend supports it. + /// + /// Backends that do not support producing these reports return `None`. A backend may + /// Support it and still return `None` if it is not using performing sub-allocation, + /// for example as a workaround for driver issues. + #[must_use] + pub fn generate_allocator_report(&self) -> Option { + self.inner.generate_allocator_report() + } + + /// Get the [`wgpu_hal`] device from this `Device`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::Device`]. + /// + /// # Types + /// + /// The returned type depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Device")] + #[doc = crate::macros::hal_type_metal!("Device")] + #[doc = crate::macros::hal_type_dx12!("Device")] + #[doc = crate::macros::hal_type_gles!("Device")] + /// + /// # Errors + /// + /// This method will return None if: + /// - The device is not from the backend specified by `A`. + /// - The device is from the `webgpu` or `custom` backend. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::Device`]: hal::Api::Device + #[cfg(wgpu_core)] + pub unsafe fn as_hal( + &self, + ) -> Option + WasmNotSendSync> { + let device = self.inner.as_core_opt()?; + unsafe { device.context.device_as_hal::(device) } + } + + /// Destroy this device. + pub fn destroy(&self) { + self.inner.destroy() + } + + /// Set a DeviceLostCallback on this device. + pub fn set_device_lost_callback( + &self, + callback: impl Fn(DeviceLostReason, String) + Send + 'static, + ) { + self.inner.set_device_lost_callback(Box::new(callback)) + } + + /// Create a [`PipelineCache`] with initial data + /// + /// This can be passed to [`Device::create_compute_pipeline`] + /// and [`Device::create_render_pipeline`] to either accelerate these + /// or add the cache results from those. + /// + /// # Safety + /// + /// If the `data` field of `desc` is set, it must have previously been returned from a call + /// to [`PipelineCache::get_data`][^saving]. This `data` will only be used if it came + /// from an adapter with the same [`util::pipeline_cache_key`]. + /// This *is* compatible across wgpu versions, as any data format change will + /// be accounted for. + /// + /// It is *not* supported to bring caches from previous direct uses of backend APIs + /// into this method. + /// + /// # Errors + /// + /// Returns an error value if: + /// * the [`PIPELINE_CACHE`](wgt::Features::PIPELINE_CACHE) feature is not enabled + /// * this device is invalid; or + /// * the device is out of memory + /// + /// This method also returns an error value if: + /// * The `fallback` field on `desc` is false; and + /// * the `data` provided would not be used[^data_not_used] + /// + /// If an error value is used in subsequent calls, default caching will be used. + /// + /// [^saving]: We do recognise that saving this data to disk means this condition + /// is impossible to fully prove. Consider the risks for your own application in this case. + /// + /// [^data_not_used]: This data may be not used if: the data was produced by a prior + /// version of wgpu; or was created for an incompatible adapter, or there was a GPU driver + /// update. In some cases, the data might not be used and a real value is returned, + /// this is left to the discretion of GPU drivers. + #[must_use] + pub unsafe fn create_pipeline_cache( + &self, + desc: &PipelineCacheDescriptor<'_>, + ) -> PipelineCache { + let cache = unsafe { self.inner.create_pipeline_cache(desc) }; + PipelineCache { inner: cache } + } +} + +/// [`Features::EXPERIMENTAL_RAY_QUERY`] must be enabled on the device in order to call these functions. +impl Device { + /// Create a bottom level acceleration structure, used inside a top level acceleration structure for ray tracing. + /// - `desc`: The descriptor of the acceleration structure. + /// - `sizes`: Size descriptor limiting what can be built into the acceleration structure. + /// + /// # Validation + /// If any of the following is not satisfied a validation error is generated + /// + /// The device ***must*** have [`Features::EXPERIMENTAL_RAY_QUERY`] enabled. + /// if `sizes` is [`BlasGeometrySizeDescriptors::Triangles`] then the following must be satisfied + /// - For every geometry descriptor (for the purposes this is called `geo_desc`) of `sizes.descriptors` the following must be satisfied: + /// - `geo_desc.vertex_format` must be within allowed formats (allowed formats for a given feature set + /// may be queried with [`Features::allowed_vertex_formats_for_blas`]). + /// - Both or neither of `geo_desc.index_format` and `geo_desc.index_count` must be provided. + /// + /// [`Features::EXPERIMENTAL_RAY_QUERY`]: wgt::Features::EXPERIMENTAL_RAY_QUERY + /// [`Features::allowed_vertex_formats_for_blas`]: wgt::Features::allowed_vertex_formats_for_blas + #[must_use] + pub fn create_blas( + &self, + desc: &CreateBlasDescriptor<'_>, + sizes: BlasGeometrySizeDescriptors, + ) -> Blas { + let (handle, blas) = self.inner.create_blas(desc, sizes); + + Blas { + inner: blas, + handle, + } + } + + /// Create a top level acceleration structure, used for ray tracing. + /// - `desc`: The descriptor of the acceleration structure. + /// + /// # Validation + /// If any of the following is not satisfied a validation error is generated + /// + /// The device ***must*** have [`Features::EXPERIMENTAL_RAY_QUERY`] enabled. + /// + /// [`Features::EXPERIMENTAL_RAY_QUERY`]: wgt::Features::EXPERIMENTAL_RAY_QUERY + #[must_use] + pub fn create_tlas(&self, desc: &CreateTlasDescriptor<'_>) -> Tlas { + let tlas = self.inner.create_tlas(desc); + + Tlas { + inner: tlas, + instances: vec![None; desc.max_instances as usize], + lowest_unmodified: 0, + } + } +} + +/// Requesting a device from an [`Adapter`] failed. +#[derive(Clone, Debug)] +pub struct RequestDeviceError { + pub(crate) inner: RequestDeviceErrorKind, +} +#[derive(Clone, Debug)] +pub(crate) enum RequestDeviceErrorKind { + /// Error from [`wgpu_core`]. + // must match dependency cfg + #[cfg(wgpu_core)] + Core(wgc::instance::RequestDeviceError), + + /// Error from web API that was called by `wgpu` to request a device. + /// + /// (This is currently never used by the webgl backend, but it could be.) + #[cfg(webgpu)] + WebGpu(String), +} + +static_assertions::assert_impl_all!(RequestDeviceError: Send, Sync); + +impl fmt::Display for RequestDeviceError { + fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.inner { + #[cfg(wgpu_core)] + RequestDeviceErrorKind::Core(error) => error.fmt(_f), + #[cfg(webgpu)] + RequestDeviceErrorKind::WebGpu(error) => { + write!(_f, "{error}") + } + #[cfg(not(any(webgpu, wgpu_core)))] + _ => unimplemented!("unknown `RequestDeviceErrorKind`"), + } + } +} + +impl error::Error for RequestDeviceError { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match &self.inner { + #[cfg(wgpu_core)] + RequestDeviceErrorKind::Core(error) => error.source(), + #[cfg(webgpu)] + RequestDeviceErrorKind::WebGpu(_) => None, + #[cfg(not(any(webgpu, wgpu_core)))] + _ => unimplemented!("unknown `RequestDeviceErrorKind`"), + } + } +} + +#[cfg(wgpu_core)] +impl From for RequestDeviceError { + fn from(error: wgc::instance::RequestDeviceError) -> Self { + Self { + inner: RequestDeviceErrorKind::Core(error), + } + } +} + +/// The callback of [`Device::on_uncaptured_error()`]. +/// +/// It must be a function with this signature. +pub trait UncapturedErrorHandler: Fn(Error) + Send + Sync + 'static {} +impl UncapturedErrorHandler for T where T: Fn(Error) + Send + Sync + 'static {} + +/// Kinds of [`Error`]s a [`Device::push_error_scope()`] may be configured to catch. +#[derive(Clone, Copy, Debug, Eq, PartialEq, PartialOrd)] +pub enum ErrorFilter { + /// Catch only out-of-memory errors. + OutOfMemory, + /// Catch only validation errors. + Validation, + /// Catch only internal errors. + Internal, +} +static_assertions::assert_impl_all!(ErrorFilter: Send, Sync); + +/// Lower level source of the error. +/// +/// `Send + Sync` varies depending on configuration. +#[cfg(send_sync)] +#[cfg_attr(docsrs, doc(cfg(all())))] +pub type ErrorSource = Box; +/// Lower level source of the error. +/// +/// `Send + Sync` varies depending on configuration. +#[cfg(not(send_sync))] +#[cfg_attr(docsrs, doc(cfg(all())))] +pub type ErrorSource = Box; + +/// Errors resulting from usage of GPU APIs. +/// +/// By default, errors translate into panics. Depending on the backend and circumstances, +/// errors may occur synchronously or asynchronously. When errors need to be handled, use +/// [`Device::push_error_scope()`] or [`Device::on_uncaptured_error()`]. +#[derive(Debug)] +pub enum Error { + /// Out of memory. + OutOfMemory { + /// Lower level source of the error. + source: ErrorSource, + }, + /// Validation error, signifying a bug in code or data provided to `wgpu`. + Validation { + /// Lower level source of the error. + source: ErrorSource, + /// Description of the validation error. + description: String, + }, + /// Internal error. Used for signalling any failures not explicitly expected by WebGPU. + /// + /// These could be due to internal implementation or system limits being reached. + Internal { + /// Lower level source of the error. + source: ErrorSource, + /// Description of the internal GPU error. + description: String, + }, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(Error: Send, Sync); + +impl error::Error for Error { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match self { + Error::OutOfMemory { source } => Some(source.as_ref()), + Error::Validation { source, .. } => Some(source.as_ref()), + Error::Internal { source, .. } => Some(source.as_ref()), + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::OutOfMemory { .. } => f.write_str("Out of Memory"), + Error::Validation { description, .. } => f.write_str(description), + Error::Internal { description, .. } => f.write_str(description), + } + } +} + +/// Guard for an error scope pushed with [`Device::push_error_scope()`]. +/// +/// Call [`pop()`] to pop the scope and get a future for the result. If +/// the guard is dropped without being popped explicitly, the scope will still be popped, +/// and the captured errors will be dropped. +/// +/// This guard is neither `Send` nor `Sync`, as error scopes are handled +/// on a per-thread basis when the `std` feature is enabled. +/// +/// [`pop()`]: ErrorScopeGuard::pop +#[must_use = "Error scopes must be explicitly popped to retrieve errors they catch"] +pub struct ErrorScopeGuard { + device: dispatch::DispatchDevice, + index: u32, + popped: bool, + // Ensure the guard is !Send and !Sync + _phantom: PhantomData<*mut ()>, +} + +static_assertions::assert_not_impl_any!(ErrorScopeGuard: Send, Sync); + +impl ErrorScopeGuard { + /// Pops the error scope. + /// + /// Returns a future which resolves to the error captured by this scope, if any. + /// The pop takes effect immediately; the future does not need to be awaited before doing work that is outside of this error scope. + pub fn pop(mut self) -> impl Future> + WasmNotSend { + self.popped = true; + self.device.pop_error_scope(self.index) + } +} + +impl Drop for ErrorScopeGuard { + fn drop(&mut self) { + if !self.popped { + drop(self.device.pop_error_scope(self.index)); + } + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/external_texture.rs b/third_party/wgpu-29.0.4-patched/src/api/external_texture.rs new file mode 100644 index 00000000..d41f41c5 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/external_texture.rs @@ -0,0 +1,31 @@ +use crate::*; + +/// Handle to an external texture on the GPU. +/// +/// It can be created with [`Device::create_external_texture`]. +/// +/// Corresponds to [WebGPU `GPUExternalTexture`](https://gpuweb.github.io/gpuweb/#gpuexternaltexture). +#[derive(Debug, Clone)] +pub struct ExternalTexture { + pub(crate) inner: dispatch::DispatchExternalTexture, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(ExternalTexture: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(ExternalTexture => .inner); + +impl ExternalTexture { + /// Destroy the associated native resources as soon as possible. + pub fn destroy(&self) { + self.inner.destroy(); + } +} + +/// Describes an [`ExternalTexture`]. +/// +/// For use with [`Device::create_external_texture`]. +/// +/// Corresponds to [WebGPU `GPUExternalTextureDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpuexternaltexturedescriptor). +pub type ExternalTextureDescriptor<'a> = wgt::ExternalTextureDescriptor>; +static_assertions::assert_impl_all!(ExternalTextureDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/instance.rs b/third_party/wgpu-29.0.4-patched/src/api/instance.rs new file mode 100644 index 00000000..673f0179 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/instance.rs @@ -0,0 +1,441 @@ +use alloc::vec::Vec; +use core::future::Future; + +use crate::{dispatch::InstanceInterface, util::Mutex, *}; + +bitflags::bitflags! { + /// WGSL language extensions. + /// + /// WGSL spec.: + #[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Hash)] + pub struct WgslLanguageFeatures: u32 { + /// + const ReadOnlyAndReadWriteStorageTextures = 1 << 0; + /// + const Packed4x8IntegerDotProduct = 1 << 1; + /// + const UnrestrictedPointerParameters = 1 << 2; + /// + const PointerCompositeAccess = 1 << 3; + } +} + +/// Contains the various entry points to start interacting with the system's GPUs. +/// +/// This is the first thing you create when using wgpu. +/// Its primary use is to create [`Adapter`]s and [`Surface`]s. +/// +/// Does not have to be kept alive. +/// +/// Corresponds to [WebGPU `GPU`](https://gpuweb.github.io/gpuweb/#gpu-interface). +#[derive(Debug, Clone)] +pub struct Instance { + inner: dispatch::DispatchInstance, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(Instance: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(Instance => .inner); + +impl Default for Instance { + /// Creates a new instance of wgpu with default options. + /// + /// Backends are set to `Backends::all()`, and FXC is chosen as the `dx12_shader_compiler`. + /// + /// # Panics + /// + /// If no backend feature for the active target platform is enabled, + /// this method will panic, see [`Instance::enabled_backend_features()`]. + fn default() -> Self { + // TODO: Differentiate constructors here too? + Self::new(InstanceDescriptor::new_without_display_handle()) + } +} + +impl Instance { + /// Create an new instance of wgpu using the given options and enabled backends. + /// + /// # Panics + /// + /// - If no backend feature for the active target platform is enabled, + /// this method will panic; see [`Instance::enabled_backend_features()`]. + #[allow(clippy::allow_attributes, unreachable_code)] + pub fn new(desc: InstanceDescriptor) -> Self { + if Self::enabled_backend_features().is_empty() { + panic!( + "No wgpu backend feature that is implemented for the target platform was enabled. \ + See `wgpu::Instance::enabled_backend_features()` for more information." + ); + } + + #[cfg(webgpu)] + { + let is_only_available_backend = !cfg!(wgpu_core); + let requested_webgpu = desc.backends.contains(Backends::BROWSER_WEBGPU); + let support_webgpu = crate::backend::get_browser_gpu_property() + .map(|maybe_gpu| maybe_gpu.is_some()) + .unwrap_or(false); + + if is_only_available_backend || (requested_webgpu && support_webgpu) { + return Self { + inner: crate::backend::ContextWebGpu::new(desc).into(), + }; + } + } + + #[cfg(wgpu_core)] + { + return Self { + inner: crate::backend::ContextWgpuCore::new(desc).into(), + }; + } + + // Silence unused variable warnings without adding _ to the parameter name (which shows up in docs). + let _ = desc; + + unreachable!( + "Earlier check of `enabled_backend_features` should have prevented getting here!" + ); + } + + /// Returns which backends can be picked for the current build configuration. + /// + /// The returned set depends on a combination of target platform and enabled features. + /// This does *not* do any runtime checks and is exclusively based on compile time information. + /// + /// `InstanceDescriptor::backends` does not need to be a subset of this, + /// but any backend that is not in this set, will not be picked. + pub const fn enabled_backend_features() -> Backends { + let mut backends = Backends::empty(); + // `.set` and `|=` don't work in a `const` context. + if cfg!(noop) { + backends = backends.union(Backends::NOOP); + } + if cfg!(vulkan) { + backends = backends.union(Backends::VULKAN); + } + if cfg!(any(gles, webgl)) { + backends = backends.union(Backends::GL); + } + if cfg!(metal) { + backends = backends.union(Backends::METAL); + } + if cfg!(dx12) { + backends = backends.union(Backends::DX12); + } + if cfg!(webgpu) { + backends = backends.union(Backends::BROWSER_WEBGPU); + } + backends + } + + /// Returns the set of [WGSL language extensions] supported by this instance. + /// + /// [WGSL language extensions]: https://www.w3.org/TR/webgpu/#gpuwgsllanguagefeatures + #[cfg(feature = "wgsl")] + pub fn wgsl_language_features(&self) -> WgslLanguageFeatures { + self.inner.wgsl_language_features() + } + + /// Retrieves all available [`Adapter`]s that match the given [`Backends`]. + /// + /// # Arguments + /// + /// - `backends` - Backends from which to enumerate adapters. + pub fn enumerate_adapters(&self, backends: Backends) -> impl Future> { + let future = self.inner.enumerate_adapters(backends); + + async move { + future + .await + .iter() + .map(|adapter| Adapter { + inner: adapter.clone(), + }) + .collect() + } + } + + /// Retrieves an [`Adapter`] which matches the given [`RequestAdapterOptions`]. + /// + /// Some options are "soft", so treated as non-mandatory. Others are "hard". + /// + /// If no adapters are found that satisfy all the "hard" options, an error is returned. + /// + /// When targeting WebGL2, a [`compatible_surface`](RequestAdapterOptions::compatible_surface) + /// must be specified; using `RequestAdapterOptions::default()` will not succeed. + pub fn request_adapter( + &self, + options: &RequestAdapterOptions<'_, '_>, + ) -> impl Future> + WasmNotSend { + let future = self.inner.request_adapter(options); + async move { future.await.map(|adapter| Adapter { inner: adapter }) } + } + + /// Creates a new surface targeting a given window/canvas/surface/etc.. + /// + /// Internally, this creates surfaces for all backends that are enabled for this instance. + /// + /// See [`SurfaceTarget`] for what targets are supported. + /// See [`Instance::create_surface_unsafe()`] for surface creation with unsafe target variants. + /// + /// Most commonly used are window handles (or provider of windows handles) + /// which can be passed directly as they're automatically converted to [`SurfaceTarget`]. + pub fn create_surface<'window>( + &self, + target: impl Into>, + ) -> Result, CreateSurfaceError> { + // Handle origin (i.e. window) to optionally take ownership of to make the surface outlast the window. + let handle_source; + + let target = target.into(); + let mut surface = match target { + SurfaceTarget::Window(window) => unsafe { + let surface = self.create_surface_unsafe( + SurfaceTargetUnsafe::from_window(&window).map_err(|e| CreateSurfaceError { + inner: CreateSurfaceErrorKind::RawHandle(e), + })?, + ); + handle_source = Some(window); + + surface + }?, + SurfaceTarget::DisplayAndWindow(display_and_window_handle) => unsafe { + let surface = self.create_surface_unsafe( + SurfaceTargetUnsafe::from_display_and_window( + &display_and_window_handle, + &display_and_window_handle, + ) + .map_err(|e| CreateSurfaceError { + inner: CreateSurfaceErrorKind::RawHandle(e), + })?, + ); + handle_source = Some(display_and_window_handle); + + surface + }?, + #[cfg(web)] + SurfaceTarget::Canvas(canvas) => { + handle_source = None; + + let value: &wasm_bindgen::JsValue = &canvas; + let obj = core::ptr::NonNull::from(value).cast(); + let raw_window_handle = raw_window_handle::WebCanvasWindowHandle::new(obj).into(); + + // Note that we need to call this while we still have `value` around. + // This is safe without storing canvas to `handle_origin` since the surface will create a copy internally. + unsafe { + self.create_surface_unsafe(SurfaceTargetUnsafe::RawHandle { + raw_display_handle: None, + raw_window_handle, + }) + }? + } + #[cfg(web)] + SurfaceTarget::OffscreenCanvas(canvas) => { + handle_source = None; + + let value: &wasm_bindgen::JsValue = &canvas; + let obj = core::ptr::NonNull::from(value).cast(); + let raw_window_handle = + raw_window_handle::WebOffscreenCanvasWindowHandle::new(obj).into(); + + // Note that we need to call this while we still have `value` around. + // This is safe without storing canvas to `handle_origin` since the surface will create a copy internally. + unsafe { + self.create_surface_unsafe(SurfaceTargetUnsafe::RawHandle { + raw_display_handle: None, + raw_window_handle, + }) + }? + } + }; + + surface._handle_source = handle_source; + + Ok(surface) + } + + /// Creates a new surface targeting a given window/canvas/surface/etc. using an unsafe target. + /// + /// Internally, this creates surfaces for all backends that are enabled for this instance. + /// + /// See [`SurfaceTargetUnsafe`] for what targets are supported. + /// See [`Instance::create_surface`] for surface creation with safe target variants. + /// + /// # Safety + /// + /// - See respective [`SurfaceTargetUnsafe`] variants for safety requirements. + pub unsafe fn create_surface_unsafe<'window>( + &self, + target: SurfaceTargetUnsafe, + ) -> Result, CreateSurfaceError> { + let surface = unsafe { self.inner.create_surface(target)? }; + + Ok(Surface { + _handle_source: None, + inner: surface, + config: Mutex::new(None), + }) + } + + /// Polls all devices. + /// + /// If `force_wait` is true and this is not running on the web, then this + /// function will block until all in-flight buffers have been mapped and + /// all submitted commands have finished execution. + /// + /// Return `true` if all devices' queues are empty, or `false` if there are + /// queue submissions still in flight. (Note that, unless access to all + /// [`Queue`s] associated with this [`Instance`] is coordinated somehow, + /// this information could be out of date by the time the caller receives + /// it. `Queue`s can be shared between threads, and other threads could + /// submit new work at any time.) + /// + /// On the web, this is a no-op. `Device`s are automatically polled. + /// + /// [`Queue`s]: Queue + pub fn poll_all(&self, force_wait: bool) -> bool { + self.inner.poll_all_devices(force_wait) + } + + /// Generates memory report. + /// + /// Returns `None` if the feature is not supported by the backend + /// which happens only when WebGPU is pre-selected by the instance creation. + #[cfg(wgpu_core)] + pub fn generate_report(&self) -> Option { + self.inner.as_core_opt().map(|ctx| ctx.generate_report()) + } +} + +/// Interop with wgpu-hal. +#[cfg(wgpu_core)] +impl Instance { + /// Create an new instance of wgpu from a wgpu-hal instance. This is often useful + /// when you need to do backend specific logic, or interop with an existing backend + /// instance. + /// + /// # Types + /// + /// The type of `A::Instance` depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Instance")] + #[doc = crate::macros::hal_type_metal!("Instance")] + #[doc = crate::macros::hal_type_dx12!("Instance")] + #[doc = crate::macros::hal_type_gles!("Instance")] + /// + /// # Safety + /// + /// - The `hal_instance` must be a valid and usable instance of the backend specified by `A`. + /// - wgpu will act like it has complete ownership of this instance, and will destroy it + /// when the last reference to the instance, internal or external, is dropped. + pub unsafe fn from_hal(hal_instance: A::Instance) -> Self { + Self { + inner: unsafe { + crate::backend::ContextWgpuCore::from_hal_instance::(hal_instance).into() + }, + } + } + + /// Get the [`wgpu_hal`] instance from this `Instance`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::Instance`]. + /// + /// # Types + /// + #[doc = crate::macros::hal_type_vulkan!("Instance")] + #[doc = crate::macros::hal_type_metal!("Instance")] + #[doc = crate::macros::hal_type_dx12!("Instance")] + #[doc = crate::macros::hal_type_gles!("Instance")] + /// + /// # Errors + /// + /// This method will return None if: + /// - The instance is not from the backend specified by `A`. + /// - The instance is from the `webgpu` or `custom` backend. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::Instance`]: hal::Api::Instance + pub unsafe fn as_hal(&self) -> Option<&A::Instance> { + self.inner + .as_core_opt() + .and_then(|ctx| unsafe { ctx.instance_as_hal::() }) + } + + /// Converts a wgpu-hal [`hal::ExposedAdapter`] to a wgpu [`Adapter`]. + /// + /// # Types + /// + /// The type of `hal_adapter.adapter` depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Adapter")] + #[doc = crate::macros::hal_type_metal!("Adapter")] + #[doc = crate::macros::hal_type_dx12!("Adapter")] + #[doc = crate::macros::hal_type_gles!("Adapter")] + /// + /// # Safety + /// + /// `hal_adapter` must be created from this instance internal handle. + pub unsafe fn create_adapter_from_hal( + &self, + hal_adapter: hal::ExposedAdapter, + ) -> Adapter { + let core_instance = self.inner.as_core(); + let adapter = unsafe { core_instance.create_adapter_from_hal(hal_adapter) }; + let core = backend::wgpu_core::CoreAdapter { + context: core_instance.clone(), + id: adapter, + }; + + Adapter { inner: core.into() } + } +} + +/// Interop with wgpu-core. +#[cfg(wgpu_core)] +impl Instance { + /// Create an new instance of wgpu from a wgpu-core instance. + /// + /// # Arguments + /// + /// - `core_instance` - wgpu-core instance. + /// + /// # Safety + /// + /// Refer to the creation of wgpu-core Instance. + pub unsafe fn from_core(core_instance: wgc::instance::Instance) -> Self { + Self { + inner: unsafe { + crate::backend::ContextWgpuCore::from_core_instance(core_instance).into() + }, + } + } +} + +/// Interop with custom backends. +#[cfg(custom)] +impl Instance { + /// Creates instance from custom context implementation + pub fn from_custom(instance: T) -> Self { + Self { + inner: dispatch::DispatchInstance::Custom(backend::custom::DynContext::new(instance)), + } + } + + #[cfg(custom)] + /// Returns custom implementation of Instance (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/mod.rs b/third_party/wgpu-29.0.4-patched/src/api/mod.rs new file mode 100644 index 00000000..b564aa75 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/mod.rs @@ -0,0 +1,103 @@ +//! Types and functions which define our public api and their +//! helper functionality. +//! +//! # Conventions +//! +//! Each major type gets its own module. The module is laid out as follows: +//! +//! - The type itself +//! - `impl` block for the type +//! - `Drop` implementation for the type (if needed) +//! - Descriptor types and their subtypes. +//! - Any non-public helper types or functions. +//! +//! # Imports +//! +//! Because our public api is "flat" (i.e. all types are directly under the `wgpu` module), +//! we use a single `crate::*` import at the top of each module to bring in all the types in +//! the public api. This is done to: +//! - Avoid having to write out a long list of imports for each module. +//! - Allow docs to be written naturally, without needing to worry about needing dedicated doc imports. +//! - Treat wgpu-types types and wgpu-core types as a single set. + +mod adapter; +mod bind_group; +mod bind_group_layout; +mod blas; +mod buffer; +mod command_buffer; +/// Not a root type, but common types for command buffer deferral actions. +mod command_buffer_actions; +mod command_encoder; +// Not a root type, but common descriptor types for pipelines. +mod common_pipeline; +mod compute_pass; +mod compute_pipeline; +mod device; +mod external_texture; +mod instance; +mod pipeline_cache; +mod pipeline_layout; +mod query_set; +mod queue; +mod render_bundle; +mod render_bundle_encoder; +mod render_pass; +mod render_pipeline; +mod sampler; +mod shader_module; +mod surface; +mod surface_texture; +mod texture; +mod texture_view; +mod tlas; + +pub use adapter::*; +pub use bind_group::*; +pub use bind_group_layout::*; +pub use blas::*; +pub use buffer::*; +pub use command_buffer::*; +use command_buffer_actions::*; +pub use command_encoder::*; +pub use common_pipeline::*; +pub use compute_pass::*; +pub use compute_pipeline::*; +pub use device::*; +pub use external_texture::*; +pub use instance::*; +pub use pipeline_cache::*; +pub use pipeline_layout::*; +pub use query_set::*; +pub use queue::*; +pub use render_bundle::*; +pub use render_bundle_encoder::*; +pub use render_pass::*; +pub use render_pipeline::*; +pub use sampler::*; +pub use shader_module::*; +pub use surface::*; +pub use surface_texture::*; +pub use texture::*; +pub use texture_view::*; +pub use tlas::*; + +/// Object debugging label. +pub type Label<'a> = Option<&'a str>; + +/// A cute utility type that works just like `PhantomData`, but also +/// implements `Drop`. This forces any lifetimes that are associated +/// with the type to be used until the `Drop` impl is ran. This prevents +/// lifetimes from being shortened. +#[derive(Debug)] +pub(crate) struct PhantomDrop(core::marker::PhantomData); + +impl Default for PhantomDrop { + fn default() -> Self { + Self(core::marker::PhantomData) + } +} + +impl Drop for PhantomDrop { + fn drop(&mut self) {} +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/pipeline_cache.rs b/third_party/wgpu-29.0.4-patched/src/api/pipeline_cache.rs new file mode 100644 index 00000000..c6f161ab --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/pipeline_cache.rs @@ -0,0 +1,96 @@ +use alloc::vec::Vec; + +use crate::*; + +/// Handle to a pipeline cache, which is used to accelerate +/// creating [`RenderPipeline`]s and [`ComputePipeline`]s +/// in subsequent executions +/// +/// This reuse is only applicable for the same or similar devices. +/// See [`util::pipeline_cache_key`] for some details and a suggested workflow. +/// +/// Created using [`Device::create_pipeline_cache`]. +/// +/// # Background +/// +/// In most GPU drivers, shader code must be converted into a machine code +/// which can be executed on the GPU. +/// Generating this machine code can require a lot of computation. +/// Pipeline caches allow this computation to be reused between executions +/// of the program. +/// This can be very useful for reducing program startup time. +/// +/// Note that most desktop GPU drivers will manage their own caches, +/// meaning that little advantage can be gained from this on those platforms. +/// However, on some platforms, especially Android, drivers leave this to the +/// application to implement. +/// +/// Unfortunately, drivers do not expose whether they manage their own caches. +/// Some reasonable policies for applications to use are: +/// - Manage their own pipeline cache on all platforms +/// - Only manage pipeline caches on Android +/// +/// # Usage +/// +/// This is used as [`RenderPipelineDescriptor::cache`] or [`ComputePipelineDescriptor::cache`]. +/// It is valid to use this resource when creating multiple pipelines, in +/// which case it will likely cache each of those pipelines. +/// It is also valid to create a new cache for each pipeline. +/// +/// This resource is most useful when the data produced from it (using +/// [`PipelineCache::get_data`]) is persisted. +/// Care should be taken that pipeline caches are only used for the same device, +/// as pipeline caches from compatible devices are unlikely to provide any advantage. +/// `util::pipeline_cache_key` can be used as a file/directory name to help ensure that. +/// +/// It is recommended to store pipeline caches atomically. If persisting to disk, +/// this can usually be achieved by creating a temporary file, then moving/[renaming] +/// the temporary file over the existing cache +/// +/// # Storage Usage +/// +/// There is not currently an API available to reduce the size of a cache. +/// This is due to limitations in the underlying graphics APIs used. +/// This is especially impactful if your application is being updated, so +/// previous caches are no longer being used. +/// +/// One option to work around this is to regenerate the cache. +/// That is, creating the pipelines which your program runs using +/// with the stored cached data, then recreating the *same* pipelines +/// using a new cache, which your application then store. +/// +/// # Implementations +/// +/// This resource currently only works on the following backends: +/// - Vulkan +/// +/// This type is unique to the Rust API of `wgpu`. +/// +/// [renaming]: std::fs::rename +#[derive(Debug, Clone)] +pub struct PipelineCache { + pub(crate) inner: crate::dispatch::DispatchPipelineCache, +} + +#[cfg(send_sync)] +static_assertions::assert_impl_all!(PipelineCache: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(PipelineCache => .inner); + +impl PipelineCache { + /// Get the data associated with this pipeline cache. + /// The data format is an implementation detail of `wgpu`. + /// The only defined operation on this data setting it as the `data` field + /// on [`PipelineCacheDescriptor`], then to [`Device::create_pipeline_cache`]. + /// + /// This function is unique to the Rust API of `wgpu`. + pub fn get_data(&self) -> Option> { + self.inner.get_data() + } + + #[cfg(custom)] + /// Returns custom implementation of PipelineCache (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/pipeline_layout.rs b/third_party/wgpu-29.0.4-patched/src/api/pipeline_layout.rs new file mode 100644 index 00000000..7705ea75 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/pipeline_layout.rs @@ -0,0 +1,47 @@ +use crate::*; + +/// Handle to a pipeline layout. +/// +/// A `PipelineLayout` object describes the available binding groups of a pipeline. +/// It can be created with [`Device::create_pipeline_layout`]. +/// +/// Corresponds to [WebGPU `GPUPipelineLayout`](https://gpuweb.github.io/gpuweb/#gpupipelinelayout). +#[derive(Debug, Clone)] +pub struct PipelineLayout { + pub(crate) inner: dispatch::DispatchPipelineLayout, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(PipelineLayout: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(PipelineLayout => .inner); + +impl PipelineLayout { + #[cfg(custom)] + /// Returns custom implementation of PipelineLayout (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Describes a [`PipelineLayout`]. +/// +/// For use with [`Device::create_pipeline_layout`]. +/// +/// Corresponds to [WebGPU `GPUPipelineLayoutDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpupipelinelayoutdescriptor). +#[derive(Clone, Debug, Default)] +pub struct PipelineLayoutDescriptor<'a> { + /// Debug label of the pipeline layout. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// Bind groups that this pipeline uses. The first entry will provide all the bindings for + /// "set = 0", second entry will provide all the bindings for "set = 1" etc. + pub bind_group_layouts: &'a [Option<&'a BindGroupLayout>], + /// The number of bytes of immediate data that are allocated for use + /// in the shader. The `var`s in the shader attached to + /// this pipeline must be equal or smaller than this size. + /// + /// If this value is non-zero, [`Features::IMMEDIATES`] must be enabled. + pub immediate_size: u32, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(PipelineLayoutDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/query_set.rs b/third_party/wgpu-29.0.4-patched/src/api/query_set.rs new file mode 100644 index 00000000..70bd0fad --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/query_set.rs @@ -0,0 +1,50 @@ +use crate::*; + +/// Handle to a query set. +/// +/// A `QuerySet` is an opaque, mutable storage location for the results of queries: +/// which are small pieces of information extracted from other operations such as render passes. +/// See [`QueryType`] for what types of information can be collected. +/// +/// Each query writes data into one or more result slots in the `QuerySet`, which must be created +/// with a sufficient number of slots for that usage. Each result slot is a an unsigned 64-bit +/// number. +/// +/// Using queries consists of the following steps: +/// +/// 1. Create a `QuerySet` of the appropriate type and number of query result slots +/// using [`Device::create_query_set()`]. +/// 2. Pass the `QuerySet` to the commands which will write to it. +/// See [`QueryType`] for the possible commands. +/// 3. Execute the command [`CommandEncoder::resolve_query_set()`]. +/// This converts the opaque data stored in a `QuerySet` into [`u64`]s stored in a [`Buffer`]. +/// 4. Make use of that buffer, such as by copying its contents to the CPU +/// or reading it from a compute shader. +/// +/// Corresponds to [WebGPU `GPUQuerySet`](https://gpuweb.github.io/gpuweb/#queryset). +#[derive(Debug, Clone)] +pub struct QuerySet { + pub(crate) inner: dispatch::DispatchQuerySet, +} +#[cfg(send_sync)] +#[cfg(send_sync)] +static_assertions::assert_impl_all!(QuerySet: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(QuerySet => .inner); + +impl QuerySet { + #[cfg(custom)] + /// Returns custom implementation of QuerySet (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Describes a [`QuerySet`]. +/// +/// For use with [`Device::create_query_set`]. +/// +/// Corresponds to [WebGPU `GPUQuerySetDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpuquerysetdescriptor). +pub type QuerySetDescriptor<'a> = wgt::QuerySetDescriptor>; +static_assertions::assert_impl_all!(QuerySetDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/queue.rs b/third_party/wgpu-29.0.4-patched/src/api/queue.rs new file mode 100644 index 00000000..aa5a0f24 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/queue.rs @@ -0,0 +1,358 @@ +use alloc::boxed::Box; +use core::ops::{Deref, RangeBounds}; + +use crate::{api::DeferredCommandBufferActions, *}; + +/// Handle to a command queue on a device. +/// +/// A `Queue` executes recorded [`CommandBuffer`] objects and provides convenience methods +/// for writing to [buffers](Queue::write_buffer) and [textures](Queue::write_texture). +/// It can be created along with a [`Device`] by calling [`Adapter::request_device`]. +/// +/// Corresponds to [WebGPU `GPUQueue`](https://gpuweb.github.io/gpuweb/#gpu-queue). +#[derive(Debug, Clone)] +pub struct Queue { + pub(crate) inner: dispatch::DispatchQueue, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(Queue: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(Queue => .inner); + +impl Queue { + #[cfg(custom)] + /// Returns custom implementation of Queue (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } + + #[cfg(custom)] + /// Creates Queue from custom implementation + pub fn from_custom(queue: T) -> Self { + Self { + inner: dispatch::DispatchQueue::custom(queue), + } + } +} + +/// Identifier for a particular call to [`Queue::submit`]. Can be used +/// as part of an argument to [`Device::poll`] to block for a particular +/// submission to finish. +/// +/// This type is unique to the Rust API of `wgpu`. +/// There is no analogue in the WebGPU specification. +#[derive(Debug, Clone)] +pub struct SubmissionIndex { + pub(crate) index: u64, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(SubmissionIndex: Send, Sync); + +/// Passed to [`Device::poll`] to control how and if it should block. +pub type PollType = wgt::PollType; +#[cfg(send_sync)] +static_assertions::assert_impl_all!(PollType: Send, Sync); + +/// A write-only view into a staging buffer. +/// +/// This type is what [`Queue::write_buffer_with()`] returns. +pub struct QueueWriteBufferView { + queue: Queue, + buffer: Buffer, + offset: BufferAddress, + inner: dispatch::DispatchQueueWriteBuffer, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(QueueWriteBufferView: Send, Sync); + +impl QueueWriteBufferView { + #[cfg(custom)] + /// Returns custom implementation of QueueWriteBufferView (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +impl Drop for QueueWriteBufferView { + fn drop(&mut self) { + self.queue + .inner + .write_staging_buffer(&self.buffer.inner, self.offset, &self.inner); + } +} + +/// These methods are equivalent to the methods of the same names on [`WriteOnly`]. +impl QueueWriteBufferView { + /// Returns the length of this view; the number of bytes to be written. + pub fn len(&self) -> usize { + self.inner.len() + } + + /// Returns `true` if the view has a length of 0. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns a [`WriteOnly`] reference to a portion of this. + /// + /// `.slice(..)` can be used to access the whole data. + pub fn slice<'a, S: RangeBounds>(&'a mut self, bounds: S) -> WriteOnly<'a, [u8]> { + // SAFETY: + // * this is a write mapping + // * function signature ensures no aliasing + unsafe { self.inner.write_slice() }.into_slice(bounds) + } + + /// Copies all elements from src into `self`. + /// + /// The length of `src` must be the same as `self`. + /// + /// This method is equivalent to + /// [`self.slice(..).copy_from_slice(src)`][WriteOnly::copy_from_slice]. + pub fn copy_from_slice(&mut self, src: &[u8]) { + self.slice(..).copy_from_slice(src) + } +} + +impl Queue { + /// Copies the bytes of `data` into `buffer` starting at `offset`. + /// + /// The data must be written fully in-bounds, that is, `offset + data.len() <= buffer.len()`. + /// + /// # Performance considerations + /// + /// * Calls to `write_buffer()` do *not* submit the transfer to the GPU + /// immediately. They begin GPU execution only on the next call to + /// [`Queue::submit()`], just before the explicitly submitted commands. + /// To get a set of scheduled transfers started immediately, + /// it's fine to call `submit` with no command buffers at all: + /// + /// ```no_run + /// # let queue: wgpu::Queue = todo!(); + /// # let buffer: wgpu::Buffer = todo!(); + /// # let data = [0u8]; + /// queue.write_buffer(&buffer, 0, &data); + /// queue.submit([]); + /// ``` + /// + /// However, `data` will be immediately copied into staging memory, so the + /// caller may discard it any time after this call completes. + /// + /// * Consider using [`Queue::write_buffer_with()`] instead. + /// That method allows you to prepare your data directly within the staging + /// memory, rather than first placing it in a separate `[u8]` to be copied. + /// That is, `queue.write_buffer(b, offset, data)` is approximately equivalent + /// to `queue.write_buffer_with(b, offset, data.len()).copy_from_slice(data)`, + /// so use `write_buffer_with()` if you can do something smarter than that + /// [`copy_from_slice()`](slice::copy_from_slice). However, for small values + /// (e.g. a typical uniform buffer whose contents come from a `struct`), + /// there will likely be no difference, since the compiler will be able to + /// optimize out unnecessary copies regardless. + /// + /// * Currently on native platforms, for both of these methods, the staging + /// memory will be a new allocation. This will then be released after the + /// next submission finishes. To entirely avoid short-lived allocations, you might + /// be able to use [`StagingBelt`](crate::util::StagingBelt), + /// or buffers you explicitly create, map, and unmap yourself. + pub fn write_buffer(&self, buffer: &Buffer, offset: BufferAddress, data: &[u8]) { + self.inner.write_buffer(&buffer.inner, offset, data); + } + + /// Prepares to write data to a buffer via a mapped staging buffer. + /// + /// This operation allocates a temporary buffer and then returns a + /// [`QueueWriteBufferView`], which + /// + /// * dereferences to a `[u8]` of length `size`, and + /// * when dropped, schedules a copy of its contents into `buffer` at `offset`. + /// + /// Therefore, this obtains the same result as [`Queue::write_buffer()`], but may + /// allow you to skip one allocation and one copy of your data, if you are able to + /// assemble your data directly into the returned [`QueueWriteBufferView`] instead of + /// into a separate allocation like a [`Vec`](alloc::vec::Vec) first. + /// + /// The data must be written fully in-bounds, that is, `offset + size <= buffer.len()`. + /// + /// # Performance considerations + /// + /// * For small data not separately heap-allocated, there is no advantage of this + /// over [`Queue::write_buffer()`]. + /// + /// * Reading from the returned view may be slow, and will not yield the current + /// contents of `buffer`. You should treat it as “write-only”. + /// + /// * Dropping the [`QueueWriteBufferView`] does *not* submit the + /// transfer to the GPU immediately. The transfer begins only on the next + /// call to [`Queue::submit()`] after the view is dropped, just before the + /// explicitly submitted commands. To get a set of scheduled transfers started + /// immediately, it's fine to call `queue.submit([])` with no command buffers at all. + /// + /// * Currently on native platforms, the staging memory will be a new allocation, which will + /// then be released after the next submission finishes. To entirely avoid short-lived + /// allocations, you might be able to use [`StagingBelt`](crate::util::StagingBelt), + /// or buffers you explicitly create, map, and unmap yourself. + #[must_use] + pub fn write_buffer_with( + &self, + buffer: &Buffer, + offset: BufferAddress, + size: BufferSize, + ) -> Option { + profiling::scope!("Queue::write_buffer_with"); + self.inner + .validate_write_buffer(&buffer.inner, offset, size)?; + let staging_buffer = self.inner.create_staging_buffer(size)?; + Some(QueueWriteBufferView { + queue: self.clone(), + buffer: buffer.clone(), + offset, + inner: staging_buffer, + }) + } + + /// Copies the bytes of `data` into a texture. + /// + /// * `data` contains the texels to be written, which must be in + /// [the same format as the texture](TextureFormat). + /// * `data_layout` describes the memory layout of `data`, which does not necessarily + /// have to have tightly packed rows. + /// * `texture` specifies the texture to write into, and the location within the + /// texture (coordinate offset, mip level) that will be overwritten. + /// * `size` is the size, in texels, of the region to be written. + /// + /// This method fails if `size` overruns the size of `texture`, or if `data` is too short. + /// + /// # Performance considerations + /// + /// This operation has the same performance considerations as [`Queue::write_buffer()`]; + /// see its documentation for details. + /// + /// However, since there is no “mapped texture” like a mapped buffer, + /// alternate techniques for writing to textures will generally consist of first copying + /// the data to a buffer, then using [`CommandEncoder::copy_buffer_to_texture()`], or in + /// some cases a compute shader, to copy texels from that buffer to the texture. + pub fn write_texture( + &self, + texture: TexelCopyTextureInfo<'_>, + data: &[u8], + data_layout: TexelCopyBufferLayout, + size: Extent3d, + ) { + self.inner.write_texture(texture, data, data_layout, size); + } + + /// Schedule a copy of data from `image` into `texture`. + #[cfg(web)] + pub fn copy_external_image_to_texture( + &self, + source: &wgt::CopyExternalImageSourceInfo, + dest: wgt::CopyExternalImageDestInfo<&api::Texture>, + size: Extent3d, + ) { + self.inner + .copy_external_image_to_texture(source, dest, size); + } + + /// Submits a series of finished command buffers for execution. + pub fn submit>( + &self, + command_buffers: I, + ) -> SubmissionIndex { + // As submit drains the iterator (even on error), collect deferred actions + // from each CommandBuffer along the way. + let mut actions = DeferredCommandBufferActions::default(); + + let mut command_buffers = command_buffers.into_iter().map(|comb| { + actions.append(&mut comb.actions.lock()); + comb.buffer + }); + let index = self.inner.submit(&mut command_buffers); + + // Execute all deferred actions after submit. + actions.execute(&self.inner); + + SubmissionIndex { index } + } + + /// Gets the amount of nanoseconds each tick of a timestamp query represents. + /// + /// Returns zero if timestamp queries are unsupported. + /// + /// Timestamp values are represented in nanosecond values on WebGPU, see + /// Therefore, this is always 1.0 on the web, but on wgpu-core a manual conversion is required. + pub fn get_timestamp_period(&self) -> f32 { + self.inner.get_timestamp_period() + } + + /// Registers a callback that is invoked when the previous [`Queue::submit`] finishes executing + /// on the GPU. When this callback runs, all mapped-buffer callbacks registered for the same + /// submission are guaranteed to have been called. + /// + /// For the callback to run, either [`queue.submit(..)`][q::s], [`instance.poll_all(..)`][i::p_a], + /// or [`device.poll(..)`][d::p] must be called elsewhere in the runtime, possibly integrated into + /// an event loop or run on a separate thread. + /// + /// The callback runs on the thread that first calls one of the above functions after the GPU work + /// completes. There are no restrictions on the code you can run in the callback; however, on native + /// the polling call will not return until the callback finishes, so keep callbacks short (set flags, + /// send messages, etc.). + /// + /// [q::s]: Queue::submit + /// [i::p_a]: Instance::poll_all + /// [d::p]: Device::poll + pub fn on_submitted_work_done(&self, callback: impl FnOnce() + Send + 'static) { + self.inner.on_submitted_work_done(Box::new(callback)); + } + + /// Get the [`wgpu_hal`] device from this `Queue`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::Queue`]. + /// + /// # Types + /// + /// The returned type depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Queue")] + #[doc = crate::macros::hal_type_metal!("Queue")] + #[doc = crate::macros::hal_type_dx12!("Queue")] + #[doc = crate::macros::hal_type_gles!("Queue")] + /// + /// # Errors + /// + /// This method will return None if: + /// - The queue is not from the backend specified by `A`. + /// - The queue is from the `webgpu` or `custom` backend. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::Queue`]: hal::Api::Queue + #[cfg(wgpu_core)] + pub unsafe fn as_hal( + &self, + ) -> Option + WasmNotSendSync> { + let queue = self.inner.as_core_opt()?; + unsafe { queue.context.queue_as_hal::(queue) } + } + + /// Compact a BLAS, it must have had [`Blas::prepare_compaction_async`] called on it and had the + /// callback provided called. + /// + /// The returned BLAS is more restricted than a normal BLAS because it may not be rebuilt or + /// compacted. + pub fn compact_blas(&self, blas: &Blas) -> Blas { + let (handle, dispatch) = self.inner.compact_blas(&blas.inner); + Blas { + handle, + inner: dispatch, + } + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/render_bundle.rs b/third_party/wgpu-29.0.4-patched/src/api/render_bundle.rs new file mode 100644 index 00000000..2ea59294 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/render_bundle.rs @@ -0,0 +1,36 @@ +use crate::*; + +/// Pre-prepared reusable bundle of GPU operations. +/// +/// It only supports a handful of render commands, but it makes them reusable. Executing a +/// [`RenderBundle`] is often more efficient than issuing the underlying commands manually. +/// +/// It can be created by use of a [`RenderBundleEncoder`], and executed onto a [`CommandEncoder`] +/// using [`RenderPass::execute_bundles`]. +/// +/// Corresponds to [WebGPU `GPURenderBundle`](https://gpuweb.github.io/gpuweb/#render-bundle). +#[derive(Debug, Clone)] +pub struct RenderBundle { + pub(crate) inner: dispatch::DispatchRenderBundle, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(RenderBundle: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(RenderBundle => .inner); + +impl RenderBundle { + #[cfg(custom)] + /// Returns custom implementation of RenderBundle (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Describes a [`RenderBundle`]. +/// +/// For use with [`RenderBundleEncoder::finish`]. +/// +/// Corresponds to [WebGPU `GPURenderBundleDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpurenderbundledescriptor). +pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor>; +static_assertions::assert_impl_all!(RenderBundleDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/render_bundle_encoder.rs b/third_party/wgpu-29.0.4-patched/src/api/render_bundle_encoder.rs new file mode 100644 index 00000000..cea7aa33 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/render_bundle_encoder.rs @@ -0,0 +1,212 @@ +use core::{marker::PhantomData, num::NonZeroU32, ops::Range}; + +use crate::dispatch::RenderBundleEncoderInterface; +use crate::*; + +/// Encodes a series of GPU operations into a reusable "render bundle". +/// +/// It only supports a handful of render commands, but it makes them reusable. +/// It can be created with [`Device::create_render_bundle_encoder`]. +/// It can be executed onto a [`CommandEncoder`] using [`RenderPass::execute_bundles`]. +/// +/// Executing a [`RenderBundle`] is often more efficient than issuing the underlying commands +/// manually. +/// +/// Corresponds to [WebGPU `GPURenderBundleEncoder`]( +/// https://gpuweb.github.io/gpuweb/#gpurenderbundleencoder). +#[derive(Debug)] +pub struct RenderBundleEncoder<'a> { + pub(crate) inner: dispatch::DispatchRenderBundleEncoder, + /// This type should be !Send !Sync, because it represents an allocation on this thread's + /// command buffer. + pub(crate) _p: PhantomData<(*const u8, &'a ())>, +} +static_assertions::assert_not_impl_any!(RenderBundleEncoder<'_>: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(RenderBundleEncoder<'_> => .inner); + +/// Describes a [`RenderBundleEncoder`]. +/// +/// For use with [`Device::create_render_bundle_encoder`]. +/// +/// Corresponds to [WebGPU `GPURenderBundleEncoderDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpurenderbundleencoderdescriptor). +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +pub struct RenderBundleEncoderDescriptor<'a> { + /// Debug label of the render bundle encoder. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The formats of the color attachments that this render bundle is capable to rendering to. This + /// must match the formats of the color attachments in the render pass this render bundle is executed in. + pub color_formats: &'a [Option], + /// Information about the depth attachment that this render bundle is capable to rendering to. This + /// must match the format of the depth attachments in the render pass this render bundle is executed in. + pub depth_stencil: Option, + /// Sample count this render bundle is capable of rendering to. This must match the pipelines and + /// the render passes it is used in. + pub sample_count: u32, + /// If this render bundle will rendering to multiple array layers in the attachments at the same time. + pub multiview: Option, +} +static_assertions::assert_impl_all!(RenderBundleEncoderDescriptor<'_>: Send, Sync); + +impl<'a> RenderBundleEncoder<'a> { + /// Finishes recording and returns a [`RenderBundle`] that can be executed in other render passes. + pub fn finish(self, desc: &RenderBundleDescriptor<'_>) -> RenderBundle { + let bundle = match self.inner { + #[cfg(wgpu_core)] + dispatch::DispatchRenderBundleEncoder::Core(b) => b.finish(desc), + #[cfg(webgpu)] + dispatch::DispatchRenderBundleEncoder::WebGPU(b) => b.finish(desc), + #[cfg(custom)] + dispatch::DispatchRenderBundleEncoder::Custom(_) => unimplemented!(), + }; + + RenderBundle { inner: bundle } + } + + /// Sets the active bind group for a given bind group index. The bind group layout + /// in the active pipeline when any `draw()` function is called must match the layout of this bind group. + /// + /// If the bind group have dynamic offsets, provide them in the binding order. + pub fn set_bind_group<'b, BG>(&mut self, index: u32, bind_group: BG, offsets: &[DynamicOffset]) + where + Option<&'b BindGroup>: From, + { + let bg: Option<&'b BindGroup> = bind_group.into(); + let bg = bg.map(|x| &x.inner); + self.inner.set_bind_group(index, bg, offsets); + } + + /// Sets the active render pipeline. + /// + /// Subsequent draw calls will exhibit the behavior defined by `pipeline`. + pub fn set_pipeline(&mut self, pipeline: &'a RenderPipeline) { + self.inner.set_pipeline(&pipeline.inner); + } + + /// Sets the active index buffer. + /// + /// Subsequent calls to [`draw_indexed`](RenderBundleEncoder::draw_indexed) on this [`RenderBundleEncoder`] will + /// use `buffer` as the source index buffer. + pub fn set_index_buffer(&mut self, buffer_slice: BufferSlice<'a>, index_format: IndexFormat) { + self.inner.set_index_buffer( + &buffer_slice.buffer.inner, + index_format, + buffer_slice.offset, + Some(buffer_slice.size), + ); + } + + /// Assign a vertex buffer to a slot. + /// + /// Subsequent calls to [`draw`] and [`draw_indexed`] on this + /// [`RenderBundleEncoder`] will use `buffer` as one of the source vertex buffers. + /// + /// The `slot` refers to the index of the matching descriptor in + /// [`VertexState::buffers`]. + /// + /// [`draw`]: RenderBundleEncoder::draw + /// [`draw_indexed`]: RenderBundleEncoder::draw_indexed + pub fn set_vertex_buffer(&mut self, slot: u32, buffer_slice: BufferSlice<'a>) { + self.inner.set_vertex_buffer( + slot, + &buffer_slice.buffer.inner, + buffer_slice.offset, + Some(buffer_slice.size), + ); + } + + /// Draws primitives from the active vertex buffer(s). + /// + /// The active vertex buffers can be set with [`RenderBundleEncoder::set_vertex_buffer`]. + /// Does not use an Index Buffer. If you need this see [`RenderBundleEncoder::draw_indexed`] + /// + /// Panics if vertices Range is outside of the range of the vertices range of any set vertex buffer. + /// + /// vertices: The range of vertices to draw. + /// instances: Range of Instances to draw. Use 0..1 if instance buffers are not used. + /// E.g.of how its used internally + /// ```rust ignore + /// for instance_id in instance_range { + /// for vertex_id in vertex_range { + /// let vertex = vertex[vertex_id]; + /// vertex_shader(vertex, vertex_id, instance_id); + /// } + /// } + /// ``` + pub fn draw(&mut self, vertices: Range, instances: Range) { + self.inner.draw(vertices, instances); + } + + /// Draws indexed primitives using the active index buffer and the active vertex buffer(s). + /// + /// The active index buffer can be set with [`RenderBundleEncoder::set_index_buffer`]. + /// The active vertex buffer(s) can be set with [`RenderBundleEncoder::set_vertex_buffer`]. + /// + /// Panics if indices Range is outside of the range of the indices range of any set index buffer. + /// + /// indices: The range of indices to draw. + /// base_vertex: value added to each index value before indexing into the vertex buffers. + /// instances: Range of Instances to draw. Use 0..1 if instance buffers are not used. + /// E.g.of how its used internally + /// ```rust ignore + /// for instance_id in instance_range { + /// for index_index in index_range { + /// let vertex_id = index_buffer[index_index]; + /// let adjusted_vertex_id = vertex_id + base_vertex; + /// let vertex = vertex[adjusted_vertex_id]; + /// vertex_shader(vertex, adjusted_vertex_id, instance_id); + /// } + /// } + /// ``` + pub fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range) { + self.inner.draw_indexed(indices, base_vertex, instances); + } + + /// Draws primitives from the active vertex buffer(s) based on the contents of the `indirect_buffer`. + /// + /// The active vertex buffers can be set with [`RenderBundleEncoder::set_vertex_buffer`]. + /// + /// The structure expected in `indirect_buffer` must conform to [`DrawIndirectArgs`](crate::util::DrawIndirectArgs). + pub fn draw_indirect(&mut self, indirect_buffer: &'a Buffer, indirect_offset: BufferAddress) { + self.inner + .draw_indirect(&indirect_buffer.inner, indirect_offset); + } + + /// Draws indexed primitives using the active index buffer and the active vertex buffers, + /// based on the contents of the `indirect_buffer`. + /// + /// The active index buffer can be set with [`RenderBundleEncoder::set_index_buffer`], while the active + /// vertex buffers can be set with [`RenderBundleEncoder::set_vertex_buffer`]. + /// + /// The structure expected in `indirect_buffer` must conform to [`DrawIndexedIndirectArgs`](crate::util::DrawIndexedIndirectArgs). + pub fn draw_indexed_indirect( + &mut self, + indirect_buffer: &'a Buffer, + indirect_offset: BufferAddress, + ) { + self.inner + .draw_indexed_indirect(&indirect_buffer.inner, indirect_offset); + } + + #[cfg(custom)] + /// Returns custom implementation of RenderBundleEncoder (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// [`Features::IMMEDIATES`] must be enabled on the device in order to call these functions. +impl RenderBundleEncoder<'_> { + /// Set immediate data for subsequent draw calls within the render bundle. + /// + /// Write the bytes in `data` at offset `offset` within immediate data + /// storage. Both `offset` and the length of `data` must be + /// multiples of [`crate::IMMEDIATE_DATA_ALIGNMENT`], which is always 4. + /// + /// For example, if `offset` is `4` and `data` is eight bytes long, this + /// call will write `data` to bytes `4..12` of immediate data storage. + pub fn set_immediates(&mut self, offset: u32, data: &[u8]) { + self.inner.set_immediates(offset, data); + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/render_pass.rs b/third_party/wgpu-29.0.4-patched/src/api/render_pass.rs new file mode 100644 index 00000000..3bd24d2c --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/render_pass.rs @@ -0,0 +1,667 @@ +use core::{num::NonZeroU32, ops::Range}; + +use crate::{ + api::{impl_deferred_command_buffer_actions, SharedDeferredCommandBufferActions}, + *, +}; +pub use wgt::{LoadOp, Operations, StoreOp}; + +/// In-progress recording of a render pass: a list of render commands in a [`CommandEncoder`]. +/// +/// It can be created with [`CommandEncoder::begin_render_pass()`], whose [`RenderPassDescriptor`] +/// specifies the attachments (textures) that will be rendered to. +/// +/// Most of the methods on `RenderPass` serve one of two purposes, identifiable by their names: +/// +/// * `draw_*()`: Drawing (that is, encoding a render command, which, when executed by the GPU, will +/// rasterize something and execute shaders). +/// * `set_*()`: Setting part of the [render state](https://gpuweb.github.io/gpuweb/#renderstate) +/// for future drawing commands. +/// +/// A render pass may contain any number of drawing commands, and before/between each command the +/// render state may be updated however you wish; each drawing command will be executed using the +/// render state that has been set when the `draw_*()` function is called. +/// +/// Corresponds to [WebGPU `GPURenderPassEncoder`]( +/// https://gpuweb.github.io/gpuweb/#render-pass-encoder). +#[derive(Debug)] +pub struct RenderPass<'encoder> { + pub(crate) inner: dispatch::DispatchRenderPass, + pub(crate) actions: SharedDeferredCommandBufferActions, + + /// This lifetime is used to protect the [`CommandEncoder`] from being used + /// while the pass is alive. This needs to be PhantomDrop to prevent the lifetime + /// from being shortened. + pub(crate) _encoder_guard: PhantomDrop<&'encoder ()>, +} + +#[cfg(send_sync)] +static_assertions::assert_impl_all!(RenderPass<'_>: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(RenderPass<'_> => .inner); + +impl RenderPass<'_> { + /// Drops the lifetime relationship to the parent command encoder, making usage of + /// the encoder while this pass is recorded a run-time error instead. + /// + /// Attention: As long as the render pass has not been ended, any mutating operation on the parent + /// command encoder will cause a run-time error and invalidate it! + /// By default, the lifetime constraint prevents this, but it can be useful + /// to handle this at run time, such as when storing the pass and encoder in the same + /// data structure. + /// + /// This operation has no effect on pass recording. + /// It's a safe operation, since [`CommandEncoder`] is in a locked state as long as the pass is active + /// regardless of the lifetime constraint or its absence. + pub fn forget_lifetime(self) -> RenderPass<'static> { + RenderPass { + inner: self.inner, + actions: self.actions, + _encoder_guard: crate::api::PhantomDrop::default(), + } + } + + /// Sets the active bind group for a given bind group index. The bind group layout + /// in the active pipeline when any `draw_*()` method is called must match the layout of + /// this bind group. + /// + /// If the bind group have dynamic offsets, provide them in binding order. + /// These offsets have to be aligned to [`Limits::min_uniform_buffer_offset_alignment`] + /// or [`Limits::min_storage_buffer_offset_alignment`] appropriately. + /// + /// Subsequent draw calls’ shader executions will be able to access data in these bind groups. + pub fn set_bind_group<'a, BG>(&mut self, index: u32, bind_group: BG, offsets: &[DynamicOffset]) + where + Option<&'a BindGroup>: From, + { + let bg: Option<&'a BindGroup> = bind_group.into(); + let bg = bg.map(|bg| &bg.inner); + + self.inner.set_bind_group(index, bg, offsets); + } + + /// Sets the active render pipeline. + /// + /// Subsequent draw calls will exhibit the behavior defined by `pipeline`. + pub fn set_pipeline(&mut self, pipeline: &RenderPipeline) { + self.inner.set_pipeline(&pipeline.inner); + } + + /// Sets the blend color as used by some of the blending modes. + /// + /// Subsequent blending tests will test against this value. + /// If this method has not been called, the blend constant defaults to [`Color::TRANSPARENT`] + /// (all components zero). + pub fn set_blend_constant(&mut self, color: Color) { + self.inner.set_blend_constant(color); + } + + /// Sets the active index buffer. + /// + /// Subsequent calls to [`draw_indexed`](RenderPass::draw_indexed) on this [`RenderPass`] will + /// use `buffer` as the source index buffer. + pub fn set_index_buffer(&mut self, buffer_slice: BufferSlice<'_>, index_format: IndexFormat) { + self.inner.set_index_buffer( + &buffer_slice.buffer.inner, + index_format, + buffer_slice.offset, + Some(buffer_slice.size), + ); + } + + /// Assign a vertex buffer to a slot. + /// + /// Subsequent calls to [`draw`] and [`draw_indexed`] on this + /// [`RenderPass`] will use `buffer` as one of the source vertex buffers. + /// The format of the data in the buffer is specified by the [`VertexBufferLayout`] in the + /// pipeline's [`VertexState`]. + /// + /// The `slot` refers to the index of the matching descriptor in + /// [`VertexState::buffers`]. + /// + /// [`draw`]: RenderPass::draw + /// [`draw_indexed`]: RenderPass::draw_indexed + pub fn set_vertex_buffer(&mut self, slot: u32, buffer_slice: BufferSlice<'_>) { + self.inner.set_vertex_buffer( + slot, + &buffer_slice.buffer.inner, + buffer_slice.offset, + Some(buffer_slice.size), + ); + } + + /// Sets the scissor rectangle used during the rasterization stage. + /// After transformation into [viewport coordinates](https://www.w3.org/TR/webgpu/#viewport-coordinates). + /// + /// Subsequent draw calls will discard any fragments which fall outside the scissor rectangle. + /// If this method has not been called, the scissor rectangle defaults to the entire bounds of + /// the render targets. + /// + /// The function of the scissor rectangle resembles [`set_viewport()`](Self::set_viewport), + /// but it does not affect the coordinate system, only which fragments are discarded. + pub fn set_scissor_rect(&mut self, x: u32, y: u32, width: u32, height: u32) { + self.inner.set_scissor_rect(x, y, width, height); + } + + /// Sets the viewport used during the rasterization stage to linearly map + /// from [normalized device coordinates](https://www.w3.org/TR/webgpu/#ndc) to [viewport coordinates](https://www.w3.org/TR/webgpu/#viewport-coordinates). + /// + /// Subsequent draw calls will only draw within this region. + /// If this method has not been called, the viewport defaults to the entire bounds of the render + /// targets. + pub fn set_viewport(&mut self, x: f32, y: f32, w: f32, h: f32, min_depth: f32, max_depth: f32) { + self.inner.set_viewport(x, y, w, h, min_depth, max_depth); + } + + /// Sets the stencil reference. + /// + /// Subsequent stencil tests will test against this value. + /// If this method has not been called, the stencil reference value defaults to `0`. + pub fn set_stencil_reference(&mut self, reference: u32) { + self.inner.set_stencil_reference(reference); + } + + /// Inserts debug marker. + pub fn insert_debug_marker(&mut self, label: &str) { + self.inner.insert_debug_marker(label); + } + + /// Start record commands and group it into debug marker group. + pub fn push_debug_group(&mut self, label: &str) { + self.inner.push_debug_group(label); + } + + /// Stops command recording and creates debug group. + pub fn pop_debug_group(&mut self) { + self.inner.pop_debug_group(); + } + + /// Draws primitives from the active vertex buffer(s). + /// + /// The active vertex buffer(s) can be set with [`RenderPass::set_vertex_buffer`]. + /// This does not use an index buffer. If you need indexed drawing, see [`RenderPass::draw_indexed`] + /// + /// Panics if `vertices` range is outside of the range of the vertices range of any set vertex buffer. + /// + /// - `vertices`: The range of vertices to draw. + /// - `instances`: Range of instances to draw. Use `0..1` if instance buffers are not used. + /// + /// E.g.of how its used internally + /// ```rust ignore + /// for instance_id in instance_range { + /// for vertex_id in vertex_range { + /// let vertex = vertex[vertex_id]; + /// vertex_shader(vertex, vertex_id, instance_id); + /// } + /// } + /// ``` + /// + /// This drawing command uses the current render state, as set by preceding `set_*()` methods. + /// It is not affected by changes to the state that are performed after it is called. + pub fn draw(&mut self, vertices: Range, instances: Range) { + self.inner.draw(vertices, instances); + } + + /// Draws indexed primitives using the active index buffer and the active vertex buffers. + /// + /// The active index buffer can be set with [`RenderPass::set_index_buffer`] + /// The active vertex buffers can be set with [`RenderPass::set_vertex_buffer`]. + /// + /// Panics if `indices` range is outside of the range of the indices range of the set index buffer. + /// + /// - `indices`: The range of indices to draw. + /// - `base_vertex`: value added to each index value before indexing into the vertex buffers. + /// - `instances`: Range of instances to draw. Use `0..1` if instance buffers are not used. + /// + /// E.g.of how its used internally + /// ```rust ignore + /// for instance_id in instance_range { + /// for index_index in index_range { + /// let vertex_id = index_buffer[index_index]; + /// let adjusted_vertex_id = vertex_id + base_vertex; + /// let vertex = vertex[adjusted_vertex_id]; + /// vertex_shader(vertex, adjusted_vertex_id, instance_id); + /// } + /// } + /// ``` + /// + /// This drawing command uses the current render state, as set by preceding `set_*()` methods. + /// It is not affected by changes to the state that are performed after it is called. + pub fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range) { + self.inner.draw_indexed(indices, base_vertex, instances); + } + + /// Draws using a mesh pipeline. + /// + /// The current pipeline must be a mesh pipeline. + /// + /// If the current pipeline has a task shader, run it with an workgroup for + /// every `vec3(i, j, k)` where `i`, `j`, and `k` are between `0` and + /// `group_count_x`, `group_count_y`, and `group_count_z`. The invocation with + /// index zero in each group is responsible for determining the mesh shader dispatch. + /// Its return value indicates the number of workgroups of mesh shaders to invoke. It also + /// passes a payload value for them to consume. Because each task workgroup is essentially + /// a mesh shader draw call, mesh workgroups dispatched by different task workgroups + /// cannot interact in any way, and `workgroup_id` corresponds to its location in the + /// calling specific task shader's dispatch group. + /// + /// If the current pipeline lacks a task shader, run its mesh shader with a + /// workgroup for every `vec3(i, j, k)` where `i`, `j`, and `k` are + /// between `0` and `group_count_x`, `group_count_y`, and `group_count_z`. + /// + /// Each mesh shader workgroup outputs a set of vertices and indices for primitives. + /// The indices outputted correspond to the vertices outputted by that same workgroup; + /// there is no global vertex buffer. These primitives are passed to the rasterizer and + /// essentially treated like a vertex shader output, except that the mesh shader may + /// choose to cull specific primitives or pass per-primitive non-interpolated values + /// to the fragment shader. As such, each primitive is then rendered with the current + /// pipeline's fragment shader, if present. Otherwise, [No Color Output mode] is used. + /// + /// [No Color Output mode]: https://www.w3.org/TR/webgpu/#no-color-output + pub fn draw_mesh_tasks(&mut self, group_count_x: u32, group_count_y: u32, group_count_z: u32) { + self.inner + .draw_mesh_tasks(group_count_x, group_count_y, group_count_z); + } + + /// Draws primitives from the active vertex buffer(s) based on the contents of the `indirect_buffer`. + /// + /// This is like calling [`RenderPass::draw`] but the contents of the call are specified in the `indirect_buffer`. + /// The structure expected in `indirect_buffer` must conform to [`DrawIndirectArgs`](crate::util::DrawIndirectArgs). + /// + /// Calling this requires the device support [`DownlevelFlags::INDIRECT_EXECUTION`]. + pub fn draw_indirect(&mut self, indirect_buffer: &Buffer, indirect_offset: BufferAddress) { + self.inner + .draw_indirect(&indirect_buffer.inner, indirect_offset); + } + + /// Draws indexed primitives using the active index buffer and the active vertex buffers, + /// based on the contents of the `indirect_buffer`. + /// + /// This is like calling [`RenderPass::draw_indexed`] but the contents of the call are specified in the `indirect_buffer`. + /// The structure expected in `indirect_buffer` must conform to [`DrawIndexedIndirectArgs`](crate::util::DrawIndexedIndirectArgs). + /// + /// Calling this requires the device support [`DownlevelFlags::INDIRECT_EXECUTION`]. + pub fn draw_indexed_indirect( + &mut self, + indirect_buffer: &Buffer, + indirect_offset: BufferAddress, + ) { + self.inner + .draw_indexed_indirect(&indirect_buffer.inner, indirect_offset); + } + + /// Draws using a mesh pipeline, + /// based on the contents of the `indirect_buffer` + /// + /// This is like calling [`RenderPass::draw_mesh_tasks`] but the contents of the call are specified in the `indirect_buffer`. + /// The structure expected in the `indirect_buffer` must conform to [`DispatchIndirectArgs`](crate::util::DispatchIndirectArgs). + /// + /// Indirect drawing has some caveats depending on the features available. We are not currently able to validate + /// these and issue an error. + /// + /// See details on the individual flags for more information. + pub fn draw_mesh_tasks_indirect( + &mut self, + indirect_buffer: &Buffer, + indirect_offset: BufferAddress, + ) { + self.inner + .draw_mesh_tasks_indirect(&indirect_buffer.inner, indirect_offset); + } + + impl_deferred_command_buffer_actions!(); + + /// Execute a [render bundle][RenderBundle], which is a set of pre-recorded commands + /// that can be run together. + /// + /// Commands in the bundle do not inherit this render pass's current render state, and after the + /// bundle has executed, the state is **cleared** (reset to defaults, not the previous state). + pub fn execute_bundles<'a, I: IntoIterator>( + &mut self, + render_bundles: I, + ) { + let mut render_bundles = render_bundles.into_iter().map(|rb| &rb.inner); + + self.inner.execute_bundles(&mut render_bundles); + } + + /// Dispatches multiple draw calls from the active vertex buffer(s) based on the contents of the `indirect_buffer`. + /// `count` draw calls are issued. + /// + /// The active vertex buffers can be set with [`RenderPass::set_vertex_buffer`]. + /// + /// The structure expected in `indirect_buffer` must conform to [`DrawIndirectArgs`](crate::util::DrawIndirectArgs). + /// These draw structures are expected to be tightly packed. + /// + /// Calling this requires the device support [`DownlevelFlags::INDIRECT_EXECUTION`]. + /// + /// This drawing command uses the current render state, as set by preceding `set_*()` methods. + /// It is not affected by changes to the state that are performed after it is called. + pub fn multi_draw_indirect( + &mut self, + indirect_buffer: &Buffer, + indirect_offset: BufferAddress, + count: u32, + ) { + self.inner + .multi_draw_indirect(&indirect_buffer.inner, indirect_offset, count); + } + + /// Dispatches multiple draw calls from the active index buffer and the active vertex buffers, + /// based on the contents of the `indirect_buffer`. `count` draw calls are issued. + /// + /// The active index buffer can be set with [`RenderPass::set_index_buffer`], while the active + /// vertex buffers can be set with [`RenderPass::set_vertex_buffer`]. + /// + /// The structure expected in `indirect_buffer` must conform to [`DrawIndexedIndirectArgs`](crate::util::DrawIndexedIndirectArgs). + /// These draw structures are expected to be tightly packed. + /// + /// Calling this requires the device support [`DownlevelFlags::INDIRECT_EXECUTION`]. + /// + /// This drawing command uses the current render state, as set by preceding `set_*()` methods. + /// It is not affected by changes to the state that are performed after it is called. + pub fn multi_draw_indexed_indirect( + &mut self, + indirect_buffer: &Buffer, + indirect_offset: BufferAddress, + count: u32, + ) { + self.inner + .multi_draw_indexed_indirect(&indirect_buffer.inner, indirect_offset, count); + } + + /// Dispatches multiple draw calls based on the contents of the `indirect_buffer`. + /// `count` draw calls are issued. + /// + /// The structure expected in the `indirect_buffer` must conform to [`DispatchIndirectArgs`](crate::util::DispatchIndirectArgs). + /// + /// This drawing command uses the current render state, as set by preceding `set_*()` methods. + /// It is not affected by changes to the state that are performed after it is called. + pub fn multi_draw_mesh_tasks_indirect( + &mut self, + indirect_buffer: &Buffer, + indirect_offset: BufferAddress, + count: u32, + ) { + self.inner + .multi_draw_mesh_tasks_indirect(&indirect_buffer.inner, indirect_offset, count); + } + + #[cfg(custom)] + /// Returns custom implementation of RenderPass (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// [`Features::MULTI_DRAW_INDIRECT_COUNT`] must be enabled on the device in order to call these functions. +impl RenderPass<'_> { + /// Dispatches multiple draw calls from the active vertex buffer(s) based on the contents of the `indirect_buffer`. + /// The count buffer is read to determine how many draws to issue. + /// + /// The indirect buffer must be long enough to account for `max_count` draws, however only `count` + /// draws will be read. If `count` is greater than `max_count`, `max_count` will be used. + /// + /// The active vertex buffers can be set with [`RenderPass::set_vertex_buffer`]. + /// + /// The structure expected in `indirect_buffer` must conform to [`DrawIndirectArgs`](crate::util::DrawIndirectArgs). + /// These draw structures are expected to be tightly packed. + /// + /// The structure expected in `count_buffer` is the following: + /// + /// ```rust + /// #[repr(C)] + /// struct DrawIndirectCount { + /// count: u32, // Number of draw calls to issue. + /// } + /// ``` + /// + /// This drawing command uses the current render state, as set by preceding `set_*()` methods. + /// It is not affected by changes to the state that are performed after it is called. + pub fn multi_draw_indirect_count( + &mut self, + indirect_buffer: &Buffer, + indirect_offset: BufferAddress, + count_buffer: &Buffer, + count_offset: BufferAddress, + max_count: u32, + ) { + self.inner.multi_draw_indirect_count( + &indirect_buffer.inner, + indirect_offset, + &count_buffer.inner, + count_offset, + max_count, + ); + } + + /// Dispatches multiple draw calls from the active index buffer and the active vertex buffers, + /// based on the contents of the `indirect_buffer`. The count buffer is read to determine how many draws to issue. + /// + /// The indirect buffer must be long enough to account for `max_count` draws, however only `count` + /// draws will be read. If `count` is greater than `max_count`, `max_count` will be used. + /// + /// The active index buffer can be set with [`RenderPass::set_index_buffer`], while the active + /// vertex buffers can be set with [`RenderPass::set_vertex_buffer`]. + /// + /// The structure expected in `indirect_buffer` must conform to [`DrawIndexedIndirectArgs`](crate::util::DrawIndexedIndirectArgs). + /// + /// These draw structures are expected to be tightly packed. + /// + /// The structure expected in `count_buffer` is the following: + /// + /// ```rust + /// #[repr(C)] + /// struct DrawIndexedIndirectCount { + /// count: u32, // Number of draw calls to issue. + /// } + /// ``` + /// + /// This drawing command uses the current render state, as set by preceding `set_*()` methods. + /// It is not affected by changes to the state that are performed after it is called. + pub fn multi_draw_indexed_indirect_count( + &mut self, + indirect_buffer: &Buffer, + indirect_offset: BufferAddress, + count_buffer: &Buffer, + count_offset: BufferAddress, + max_count: u32, + ) { + self.inner.multi_draw_indexed_indirect_count( + &indirect_buffer.inner, + indirect_offset, + &count_buffer.inner, + count_offset, + max_count, + ); + } + + /// Dispatches multiple draw calls based on the contents of the `indirect_buffer`. The count buffer is read to determine how many draws to issue. + /// + /// The indirect buffer must be long enough to account for `max_count` draws, however only `count` + /// draws will be read. If `count` is greater than `max_count`, `max_count` will be used. + /// + /// The structure expected in the `indirect_buffer` must conform to [`DispatchIndirectArgs`](crate::util::DispatchIndirectArgs). + /// + /// These draw structures are expected to be tightly packed. + /// + /// This drawing command uses the current render state, as set by preceding `set_*()` methods. + /// It is not affected by changes to the state that are performed after it is called. + pub fn multi_draw_mesh_tasks_indirect_count( + &mut self, + indirect_buffer: &Buffer, + indirect_offset: BufferAddress, + count_buffer: &Buffer, + count_offset: BufferAddress, + max_count: u32, + ) { + self.inner.multi_draw_mesh_tasks_indirect_count( + &indirect_buffer.inner, + indirect_offset, + &count_buffer.inner, + count_offset, + max_count, + ); + } +} + +/// [`Features::IMMEDIATES`] must be enabled on the device in order to call these functions. +impl RenderPass<'_> { + /// Set immediate data for subsequent draw calls. + /// + /// Write the bytes in `data` at offset `offset` within immediate data + /// storage. Both `offset` and the length of `data` must be + /// multiples of [`crate::IMMEDIATE_DATA_ALIGNMENT`], which is always 4. + /// + /// For example, if `offset` is `4` and `data` is eight bytes long, this + /// call will write `data` to bytes `4..12` of immediate data storage. + pub fn set_immediates(&mut self, offset: u32, data: &[u8]) { + self.inner.set_immediates(offset, data); + } +} + +/// [`Features::TIMESTAMP_QUERY_INSIDE_PASSES`] must be enabled on the device in order to call these functions. +impl RenderPass<'_> { + /// Issue a timestamp command at this point in the queue. The + /// timestamp will be written to the specified query set, at the specified index. + /// + /// Must be multiplied by [`Queue::get_timestamp_period`] to get + /// the value in nanoseconds. Absolute values have no meaning, + /// but timestamps can be subtracted to get the time it takes + /// for a string of operations to complete. + pub fn write_timestamp(&mut self, query_set: &QuerySet, query_index: u32) { + self.inner.write_timestamp(&query_set.inner, query_index); + } +} + +impl RenderPass<'_> { + /// Start a occlusion query on this render pass. It can be ended with + /// [`end_occlusion_query`](Self::end_occlusion_query). + /// Occlusion queries may not be nested. + pub fn begin_occlusion_query(&mut self, query_index: u32) { + self.inner.begin_occlusion_query(query_index); + } + + /// End the occlusion query on this render pass. It can be started with + /// [`begin_occlusion_query`](Self::begin_occlusion_query). + /// Occlusion queries may not be nested. + pub fn end_occlusion_query(&mut self) { + self.inner.end_occlusion_query(); + } +} + +/// [`Features::PIPELINE_STATISTICS_QUERY`] must be enabled on the device in order to call these functions. +impl RenderPass<'_> { + /// Start a pipeline statistics query on this render pass. It can be ended with + /// [`end_pipeline_statistics_query`](Self::end_pipeline_statistics_query). + /// Pipeline statistics queries may not be nested. + /// + /// The amount of information collected by this query, and the space occupied in the query set, + /// is determined by the [`PipelineStatisticsTypes`] the query set was created with. + /// `query_index` is the index of the first query result slot that will be written to, and + /// `query_set` must have sufficient size to hold all results written starting at that slot. + pub fn begin_pipeline_statistics_query(&mut self, query_set: &QuerySet, query_index: u32) { + self.inner + .begin_pipeline_statistics_query(&query_set.inner, query_index); + } + + /// End the pipeline statistics query on this render pass. It can be started with + /// [`begin_pipeline_statistics_query`](Self::begin_pipeline_statistics_query). + /// Pipeline statistics queries may not be nested. + pub fn end_pipeline_statistics_query(&mut self) { + self.inner.end_pipeline_statistics_query(); + } +} + +/// Describes the timestamp writes of a render pass. +/// +/// For use with [`RenderPassDescriptor`]. +/// At least one of [`Self::beginning_of_pass_write_index`] and [`Self::end_of_pass_write_index`] +/// must be `Some`. +/// +/// Corresponds to [WebGPU `GPURenderPassTimestampWrite`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpurenderpasstimestampwrites). +#[derive(Clone, Debug)] +pub struct RenderPassTimestampWrites<'a> { + /// The query set to write to. + pub query_set: &'a QuerySet, + /// The index of the query set at which a start timestamp of this pass is written, if any. + pub beginning_of_pass_write_index: Option, + /// The index of the query set at which an end timestamp of this pass is written, if any. + pub end_of_pass_write_index: Option, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(RenderPassTimestampWrites<'_>: Send, Sync); + +/// Describes a color attachment to a [`RenderPass`]. +/// +/// For use with [`RenderPassDescriptor`]. +/// +/// Corresponds to [WebGPU `GPURenderPassColorAttachment`]( +/// https://gpuweb.github.io/gpuweb/#color-attachments). +#[derive(Clone, Debug)] +pub struct RenderPassColorAttachment<'tex> { + /// The view to use as an attachment. + pub view: &'tex TextureView, + /// The depth slice index of a 3D view. It must not be provided if the view is not 3D. + pub depth_slice: Option, + /// The view that will receive the resolved output if multisampling is used. + /// + /// If set, it is always written to, regardless of how [`Self::ops`] is configured. + pub resolve_target: Option<&'tex TextureView>, + /// What operations will be performed on this color attachment. + pub ops: Operations, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(RenderPassColorAttachment<'_>: Send, Sync); + +/// Describes a depth/stencil attachment to a [`RenderPass`]. +/// +/// For use with [`RenderPassDescriptor`]. +/// +/// Corresponds to [WebGPU `GPURenderPassDepthStencilAttachment`]( +/// https://gpuweb.github.io/gpuweb/#depth-stencil-attachments). +#[derive(Clone, Debug)] +pub struct RenderPassDepthStencilAttachment<'tex> { + /// The view to use as an attachment. + pub view: &'tex TextureView, + /// What operations will be performed on the depth part of the attachment. + pub depth_ops: Option>, + /// What operations will be performed on the stencil part of the attachment. + pub stencil_ops: Option>, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(RenderPassDepthStencilAttachment<'_>: Send, Sync); + +/// Describes the attachments of a render pass. +/// +/// For use with [`CommandEncoder::begin_render_pass`]. +/// +/// Corresponds to [WebGPU `GPURenderPassDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpurenderpassdescriptor). +#[derive(Clone, Debug, Default)] +pub struct RenderPassDescriptor<'a> { + /// Debug label of the render pass. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The color attachments of the render pass. + pub color_attachments: &'a [Option>], + /// The depth and stencil attachment of the render pass, if any. + pub depth_stencil_attachment: Option>, + /// Defines which timestamp values will be written for this pass, and where to write them to. + /// + /// Requires [`Features::TIMESTAMP_QUERY`] to be enabled. + pub timestamp_writes: Option>, + /// Defines where the occlusion query results will be stored for this pass. + pub occlusion_query_set: Option<&'a QuerySet>, + /// The mask of multiview image layers to use for this render pass. For example, if you wish + /// to render to the first 2 layers, you would use 3=0b11. If you wanted ro render to only the + /// 2nd layer, you would use 2=0b10. If you aren't using multiview this should be `None`. + /// + /// Note that setting bits higher than the number of texture layers is a validation error. + /// + /// This doesn't influence load/store/clear/etc operations, as those are defined for attachments, + /// therefore affecting all attachments. Meaning, this affects only any shaders executed on the `RenderPass`. + pub multiview_mask: Option, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(RenderPassDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/render_pipeline.rs b/third_party/wgpu-29.0.4-patched/src/api/render_pipeline.rs new file mode 100644 index 00000000..ebc89633 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/render_pipeline.rs @@ -0,0 +1,334 @@ +use core::num::NonZeroU32; + +use crate::*; + +/// Handle to a rendering (graphics) pipeline. +/// +/// A `RenderPipeline` object represents a graphics pipeline and its stages, bindings, vertex +/// buffers and targets. It can be created with [`Device::create_render_pipeline`]. +/// +/// Corresponds to [WebGPU `GPURenderPipeline`](https://gpuweb.github.io/gpuweb/#render-pipeline). +#[derive(Debug, Clone)] +pub struct RenderPipeline { + pub(crate) inner: dispatch::DispatchRenderPipeline, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(RenderPipeline: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(RenderPipeline => .inner); + +impl RenderPipeline { + /// Get an object representing the bind group layout at a given index. + /// + /// If this pipeline was created with a [default layout][RenderPipelineDescriptor::layout], then + /// bind groups created with the returned `BindGroupLayout` can only be used with this pipeline. + /// + /// This method will raise a validation error if there is no bind group layout at `index`. + pub fn get_bind_group_layout(&self, index: u32) -> BindGroupLayout { + let layout = self.inner.get_bind_group_layout(index); + BindGroupLayout { inner: layout } + } + + #[cfg(custom)] + /// Returns custom implementation of RenderPipeline (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Specifies an interpretation of the bytes of a vertex buffer as vertex attributes. +/// +/// Use this in a [`RenderPipelineDescriptor`] to describe the format of the vertex buffers that +/// are passed to [`RenderPass::set_vertex_buffer()`]. +/// +/// Corresponds to [WebGPU `GPUVertexBufferLayout`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpuvertexbufferlayout). +/// +/// # Example +/// +/// The following example defines a `struct` with three fields, +/// and a [`VertexBufferLayout`] that contains [`VertexAttribute`]s for each field, +/// using the [`vertex_attr_array!`] macro to compute attribute offsets: +/// +/// ``` +/// #[repr(C, packed)] +/// struct Vertex { +/// foo: [f32; 2], +/// bar: f32, +/// baz: [u16; 4], +/// } +/// +/// impl Vertex { +/// /// Layout to use with a buffer whose contents are a `[Vertex]`. +/// pub const LAYOUT: wgpu::VertexBufferLayout<'static> = wgpu::VertexBufferLayout { +/// array_stride: size_of::() as wgpu::BufferAddress, +/// step_mode: wgpu::VertexStepMode::Vertex, +/// attributes: &wgpu::vertex_attr_array![ +/// 0 => Float32x2, +/// 1 => Float32, +/// 2 => Uint16x4, +/// ], +/// }; +/// } +/// +/// # assert_eq!(Vertex::LAYOUT.attributes[2].offset, Vertex::LAYOUT.array_stride - 2 * 4); +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +pub struct VertexBufferLayout<'a> { + /// The stride, in bytes, between elements of this buffer (between vertices). + /// + /// This must be a multiple of [`VERTEX_ALIGNMENT`]. + pub array_stride: BufferAddress, + /// How often this vertex buffer is "stepped" forward. + pub step_mode: VertexStepMode, + /// The list of attributes which comprise a single vertex. + pub attributes: &'a [VertexAttribute], +} +static_assertions::assert_impl_all!(VertexBufferLayout<'_>: Send, Sync); + +/// Describes the vertex processing in a render pipeline. +/// +/// For use in [`RenderPipelineDescriptor`]. +/// +/// Corresponds to [WebGPU `GPUVertexState`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpuvertexstate). +#[derive(Clone, Debug)] +pub struct VertexState<'a> { + /// The compiled shader module for this stage. + pub module: &'a ShaderModule, + /// The name of the entry point in the compiled shader to use. + /// + /// If [`Some`], there must be a vertex-stage shader entry point with this name in `module`. + /// Otherwise, expect exactly one vertex-stage entry point in `module`, which will be + /// selected. + // NOTE: keep phrasing in sync. with `ComputePipelineDescriptor::entry_point` + // NOTE: keep phrasing in sync. with `FragmentState::entry_point` + pub entry_point: Option<&'a str>, + /// Advanced options for when this pipeline is compiled + /// + /// This implements `Default`, and for most users can be set to `Default::default()` + pub compilation_options: PipelineCompilationOptions<'a>, + /// The format of any vertex buffers used with this pipeline via + /// [`RenderPass::set_vertex_buffer()`]. + /// + /// The attribute locations and types specified in this layout must match the + /// locations and types of the inputs to the `entry_point` function. + pub buffers: &'a [VertexBufferLayout<'a>], +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(VertexState<'_>: Send, Sync); + +/// Describes the fragment processing in a render pipeline. +/// +/// For use in [`RenderPipelineDescriptor`]. +/// +/// Corresponds to [WebGPU `GPUFragmentState`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpufragmentstate). +#[derive(Clone, Debug)] +pub struct FragmentState<'a> { + /// The compiled shader module for this stage. + pub module: &'a ShaderModule, + /// The name of the entry point in the compiled shader to use. + /// + /// If [`Some`], there must be a `@fragment` shader entry point with this name in `module`. + /// Otherwise, expect exactly one fragment-stage entry point in `module`, which will be + /// selected. + // NOTE: keep phrasing in sync. with `ComputePipelineDescriptor::entry_point` + // NOTE: keep phrasing in sync. with `VertexState::entry_point` + pub entry_point: Option<&'a str>, + /// Advanced options for when this pipeline is compiled + /// + /// This implements `Default`, and for most users can be set to `Default::default()` + pub compilation_options: PipelineCompilationOptions<'a>, + /// The color state of the render targets. + pub targets: &'a [Option], +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(FragmentState<'_>: Send, Sync); + +/// Describes the task shader stage in a mesh shader pipeline. +/// +/// For use in [`MeshPipelineDescriptor`] +#[derive(Clone, Debug)] +pub struct TaskState<'a> { + /// The compiled shader module for this stage. + pub module: &'a ShaderModule, + + /// The name of the task shader entry point in the shader module to use. + /// + /// If [`Some`], there must be a task shader entry point with the given name + /// in `module`. Otherwise, there must be exactly one task shader entry + /// point in `module`, which will be selected. + pub entry_point: Option<&'a str>, + + /// Advanced options for when this pipeline is compiled. + /// + /// This implements `Default`, and for most users can be set to `Default::default()` + pub compilation_options: PipelineCompilationOptions<'a>, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(TaskState<'_>: Send, Sync); + +/// Describes the mesh shader stage in a mesh shader pipeline. +/// +/// For use in [`MeshPipelineDescriptor`] +#[derive(Clone, Debug)] +pub struct MeshState<'a> { + /// The compiled shader module for this stage. + pub module: &'a ShaderModule, + /// The name of the entry point in the compiled shader to use. + /// + /// If [`Some`], there must be a vertex-stage shader entry point with this name in `module`. + /// Otherwise, expect exactly one vertex-stage entry point in `module`, which will be + /// selected. + pub entry_point: Option<&'a str>, + /// Advanced options for when this pipeline is compiled + /// + /// This implements `Default`, and for most users can be set to `Default::default()` + pub compilation_options: PipelineCompilationOptions<'a>, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(MeshState<'_>: Send, Sync); + +/// Describes a render (graphics) pipeline. +/// +/// For use with [`Device::create_render_pipeline`]. +/// +/// Corresponds to [WebGPU `GPURenderPipelineDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpurenderpipelinedescriptor). +#[derive(Clone, Debug)] +pub struct RenderPipelineDescriptor<'a> { + /// Debug label of the pipeline. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The layout of bind groups for this pipeline. + /// + /// If this is set, then [`Device::create_render_pipeline`] will raise a validation error if + /// the layout doesn't match what the shader module(s) expect. + /// + /// Using the same [`PipelineLayout`] for many [`RenderPipeline`] or [`ComputePipeline`] + /// pipelines guarantees that you don't have to rebind any resources when switching between + /// those pipelines. + /// + /// ## Default pipeline layout + /// + /// If `layout` is `None`, then the pipeline has a [default layout] created and used instead. + /// The default layout is deduced from the shader modules. + /// + /// You can use [`RenderPipeline::get_bind_group_layout`] to create bind groups for use with the + /// default layout. However, these bind groups cannot be used with any other pipelines. This is + /// convenient for simple pipelines, but using an explicit layout is recommended in most cases. + /// + /// [default layout]: https://www.w3.org/TR/webgpu/#default-pipeline-layout + pub layout: Option<&'a PipelineLayout>, + /// The compiled vertex stage, its entry point, and the input buffers layout. + pub vertex: VertexState<'a>, + /// The properties of the pipeline at the primitive assembly and rasterization level. + pub primitive: PrimitiveState, + /// The effect of draw calls on the depth and stencil aspects of the output target, if any. + pub depth_stencil: Option, + /// The multi-sampling properties of the pipeline. + pub multisample: MultisampleState, + /// The compiled fragment stage, its entry point, and the color targets. + pub fragment: Option>, + /// If the pipeline will be used with a multiview render pass, this indicates what multiview + /// mask the render pass will be used with. The masks must match exactly. + /// + /// For example, if you wish to render to the first 2 layers, you would use 3=0b11. If you + /// wanted to render to only the 2nd layer, you would use 2=0b10. If you aren't using + /// multiview this should be `None`. + pub multiview_mask: Option, + /// The pipeline cache to use when creating this pipeline. + pub cache: Option<&'a PipelineCache>, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(RenderPipelineDescriptor<'_>: Send, Sync); + +/// Describes a mesh shader (graphics) pipeline. +/// +/// For use with [`Device::create_mesh_pipeline`]. A mesh pipeline is very much +/// like a render pipeline, except that instead of [`RenderPass::draw`] it is +/// invoked with [`RenderPass::draw_mesh_tasks`], and instead of a vertex shader +/// and a fragment shader: +/// +/// - [`task`] specifies an optional task shader entry point, which determines how +/// many groups of mesh shaders to dispatch. +/// +/// - [`mesh`] specifies a mesh shader entry point, which generates groups of +/// primitives to draw +/// +/// - [`fragment`] specifies as fragment shader for drawing those primitives, +/// just like in an ordinary render pipeline. +/// +/// The key difference is that, whereas a vertex shader is invoked on the +/// elements of vertex buffers, the task shader gets to decide how many mesh +/// shader workgroups to make, and then each mesh shader workgroup gets to +/// decide which primitives it wants to generate, and what their vertex +/// attributes are. Task and mesh shaders can use whatever they please as +/// inputs, like a compute shader. However, they cannot use specialized vertex +/// or index buffers. +/// +/// A mesh pipeline is invoked by [`RenderPass::draw_mesh_tasks`], which looks +/// like a compute shader dispatch with [`ComputePass::dispatch_workgroups`]: +/// you pass `x`, `y`, and `z` values indicating the number of task shaders to +/// invoke in parallel. The output value of the first thread in a task shader +/// workgroup determines how many mesh workgroups should be dispatched from there. +/// Those mesh workgroups also get a special payload passed from the task shader. +/// +/// If the task shader is omitted, then the (`x`, `y`, `z`) parameters to +/// `draw_mesh_tasks` are used to decide how many invocations of the mesh shader +/// to invoke directly, without a task payload. +/// +/// [vertex formats]: wgpu_types::VertexFormat +/// [`task`]: Self::task +/// [`mesh`]: Self::mesh +/// [`fragment`]: Self::fragment +#[derive(Clone, Debug)] +pub struct MeshPipelineDescriptor<'a> { + /// Debug label of the pipeline. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The layout of bind groups for this pipeline. + /// + /// If this is set, then [`Device::create_render_pipeline`] will raise a validation error if + /// the layout doesn't match what the shader module(s) expect. + /// + /// Using the same [`PipelineLayout`] for many [`RenderPipeline`] or [`ComputePipeline`] + /// pipelines guarantees that you don't have to rebind any resources when switching between + /// those pipelines. + /// + /// ## Default pipeline layout + /// + /// If `layout` is `None`, then the pipeline has a [default layout] created and used instead. + /// The default layout is deduced from the shader modules. + /// + /// You can use [`RenderPipeline::get_bind_group_layout`] to create bind groups for use with the + /// default layout. However, these bind groups cannot be used with any other pipelines. This is + /// convenient for simple pipelines, but using an explicit layout is recommended in most cases. + /// + /// [default layout]: https://www.w3.org/TR/webgpu/#default-pipeline-layout + pub layout: Option<&'a PipelineLayout>, + + /// The mesh pipeline's task shader. + /// + /// If this is `None`, the mesh pipeline has no task shader. Executing a + /// mesh drawing command simply dispatches a grid of mesh shaders directly. + /// + /// [`draw_mesh_tasks`]: RenderPass::draw_mesh_tasks + pub task: Option>, + + /// The compiled mesh stage and its entry point + pub mesh: MeshState<'a>, + /// The properties of the pipeline at the primitive assembly and rasterization level. + pub primitive: PrimitiveState, + /// The effect of draw calls on the depth and stencil aspects of the output target, if any. + pub depth_stencil: Option, + /// The multi-sampling properties of the pipeline. + pub multisample: MultisampleState, + /// The compiled fragment stage, its entry point, and the color targets. + pub fragment: Option>, + /// If the pipeline will be used with a multiview render pass, this indicates how many array + /// layers the attachments will have. + pub multiview: Option, + /// The pipeline cache to use when creating this pipeline. + pub cache: Option<&'a PipelineCache>, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(MeshPipelineDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/sampler.rs b/third_party/wgpu-29.0.4-patched/src/api/sampler.rs new file mode 100644 index 00000000..a8916829 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/sampler.rs @@ -0,0 +1,36 @@ +use crate::*; + +/// Handle to a sampler. +/// +/// A `Sampler` object defines how a pipeline will sample from a [`TextureView`]. Samplers define +/// image filters (including anisotropy) and address (wrapping) modes, among other things. See +/// the documentation for [`SamplerDescriptor`] for more information. +/// +/// It can be created with [`Device::create_sampler`]. +/// +/// Corresponds to [WebGPU `GPUSampler`](https://gpuweb.github.io/gpuweb/#sampler-interface). +#[derive(Debug, Clone)] +pub struct Sampler { + pub(crate) inner: dispatch::DispatchSampler, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(Sampler: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(Sampler => .inner); + +impl Sampler { + #[cfg(custom)] + /// Returns custom implementation of Sampler (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Describes a [`Sampler`]. +/// +/// For use with [`Device::create_sampler`]. +/// +/// Corresponds to [WebGPU `GPUSamplerDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpusamplerdescriptor). +pub type SamplerDescriptor<'a> = wgt::SamplerDescriptor>; +static_assertions::assert_impl_all!(SamplerDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/shader_module.rs b/third_party/wgpu-29.0.4-patched/src/api/shader_module.rs new file mode 100644 index 00000000..b2aad03a --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/shader_module.rs @@ -0,0 +1,237 @@ +use alloc::{string::String, vec::Vec}; +use core::{future::Future, marker::PhantomData}; + +use crate::*; + +/// Handle to a compiled shader module. +/// +/// A `ShaderModule` represents a compiled shader module on the GPU. It can be created by passing +/// source code to [`Device::create_shader_module`]. MSL shader or SPIR-V binary can also be passed +/// directly using [`Device::create_shader_module_passthrough`]. Shader modules are used to define +/// programmable stages of a pipeline. +/// +/// Corresponds to [WebGPU `GPUShaderModule`](https://gpuweb.github.io/gpuweb/#shader-module). +#[derive(Debug, Clone)] +pub struct ShaderModule { + pub(crate) inner: dispatch::DispatchShaderModule, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(ShaderModule: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(ShaderModule => .inner); + +impl ShaderModule { + /// Get the compilation info for the shader module. + pub fn get_compilation_info(&self) -> impl Future + WasmNotSend { + self.inner.get_compilation_info() + } + + #[cfg(custom)] + /// Returns custom implementation of ShaderModule (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Compilation information for a shader module. +/// +/// Corresponds to [WebGPU `GPUCompilationInfo`](https://gpuweb.github.io/gpuweb/#gpucompilationinfo). +/// The source locations use bytes, and index a UTF-8 encoded string. +#[derive(Debug, Clone)] +pub struct CompilationInfo { + /// The messages from the shader compilation process. + pub messages: Vec, +} + +/// A single message from the shader compilation process. +/// +/// Roughly corresponds to [`GPUCompilationMessage`](https://www.w3.org/TR/webgpu/#gpucompilationmessage), +/// except that the location uses UTF-8 for all positions. +#[derive(Debug, Clone)] +pub struct CompilationMessage { + /// The text of the message. + pub message: String, + /// The type of the message. + pub message_type: CompilationMessageType, + /// Where in the source code the message points at. + pub location: Option, +} + +/// The type of a compilation message. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CompilationMessageType { + /// An error message. + Error, + /// A warning message. + Warning, + /// An informational message. + Info, +} + +/// A human-readable representation for a span, tailored for text source. +/// +/// Roughly corresponds to the positional members of [`GPUCompilationMessage`][gcm] from +/// the WebGPU specification, except +/// - `offset` and `length` are in bytes (UTF-8 code units), instead of UTF-16 code units. +/// - `line_position` is in bytes (UTF-8 code units), and is usually not directly intended for humans. +/// +/// [gcm]: https://www.w3.org/TR/webgpu/#gpucompilationmessage +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct SourceLocation { + /// 1-based line number. + pub line_number: u32, + /// 1-based column in code units (in bytes) of the start of the span. + /// Remember to convert accordingly when displaying to the user. + pub line_position: u32, + /// 0-based Offset in code units (in bytes) of the start of the span. + pub offset: u32, + /// Length in code units (in bytes) of the span. + pub length: u32, +} + +#[cfg(all(feature = "wgsl", wgpu_core))] +impl From> + for CompilationInfo +{ + fn from(value: crate::naga::error::ShaderError) -> Self { + use alloc::{string::ToString, vec}; + CompilationInfo { + messages: vec![CompilationMessage { + message: value.to_string(), + message_type: CompilationMessageType::Error, + location: value.inner.location(&value.source).map(Into::into), + }], + } + } +} +#[cfg(feature = "glsl")] +impl From> for CompilationInfo { + fn from(value: naga::error::ShaderError) -> Self { + use alloc::string::ToString; + let messages = value + .inner + .errors + .into_iter() + .map(|err| CompilationMessage { + message: err.to_string(), + message_type: CompilationMessageType::Error, + location: err.location(&value.source).map(Into::into), + }) + .collect(); + CompilationInfo { messages } + } +} + +#[cfg(feature = "spirv")] +impl From> for CompilationInfo { + fn from(value: naga::error::ShaderError) -> Self { + use alloc::{string::ToString, vec}; + CompilationInfo { + messages: vec![CompilationMessage { + message: value.to_string(), + message_type: CompilationMessageType::Error, + location: None, + }], + } + } +} + +#[cfg(any(wgpu_core, naga))] +impl + From< + crate::naga::error::ShaderError>, + > for CompilationInfo +{ + fn from( + value: crate::naga::error::ShaderError< + crate::naga::WithSpan, + >, + ) -> Self { + use alloc::{string::ToString, vec}; + CompilationInfo { + messages: vec![CompilationMessage { + message: value.to_string(), + message_type: CompilationMessageType::Error, + location: value.inner.location(&value.source).map(Into::into), + }], + } + } +} + +#[cfg(any(wgpu_core, naga))] +impl From for SourceLocation { + fn from(value: crate::naga::SourceLocation) -> Self { + SourceLocation { + length: value.length, + offset: value.offset, + line_number: value.line_number, + line_position: value.line_position, + } + } +} + +/// Source of a shader module. +/// +/// The source will be parsed and validated. +/// +/// Any necessary shader translation (e.g. from WGSL to SPIR-V or vice versa) +/// will be done internally by wgpu. +/// +/// This type is unique to the Rust API of `wgpu`. In the WebGPU specification, +/// only WGSL source code strings are accepted. +#[cfg_attr(feature = "naga-ir", expect(clippy::large_enum_variant))] +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum ShaderSource<'a> { + /// SPIR-V module represented as a slice of words. + /// + /// See also: [`util::make_spirv`], [`include_spirv`] + #[cfg(feature = "spirv")] + SpirV(alloc::borrow::Cow<'a, [u32]>), + /// GLSL module as a string slice. + /// + /// Note: GLSL is not yet fully supported and must be a specific ShaderStage. + #[cfg(feature = "glsl")] + Glsl { + /// The source code of the shader. + shader: alloc::borrow::Cow<'a, str>, + /// The shader stage that the shader targets. For example, `naga::ShaderStage::Vertex` + stage: naga::ShaderStage, + /// Key-value pairs to represent defines sent to the glsl preprocessor. + /// + /// If the same name is defined multiple times, the last value is used. + defines: &'a [(&'a str, &'a str)], + }, + /// WGSL module as a string slice. + #[cfg(feature = "wgsl")] + Wgsl(alloc::borrow::Cow<'a, str>), + /// Naga module. + #[cfg(feature = "naga-ir")] + Naga(alloc::borrow::Cow<'static, naga::Module>), + /// Dummy variant because `Naga` doesn't have a lifetime and without enough active features it + /// could be the last one active. + #[doc(hidden)] + Dummy(PhantomData<&'a ()>), +} +static_assertions::assert_impl_all!(ShaderSource<'_>: Send, Sync); + +/// Descriptor for use with [`Device::create_shader_module`]. +/// +/// Corresponds to [WebGPU `GPUShaderModuleDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gpushadermoduledescriptor). +#[derive(Clone, Debug)] +pub struct ShaderModuleDescriptor<'a> { + /// Debug label of the shader module. This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// Source code for the shader. + pub source: ShaderSource<'a>, +} +static_assertions::assert_impl_all!(ShaderModuleDescriptor<'_>: Send, Sync); + +/// Descriptor for a shader module given by any of several sources. +/// At least one of the shader types that may be used by the backend must be `Some` +/// +/// This type is unique to the Rust API of `wgpu`. In the WebGPU specification, +/// only WGSL source code strings are accepted. +pub type ShaderModuleDescriptorPassthrough<'a> = + wgt::CreateShaderModuleDescriptorPassthrough<'a, Label<'a>>; diff --git a/third_party/wgpu-29.0.4-patched/src/api/surface.rs b/third_party/wgpu-29.0.4-patched/src/api/surface.rs new file mode 100644 index 00000000..18769bc1 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/surface.rs @@ -0,0 +1,517 @@ +use alloc::{boxed::Box, string::String, vec, vec::Vec}; +#[cfg(wgpu_core)] +use core::ops::Deref; +use core::{error, fmt}; + +use raw_window_handle::{HasDisplayHandle, HasWindowHandle}; + +use crate::util::Mutex; +use crate::*; + +/// Describes a [`Surface`]. +/// +/// For use with [`Surface::configure`]. +/// +/// Corresponds to [WebGPU `GPUCanvasConfiguration`]( +/// https://gpuweb.github.io/gpuweb/#canvas-configuration). +pub type SurfaceConfiguration = wgt::SurfaceConfiguration>; +static_assertions::assert_impl_all!(SurfaceConfiguration: Send, Sync); + +/// Handle to a presentable surface. +/// +/// A `Surface` represents a platform-specific surface (e.g. a window) onto which rendered images may +/// be presented. A `Surface` may be created with the function [`Instance::create_surface`]. +/// +/// This type is unique to the Rust API of `wgpu`. In the WebGPU specification, +/// [`GPUCanvasContext`](https://gpuweb.github.io/gpuweb/#canvas-context) +/// serves a similar role. +pub struct Surface<'window> { + /// Additional surface data returned by [`InstanceInterface::create_surface`][cs]. + /// + /// [cs]: crate::dispatch::InstanceInterface::create_surface + pub(crate) inner: dispatch::DispatchSurface, + + // Stores the latest `SurfaceConfiguration` that was set using `Surface::configure`. + // It is required to set the attributes of the `SurfaceTexture` in the + // `Surface::get_current_texture` method. + // Because the `Surface::configure` method operates on an immutable reference this type has to + // be wrapped in a mutex and since the configuration is only supplied after the surface has + // been created is is additionally wrapped in an option. + pub(crate) config: Mutex>, + + /// Optionally, keep the source of the handle used for the surface alive. + /// + /// This is useful for platforms where the surface is created from a window and the surface + /// would become invalid when the window is dropped. + /// + /// SAFETY: This field must be dropped *after* all other fields to ensure proper cleanup. + pub(crate) _handle_source: Option>, +} + +impl Surface<'_> { + /// Returns the capabilities of the surface when used with the given adapter. + /// + /// Returns specified values (see [`SurfaceCapabilities`]) if surface is incompatible with the adapter. + pub fn get_capabilities(&self, adapter: &Adapter) -> SurfaceCapabilities { + self.inner.get_capabilities(&adapter.inner) + } + + /// Return a default `SurfaceConfiguration` from width and height to use for the [`Surface`] with this adapter. + /// + /// Returns None if the surface isn't supported by this adapter + pub fn get_default_config( + &self, + adapter: &Adapter, + width: u32, + height: u32, + ) -> Option { + let caps = self.get_capabilities(adapter); + Some(SurfaceConfiguration { + usage: wgt::TextureUsages::RENDER_ATTACHMENT, + format: *caps.formats.first()?, + width, + height, + desired_maximum_frame_latency: 2, + present_mode: *caps.present_modes.first()?, + alpha_mode: wgt::CompositeAlphaMode::Auto, + view_formats: vec![], + }) + } + + /// Initializes [`Surface`] for presentation. + /// + /// If the surface is already configured, this will wait for the GPU to come idle + /// before recreating the swapchain to prevent race conditions. + /// + /// # Validation Errors + /// - Submissions that happen _during_ the configure may cause the + /// internal wait-for-idle to fail, raising a validation error. + /// + /// # Panics + /// + /// - A old [`SurfaceTexture`] is still alive referencing an old surface. + /// - Texture format requested is unsupported on the surface. + /// - `config.width` or `config.height` is zero. + pub fn configure(&self, device: &Device, config: &SurfaceConfiguration) { + self.inner.configure(&device.inner, config); + + let mut conf = self.config.lock(); + *conf = Some(config.clone()); + } + + /// Returns the current configuration of [`Surface`], if configured. + /// + /// This is similar to [WebGPU `GPUcCanvasContext::getConfiguration`](https://gpuweb.github.io/gpuweb/#dom-gpucanvascontext-getconfiguration). + pub fn get_configuration(&self) -> Option { + self.config.lock().clone() + } + + /// Returns the next texture to be presented by the surface for drawing. + /// + /// In order to present the [`SurfaceTexture`] returned by this method, + /// first a [`Queue::submit`] needs to be done with some work rendering to this texture. + /// Then [`SurfaceTexture::present`] needs to be called. + /// + /// If a [`SurfaceTexture`] referencing this surface is alive when [`Surface::configure()`] + /// is called, the configure call will panic. + /// + /// See the documentation of [`CurrentSurfaceTexture`] for how each possible result + /// should be handled. + pub fn get_current_texture(&self) -> CurrentSurfaceTexture { + let (texture, status, detail) = self.inner.get_current_texture(); + + let suboptimal = match status { + SurfaceStatus::Good => false, + SurfaceStatus::Suboptimal => true, + SurfaceStatus::Timeout => return CurrentSurfaceTexture::Timeout, + SurfaceStatus::Occluded => return CurrentSurfaceTexture::Occluded, + SurfaceStatus::Outdated => return CurrentSurfaceTexture::Outdated, + SurfaceStatus::Lost => return CurrentSurfaceTexture::Lost, + SurfaceStatus::Validation => return CurrentSurfaceTexture::Validation, + }; + + let guard = self.config.lock(); + let config = guard + .as_ref() + .expect("This surface has not been configured yet."); + + let descriptor = TextureDescriptor { + label: None, + size: Extent3d { + width: config.width, + height: config.height, + depth_or_array_layers: 1, + }, + format: config.format, + usage: config.usage, + mip_level_count: 1, + sample_count: 1, + dimension: TextureDimension::D2, + view_formats: &[], + }; + + match texture { + Some(texture) => { + let surface_texture = SurfaceTexture { + texture: Texture { + inner: texture, + descriptor, + }, + presented: false, + detail, + }; + if suboptimal { + CurrentSurfaceTexture::Suboptimal(surface_texture) + } else { + CurrentSurfaceTexture::Success(surface_texture) + } + } + None => CurrentSurfaceTexture::Lost, + } + } + + /// Get the [`wgpu_hal`] surface from this `Surface`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::Surface`]. + /// + /// # Types + /// + /// The returned type depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Surface")] + #[doc = crate::macros::hal_type_metal!("Surface")] + #[doc = crate::macros::hal_type_dx12!("Surface")] + #[doc = crate::macros::hal_type_gles!("Surface")] + /// + /// # Errors + /// + /// This method will return None if: + /// - The surface is not from the backend specified by `A`. + /// - The surface is from the `webgpu` or `custom` backend. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::Surface`]: hal::Api::Surface + #[cfg(wgpu_core)] + pub unsafe fn as_hal( + &self, + ) -> Option + WasmNotSendSync> { + let core_surface = self.inner.as_core_opt()?; + + unsafe { core_surface.context.surface_as_hal::(core_surface) } + } + + #[cfg(custom)] + /// Returns custom implementation of Surface (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +// This custom implementation is required because [`Surface::_surface`] doesn't +// require [`Debug`](fmt::Debug), which we should not require from the user. +impl fmt::Debug for Surface<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Surface") + .field( + "_handle_source", + &if self._handle_source.is_some() { + "Some" + } else { + "None" + }, + ) + .field("inner", &self.inner) + .field("config", &self.config) + .finish() + } +} + +#[cfg(send_sync)] +static_assertions::assert_impl_all!(Surface<'_>: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(Surface<'_> => .inner); + +/// [`Send`]/[`Sync`] blanket trait for [`HasWindowHandle`] used in [`SurfaceTarget`]. +pub trait WindowHandle: HasWindowHandle + WasmNotSendSync {} + +impl WindowHandle for T {} + +/// Super trait for a pair of display and window handles as used in [`SurfaceTarget`]. +pub trait DisplayAndWindowHandle: WindowHandle + HasDisplayHandle {} + +impl DisplayAndWindowHandle for T where T: WindowHandle + HasDisplayHandle {} + +/// The window/canvas/surface/swap-chain/etc. a surface is attached to, for use with safe surface creation. +/// +/// This is either a window or an actual web canvas depending on the platform and +/// enabled features. +/// Refer to the individual variants for more information. +/// +/// See also [`SurfaceTargetUnsafe`] for unsafe variants. +#[non_exhaustive] +pub enum SurfaceTarget<'window> { + /// Window and display handle producer. + /// + /// If the specified display and window handle are not supported by any of the backends, then the surface + /// will not be supported by any adapters. + /// + /// # Errors + /// + /// - On WebGL2: surface creation returns an error if the browser does not support WebGL2, + /// or declines to provide GPU access (such as due to a resource shortage). + /// + /// # Panics + /// + /// - On macOS/Metal: will panic if not called on the main thread. + /// - On web: will panic if the [`HasWindowHandle`] does not properly refer to a + /// canvas element. + /// - On all platforms: If [`crate::InstanceDescriptor::display`] was not [`None`] + /// but its value is not identical to that returned by [`HasDisplayHandle::display_handle()`]. + DisplayAndWindow(Box), + + /// Window handle producer. + /// + /// [`HasWindowHandle`]-only version of [`SurfaceTarget::DisplayAndWindow`]. + /// + /// This requires that the display handle was already passed through + /// [`crate::InstanceDescriptor::display`]. + Window(Box), + + /// Surface from a `web_sys::HtmlCanvasElement`. + /// + /// The `canvas` argument must be a valid `` element to + /// create a surface upon. + /// + /// # Errors + /// + /// - On WebGL2: surface creation will return an error if the browser does not support WebGL2, + /// or declines to provide GPU access (such as due to a resource shortage). + #[cfg(web)] + Canvas(web_sys::HtmlCanvasElement), + + /// Surface from a `web_sys::OffscreenCanvas`. + /// + /// The `canvas` argument must be a valid `OffscreenCanvas` object + /// to create a surface upon. + /// + /// # Errors + /// + /// - On WebGL2: surface creation will return an error if the browser does not support WebGL2, + /// or declines to provide GPU access (such as due to a resource shortage). + #[cfg(web)] + OffscreenCanvas(web_sys::OffscreenCanvas), +} + +impl<'a> SurfaceTarget<'a> { + /// Constructor for [`Self::Window`] without consuming a display handle + pub fn from_window_without_display(window: impl WindowHandle + 'a) -> Self { + Self::Window(Box::new(window)) + } +} + +impl<'a, T> From for SurfaceTarget<'a> +where + T: DisplayAndWindowHandle + 'a, +{ + fn from(window: T) -> Self { + Self::DisplayAndWindow(Box::new(window)) + } +} + +/// The window/canvas/surface/swap-chain/etc. a surface is attached to, for use with unsafe surface creation. +/// +/// This is either a window or an actual web canvas depending on the platform and +/// enabled features. +/// Refer to the individual variants for more information. +/// +/// See also [`SurfaceTarget`] for safe variants. +#[non_exhaustive] +pub enum SurfaceTargetUnsafe { + /// Raw window & display handle. + /// + /// If the specified display and window handle are not supported by any of the backends, then the surface + /// will not be supported by any adapters. + /// + /// If the `raw_display_handle` is not [`None`] here and was not [`None`] in + /// [`crate::InstanceDescriptor::display`], their values _must_ be identical. + /// + /// # Safety + /// + /// - `raw_window_handle` & `raw_display_handle` must be valid objects to create a surface upon. + /// - `raw_window_handle` & `raw_display_handle` must remain valid until after the returned + /// [`Surface`] is dropped. + RawHandle { + /// Raw display handle, underlying display must outlive the surface created from this. + raw_display_handle: Option, + + /// Raw window handle, underlying window must outlive the surface created from this. + raw_window_handle: raw_window_handle::RawWindowHandle, + }, + + /// Surface from a DRM device. + /// + /// If the specified DRM configuration is not supported by any of the backends, then the surface + /// will not be supported by any adapters. + /// + /// # Safety + /// + /// - All parameters must point to valid DRM values and remain valid for as long as the resulting [`Surface`] exists. + /// - The file descriptor (`fd`), plane, connector, and mode configuration must be valid and compatible. + #[cfg(all(unix, not(target_vendor = "apple"), not(target_family = "wasm")))] + Drm { + /// The file descriptor of the DRM device. + fd: i32, + /// The plane index on which to create the surface. + plane: u32, + /// The ID of the connector associated with the selected mode. + connector_id: u32, + /// The display width of the selected mode. + width: u32, + /// The display height of the selected mode. + height: u32, + /// The display refresh rate of the selected mode multiplied by 1000 (e.g., 60Hz → 60000). + refresh_rate: u32, + }, + + /// Surface from `CoreAnimationLayer`. + /// + /// # Safety + /// + /// - layer must be a valid object to create a surface upon. + #[cfg(metal)] + CoreAnimationLayer(*mut core::ffi::c_void), + + /// Surface from `IDCompositionVisual`. + /// + /// # Safety + /// + /// - visual must be a valid `IDCompositionVisual` to create a surface upon. Its refcount will be incremented internally and kept live as long as the resulting [`Surface`] is live. + #[cfg(dx12)] + CompositionVisual(*mut core::ffi::c_void), + + /// Surface from DX12 `DirectComposition` handle. + /// + /// + /// + /// # Safety + /// + /// - surface_handle must be a valid `DirectComposition` handle to create a surface upon. Its lifetime **will not** be internally managed: this handle **should not** be freed before + /// the resulting [`Surface`] is destroyed. + #[cfg(dx12)] + SurfaceHandle(*mut core::ffi::c_void), + + /// Surface from DX12 `SwapChainPanel`. + /// + /// # Safety + /// + /// - visual must be a valid SwapChainPanel to create a surface upon. Its refcount will be incremented internally and kept live as long as the resulting [`Surface`] is live. + #[cfg(dx12)] + SwapChainPanel(*mut core::ffi::c_void), +} + +impl SurfaceTargetUnsafe { + /// Creates a [`SurfaceTargetUnsafe::RawHandle`] from a display and window. + /// + /// The `display` is optional and may be omitted if it was also passed to + /// [`crate::InstanceDescriptor::display`]. If passed to both it must (currently) be identical. + /// + /// # Safety + /// + /// - `display` must outlive the resulting surface target + /// (and subsequently the surface created for this target). + /// - `window` must outlive the resulting surface target + /// (and subsequently the surface created for this target). + pub unsafe fn from_display_and_window( + display: &impl HasDisplayHandle, + window: &impl HasWindowHandle, + ) -> Result { + Ok(Self::RawHandle { + raw_display_handle: Some(display.display_handle()?.as_raw()), + raw_window_handle: window.window_handle()?.as_raw(), + }) + } + + /// Creates a [`SurfaceTargetUnsafe::RawHandle`] from a window. + /// + /// # Safety + /// + /// - `window` must outlive the resulting surface target + /// (and subsequently the surface created for this target). + pub unsafe fn from_window( + window: &impl HasWindowHandle, + ) -> Result { + Ok(Self::RawHandle { + raw_display_handle: None, + raw_window_handle: window.window_handle()?.as_raw(), + }) + } +} + +/// [`Instance::create_surface()`] or a related function failed. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct CreateSurfaceError { + pub(crate) inner: CreateSurfaceErrorKind, +} +#[derive(Clone, Debug)] +pub(crate) enum CreateSurfaceErrorKind { + /// Error from [`wgpu_hal`]. + #[cfg(wgpu_core)] + Hal(wgc::instance::CreateSurfaceError), + + /// Error from WebGPU surface creation. + #[cfg_attr(not(webgpu), expect(dead_code))] + Web(String), + + /// Error when trying to get a [`RawDisplayHandle`][rdh] or a + /// [`RawWindowHandle`][rwh] from a [`SurfaceTarget`]. + /// + /// [rdh]: raw_window_handle::RawDisplayHandle + /// [rwh]: raw_window_handle::RawWindowHandle + RawHandle(raw_window_handle::HandleError), +} +static_assertions::assert_impl_all!(CreateSurfaceError: Send, Sync); + +impl fmt::Display for CreateSurfaceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.inner { + #[cfg(wgpu_core)] + CreateSurfaceErrorKind::Hal(e) => e.fmt(f), + CreateSurfaceErrorKind::Web(e) => e.fmt(f), + CreateSurfaceErrorKind::RawHandle(e) => e.fmt(f), + } + } +} + +impl error::Error for CreateSurfaceError { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match &self.inner { + #[cfg(wgpu_core)] + CreateSurfaceErrorKind::Hal(e) => e.source(), + CreateSurfaceErrorKind::Web(_) => None, + #[cfg(feature = "std")] + CreateSurfaceErrorKind::RawHandle(e) => e.source(), + #[cfg(not(feature = "std"))] + CreateSurfaceErrorKind::RawHandle(_) => None, + } + } +} + +#[cfg(wgpu_core)] +impl From for CreateSurfaceError { + fn from(e: wgc::instance::CreateSurfaceError) -> Self { + Self { + inner: CreateSurfaceErrorKind::Hal(e), + } + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/surface_texture.rs b/third_party/wgpu-29.0.4-patched/src/api/surface_texture.rs new file mode 100644 index 00000000..f08c4621 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/surface_texture.rs @@ -0,0 +1,106 @@ +use crate::*; + +/// Surface texture that can be rendered to. +/// Result of a successful call to [`Surface::get_current_texture`]. +/// +/// This type is unique to the Rust API of `wgpu`. In the WebGPU specification, +/// the [`GPUCanvasContext`](https://gpuweb.github.io/gpuweb/#canvas-context) provides +/// a texture without any additional information. +#[derive(Debug, Clone)] +pub struct SurfaceTexture { + /// Accessible view of the frame. + pub texture: Texture, + pub(crate) presented: bool, + pub(crate) detail: dispatch::DispatchSurfaceOutputDetail, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(SurfaceTexture: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(SurfaceTexture => .texture.inner); + +impl SurfaceTexture { + /// Schedule this texture to be presented on the owning surface. + /// + /// Needs to be called after any work on the texture is scheduled via [`Queue::submit`]. + /// + /// # Platform dependent behavior + /// + /// On Wayland, `present` will attach a `wl_buffer` to the underlying `wl_surface` and commit the new surface + /// state. If it is desired to do things such as request a frame callback, scale the surface using the viewporter + /// or synchronize other double buffered state, then these operations should be done before the call to `present`. + pub fn present(mut self) { + self.presented = true; + self.detail.present(); + } + + #[cfg(custom)] + /// Returns custom implementation of SurfaceTexture (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.detail.as_custom() + } +} + +impl Drop for SurfaceTexture { + fn drop(&mut self) { + if !self.presented && !thread_panicking() { + self.detail.texture_discard(); + } + } +} + +/// Result of a call to [`Surface::get_current_texture`]. +/// +/// See variant documentation for how to handle each case. +#[derive(Debug)] +pub enum CurrentSurfaceTexture { + /// Successfully acquired a surface texture with no issues. + Success(SurfaceTexture), + /// Successfully acquired a surface texture, but texture no longer matches the properties of the underlying surface. + /// It's highly recommended to call [`Surface::configure`] again for optimal performance. + Suboptimal(SurfaceTexture), + /// A timeout was encountered while trying to acquire the next frame. + /// + /// Applications should skip the current frame and try again later. + Timeout, + /// The window is occluded (e.g. minimized or behind another window). + /// + /// Applications should skip the current frame and try again once the window + /// is no longer occluded. + Occluded, + /// The underlying surface has changed, and therefore the surface configuration is outdated. + /// + /// Call [`Surface::configure()`] and try again. + Outdated, + /// The surface has been lost and needs to be recreated. + /// + /// If the device as a whole is lost (see [`set_device_lost_callback()`][crate::Device::set_device_lost_callback]), then + /// you need to recreate the device and all resources. + /// Otherwise, call [`Instance::create_surface()`] to recreate the surface, + /// then [`Surface::configure()`], and try again. + Lost, + /// A validation error inside [`Surface::get_current_texture()`] was raised + /// and caught by an [error scope](crate::Device::push_error_scope) or + /// [`on_uncaptured_error()`][crate::Device::on_uncaptured_error]. + /// + /// Applications should attend to the validation error and try again. + Validation, +} + +fn thread_panicking() -> bool { + cfg_if::cfg_if! { + if #[cfg(std)] { + std::thread::panicking() + } else if #[cfg(panic = "abort")] { + // If `panic = "abort"` then a thread _cannot_ be observably panicking by definition. + false + } else { + // TODO: This is potentially overly pessimistic; it may be appropriate to instead allow a + // texture to not be discarded. + // Alternatively, this could _also_ be a `panic!`, since we only care if the thread is panicking + // when the surface has not been presented. + compile_error!( + "cannot determine if a thread is panicking without either `panic = \"abort\"` or `std`" + ); + } + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/api/texture.rs b/third_party/wgpu-29.0.4-patched/src/api/texture.rs new file mode 100644 index 00000000..a2be922b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/texture.rs @@ -0,0 +1,188 @@ +#[cfg(wgpu_core)] +use core::ops::Deref; + +use crate::*; + +/// Handle to a texture on the GPU. +/// +/// It can be created with [`Device::create_texture`]. +/// +/// Corresponds to [WebGPU `GPUTexture`](https://gpuweb.github.io/gpuweb/#texture-interface). +#[derive(Debug, Clone)] +pub struct Texture { + pub(crate) inner: dispatch::DispatchTexture, + pub(crate) descriptor: TextureDescriptor<'static>, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(Texture: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(Texture => .inner); + +impl Texture { + /// Get the [`wgpu_hal`] texture from this `Texture`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::Texture`]. + /// + /// # Types + /// + /// The returned type depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("Texture")] + #[doc = crate::macros::hal_type_metal!("Texture")] + #[doc = crate::macros::hal_type_dx12!("Texture")] + #[doc = crate::macros::hal_type_gles!("Texture")] + /// + /// # Deadlocks + /// + /// - The returned guard holds a read-lock on a device-local "destruction" + /// lock, which will cause all calls to `destroy` to block until the + /// guard is released. + /// + /// # Errors + /// + /// This method will return None if: + /// - The texture is not from the backend specified by `A`. + /// - The texture is from the `webgpu` or `custom` backend. + /// - The texture has had [`Self::destroy()`] called on it. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::Texture`]: hal::Api::Texture + #[cfg(wgpu_core)] + pub unsafe fn as_hal(&self) -> Option> { + let texture = self.inner.as_core_opt()?; + unsafe { texture.context.texture_as_hal::(texture) } + } + + #[cfg(custom)] + /// Returns custom implementation of Texture (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } + + #[cfg(custom)] + /// Creates a texture from already created custom implementation with the given description + pub fn from_custom( + texture: T, + desc: &TextureDescriptor<'_>, + ) -> Self { + Self { + inner: dispatch::DispatchTexture::custom(texture), + descriptor: TextureDescriptor { + label: None, + view_formats: &[], + ..desc.clone() + }, + } + } + + /// Creates a view of this texture, specifying an interpretation of its texels and + /// possibly a subset of its layers and mip levels. + /// + /// Texture views are needed to use a texture as a binding in a [`BindGroup`] + /// or as an attachment in a [`RenderPass`]. + pub fn create_view(&self, desc: &TextureViewDescriptor<'_>) -> TextureView { + let view = self.inner.create_view(desc); + + TextureView { + inner: view, + texture: self.clone(), + } + } + + /// Destroy the associated native resources as soon as possible. + pub fn destroy(&self) { + self.inner.destroy(); + } + + /// Make an `TexelCopyTextureInfo` representing the whole texture. + pub fn as_image_copy(&self) -> TexelCopyTextureInfo<'_> { + TexelCopyTextureInfo { + texture: self, + mip_level: 0, + origin: Origin3d::ZERO, + aspect: TextureAspect::All, + } + } + + /// Returns the size of this `Texture`. + /// + /// This is always equal to the `size` that was specified when creating the texture. + pub fn size(&self) -> Extent3d { + self.descriptor.size + } + + /// Returns the width of this `Texture`. + /// + /// This is always equal to the `size.width` that was specified when creating the texture. + pub fn width(&self) -> u32 { + self.descriptor.size.width + } + + /// Returns the height of this `Texture`. + /// + /// This is always equal to the `size.height` that was specified when creating the texture. + pub fn height(&self) -> u32 { + self.descriptor.size.height + } + + /// Returns the depth or layer count of this `Texture`. + /// + /// This is always equal to the `size.depth_or_array_layers` that was specified when creating the texture. + pub fn depth_or_array_layers(&self) -> u32 { + self.descriptor.size.depth_or_array_layers + } + + /// Returns the mip_level_count of this `Texture`. + /// + /// This is always equal to the `mip_level_count` that was specified when creating the texture. + pub fn mip_level_count(&self) -> u32 { + self.descriptor.mip_level_count + } + + /// Returns the sample_count of this `Texture`. + /// + /// This is always equal to the `sample_count` that was specified when creating the texture. + pub fn sample_count(&self) -> u32 { + self.descriptor.sample_count + } + + /// Returns the dimension of this `Texture`. + /// + /// This is always equal to the `dimension` that was specified when creating the texture. + pub fn dimension(&self) -> TextureDimension { + self.descriptor.dimension + } + + /// Returns the format of this `Texture`. + /// + /// This is always equal to the `format` that was specified when creating the texture. + pub fn format(&self) -> TextureFormat { + self.descriptor.format + } + + /// Returns the allowed usages of this `Texture`. + /// + /// This is always equal to the `usage` that was specified when creating the texture. + pub fn usage(&self) -> TextureUsages { + self.descriptor.usage + } +} + +/// Describes a [`Texture`]. +/// +/// For use with [`Device::create_texture`]. +/// +/// Corresponds to [WebGPU `GPUTextureDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gputexturedescriptor). +pub type TextureDescriptor<'a> = wgt::TextureDescriptor, &'a [TextureFormat]>; +static_assertions::assert_impl_all!(TextureDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/texture_view.rs b/third_party/wgpu-29.0.4-patched/src/api/texture_view.rs new file mode 100644 index 00000000..62aa45c4 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/texture_view.rs @@ -0,0 +1,92 @@ +#[cfg(wgpu_core)] +use core::ops::Deref; + +use crate::*; + +/// Handle to a texture view. +/// +/// A `TextureView` object refers to a [`Texture`], or a subset of its layers and mip levels, and +/// specifies an interpretation of the texture’s texels, which is needed to use a texture as a +/// binding in a [`BindGroup`] or as an attachment in a [`RenderPass`]. +/// It can be created using [`Texture::create_view()`], which accepts a [`TextureViewDescriptor`] +/// specifying the properties of the view. +/// +/// Corresponds to [WebGPU `GPUTextureView`](https://gpuweb.github.io/gpuweb/#gputextureview). +#[derive(Debug, Clone)] +pub struct TextureView { + pub(crate) inner: dispatch::DispatchTextureView, + pub(crate) texture: Texture, +} +#[cfg(send_sync)] +static_assertions::assert_impl_all!(TextureView: Send, Sync); + +crate::cmp::impl_eq_ord_hash_proxy!(TextureView => .inner); + +impl TextureView { + /// Returns the [`Texture`] that this `TextureView` refers to. + /// + /// All wgpu resources are refcounted, so you can own the returned [`Texture`] + /// by cloning it. + pub fn texture(&self) -> &Texture { + &self.texture + } + + /// Get the [`wgpu_hal`] texture view from this `TextureView`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::TextureView`]. + /// + /// # Types + /// + /// The returned type depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("TextureView")] + #[doc = crate::macros::hal_type_metal!("TextureView")] + #[doc = crate::macros::hal_type_dx12!("TextureView")] + #[doc = crate::macros::hal_type_gles!("TextureView")] + /// + /// # Deadlocks + /// + /// - The returned guard holds a read-lock on a device-local "destruction" + /// lock, which will cause all calls to `destroy` to block until the + /// guard is released. + /// + /// # Errors + /// + /// This method will return None if: + /// - The texture view is not from the backend specified by `A`. + /// - The texture view is from the `webgpu` or `custom` backend. + /// - The texture this view points to has had [`Texture::destroy()`] called on it. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::TextureView`]: hal::Api::TextureView + #[cfg(wgpu_core)] + pub unsafe fn as_hal(&self) -> Option> { + let view = self.inner.as_core_opt()?; + unsafe { view.context.texture_view_as_hal::(view) } + } + + #[cfg(custom)] + /// Returns custom implementation of TextureView (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } +} + +/// Describes a [`TextureView`]. +/// +/// For use with [`Texture::create_view`]. +/// +/// Corresponds to [WebGPU `GPUTextureViewDescriptor`]( +/// https://gpuweb.github.io/gpuweb/#dictdef-gputextureviewdescriptor). +pub type TextureViewDescriptor<'a> = wgt::TextureViewDescriptor>; +static_assertions::assert_impl_all!(TextureViewDescriptor<'_>: Send, Sync); diff --git a/third_party/wgpu-29.0.4-patched/src/api/tlas.rs b/third_party/wgpu-29.0.4-patched/src/api/tlas.rs new file mode 100644 index 00000000..b897824d --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/api/tlas.rs @@ -0,0 +1,165 @@ +use crate::{api::blas::TlasInstance, dispatch}; +use crate::{BindingResource, Label}; +use alloc::vec::Vec; +#[cfg(wgpu_core)] +use core::ops::Deref; +use core::ops::{Index, IndexMut, Range}; +use wgt::WasmNotSendSync; + +/// Descriptor to create top level acceleration structures. +pub type CreateTlasDescriptor<'a> = wgt::CreateTlasDescriptor>; +static_assertions::assert_impl_all!(CreateTlasDescriptor<'_>: Send, Sync); + +#[derive(Debug, Clone)] +/// Top Level Acceleration Structure (TLAS). +/// +/// A TLAS contains a series of [TLAS instances], which are a reference to +/// a BLAS and a transformation matrix placing the geometry in the world. +/// +/// A TLAS also contains an extra set of TLAS instances in a device readable form, you cant interact +/// directly with these, instead you have to build the TLAS with [TLAS instances]. +/// +/// [TLAS instances]: TlasInstance +pub struct Tlas { + pub(crate) inner: dispatch::DispatchTlas, + pub(crate) instances: Vec>, + pub(crate) lowest_unmodified: u32, +} +static_assertions::assert_impl_all!(Tlas: WasmNotSendSync); + +crate::cmp::impl_eq_ord_hash_proxy!(Tlas => .inner); + +impl Tlas { + /// Get the [`wgpu_hal`] acceleration structure from this `Tlas`. + /// + /// Find the Api struct corresponding to the active backend in [`wgpu_hal::api`], + /// and pass that struct to the to the `A` type parameter. + /// + /// Returns a guard that dereferences to the type of the hal backend + /// which implements [`A::AccelerationStructure`]. + /// + /// # Types + /// + /// The returned type depends on the backend: + /// + #[doc = crate::macros::hal_type_vulkan!("AccelerationStructure")] + #[doc = crate::macros::hal_type_metal!("AccelerationStructure")] + #[doc = crate::macros::hal_type_dx12!("AccelerationStructure")] + #[doc = crate::macros::hal_type_gles!("AccelerationStructure")] + /// + /// # Deadlocks + /// + /// - The returned guard holds a read-lock on a device-local "destruction" + /// lock, which will cause all calls to `destroy` to block until the + /// guard is released. + /// + /// # Errors + /// + /// This method will return None if: + /// - The acceleration structure is not from the backend specified by `A`. + /// - The acceleration structure is from the `webgpu` or `custom` backend. + /// + /// # Safety + /// + /// - The returned resource must not be destroyed unless the guard + /// is the last reference to it and it is not in use by the GPU. + /// The guard and handle may be dropped at any time however. + /// - All the safety requirements of wgpu-hal must be upheld. + /// + /// [`A::AccelerationStructure`]: hal::Api::AccelerationStructure + #[cfg(wgpu_core)] + pub unsafe fn as_hal( + &mut self, + ) -> Option> { + let tlas = self.inner.as_core_opt()?; + unsafe { tlas.context.tlas_as_hal::(tlas) } + } + + #[cfg(custom)] + /// Returns custom implementation of Tlas (if custom backend and is internally T) + pub fn as_custom(&self) -> Option<&T> { + self.inner.as_custom() + } + + /// Get a reference to all instances. + pub fn get(&self) -> &[Option] { + &self.instances + } + + /// Get a mutable slice to a range of instances. + /// Returns None if the range is out of bounds. + /// All elements from the lowest accessed index up are marked as modified. + // this recommendation is not useful yet, but is likely to be when ability to update arrives or possible optimisations for building get implemented. + /// For best performance it is recommended to prefer access to low elements and modify higher elements as little as possible. + /// This can be done by ordering instances from the most to the least used. It is recommended + /// to use [`Self::index_mut`] unless the option if out of bounds is required + pub fn get_mut_slice(&mut self, range: Range) -> Option<&mut [Option]> { + if range.end > self.instances.len() { + return None; + } + if range.end as u32 > self.lowest_unmodified { + self.lowest_unmodified = range.end as u32; + } + Some(&mut self.instances[range]) + } + + /// Get a single mutable reference to an instance. + /// Returns None if the range is out of bounds. + /// All elements from the lowest accessed index up are marked as modified. + // this recommendation is not useful yet, but is likely to be when ability to update arrives or possible optimisations for building get implemented. + /// For best performance it is recommended to prefer access to low elements and modify higher elements as little as possible. + /// This can be done by ordering instances from the most to the least used. It is recommended + /// to use [`Self::index_mut`] unless the option if out of bounds is required + pub fn get_mut_single(&mut self, index: usize) -> Option<&mut Option> { + if index >= self.instances.len() { + return None; + } + if index as u32 + 1 > self.lowest_unmodified { + self.lowest_unmodified = index as u32 + 1; + } + Some(&mut self.instances[index]) + } + + /// Get the binding resource for the underling acceleration structure, to be used when creating a [`BindGroup`] + /// + /// [`BindGroup`]: super::BindGroup + pub fn as_binding(&self) -> BindingResource<'_> { + BindingResource::AccelerationStructure(self) + } +} + +impl Index for Tlas { + type Output = Option; + + fn index(&self, index: usize) -> &Self::Output { + self.instances.index(index) + } +} + +impl Index> for Tlas { + type Output = [Option]; + + fn index(&self, index: Range) -> &Self::Output { + self.instances.index(index) + } +} + +impl IndexMut for Tlas { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + let idx = self.instances.index_mut(index); + if index as u32 + 1 > self.lowest_unmodified { + self.lowest_unmodified = index as u32 + 1; + } + idx + } +} + +impl IndexMut> for Tlas { + fn index_mut(&mut self, index: Range) -> &mut Self::Output { + let idx = self.instances.index_mut(index.clone()); + if index.end > self.lowest_unmodified as usize { + self.lowest_unmodified = index.end as u32; + } + idx + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/custom.rs b/third_party/wgpu-29.0.4-patched/src/backend/custom.rs new file mode 100644 index 00000000..4d205e00 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/custom.rs @@ -0,0 +1,99 @@ +//! Provides wrappers custom backend implementations + +#![allow(ambiguous_wide_pointer_comparisons)] + +pub use crate::dispatch::*; + +use alloc::sync::Arc; + +macro_rules! dyn_type { + // cloning of arc forbidden + // but we still use it to provide Eq,Ord,Hash implementations + (pub mut struct $name:ident(dyn $interface:tt)) => { + #[derive(Debug)] + pub(crate) struct $name(Arc); + crate::cmp::impl_eq_ord_hash_arc_address!($name => .0); + + impl $name { + pub(crate) fn new(t: T) -> Self { + Self(Arc::new(t)) + } + + #[allow(clippy::allow_attributes, dead_code)] + pub(crate) fn downcast(&self) -> Option<&T> { + self.0.as_ref().as_any().downcast_ref() + } + } + + impl core::ops::Deref for $name { + type Target = dyn $interface; + + #[inline] + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } + } + + impl core::ops::DerefMut for $name { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + Arc::get_mut(&mut self.0).expect("") + } + } + }; + // cloning of arc is allowed + (pub ref struct $name:ident(dyn $interface:tt)) => { + #[derive(Debug, Clone)] + pub(crate) struct $name(Arc); + crate::cmp::impl_eq_ord_hash_arc_address!($name => .0); + + impl $name { + pub(crate) fn new(t: T) -> Self { + Self(Arc::new(t)) + } + + pub(crate) fn downcast(&self) -> Option<&T> { + self.0.as_ref().as_any().downcast_ref() + } + } + + impl core::ops::Deref for $name { + type Target = dyn $interface; + + #[inline] + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } + } + }; +} + +dyn_type!(pub ref struct DynContext(dyn InstanceInterface)); +dyn_type!(pub ref struct DynAdapter(dyn AdapterInterface)); +dyn_type!(pub ref struct DynDevice(dyn DeviceInterface)); +dyn_type!(pub ref struct DynQueue(dyn QueueInterface)); +dyn_type!(pub ref struct DynShaderModule(dyn ShaderModuleInterface)); +dyn_type!(pub ref struct DynBindGroupLayout(dyn BindGroupLayoutInterface)); +dyn_type!(pub ref struct DynBindGroup(dyn BindGroupInterface)); +dyn_type!(pub ref struct DynTextureView(dyn TextureViewInterface)); +dyn_type!(pub ref struct DynSampler(dyn SamplerInterface)); +dyn_type!(pub ref struct DynBuffer(dyn BufferInterface)); +dyn_type!(pub ref struct DynTexture(dyn TextureInterface)); +dyn_type!(pub ref struct DynExternalTexture(dyn ExternalTextureInterface)); +dyn_type!(pub ref struct DynBlas(dyn BlasInterface)); +dyn_type!(pub ref struct DynTlas(dyn TlasInterface)); +dyn_type!(pub ref struct DynQuerySet(dyn QuerySetInterface)); +dyn_type!(pub ref struct DynPipelineLayout(dyn PipelineLayoutInterface)); +dyn_type!(pub ref struct DynRenderPipeline(dyn RenderPipelineInterface)); +dyn_type!(pub ref struct DynComputePipeline(dyn ComputePipelineInterface)); +dyn_type!(pub ref struct DynPipelineCache(dyn PipelineCacheInterface)); +dyn_type!(pub mut struct DynCommandEncoder(dyn CommandEncoderInterface)); +dyn_type!(pub mut struct DynComputePass(dyn ComputePassInterface)); +dyn_type!(pub mut struct DynRenderPass(dyn RenderPassInterface)); +dyn_type!(pub mut struct DynCommandBuffer(dyn CommandBufferInterface)); +dyn_type!(pub mut struct DynRenderBundleEncoder(dyn RenderBundleEncoderInterface)); +dyn_type!(pub ref struct DynRenderBundle(dyn RenderBundleInterface)); +dyn_type!(pub ref struct DynSurface(dyn SurfaceInterface)); +dyn_type!(pub ref struct DynSurfaceOutputDetail(dyn SurfaceOutputDetailInterface)); +dyn_type!(pub mut struct DynQueueWriteBuffer(dyn QueueWriteBufferInterface)); +dyn_type!(pub mut struct DynBufferMappedRange(dyn BufferMappedRangeInterface)); diff --git a/third_party/wgpu-29.0.4-patched/src/backend/mod.rs b/third_party/wgpu-29.0.4-patched/src/backend/mod.rs new file mode 100644 index 00000000..1d1a40c2 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/mod.rs @@ -0,0 +1,13 @@ +#[cfg(webgpu)] +pub mod webgpu; +#[cfg(webgpu)] +pub(crate) use webgpu::{get_browser_gpu_property, ContextWebGpu}; + +#[cfg(wgpu_core)] +pub mod wgpu_core; + +#[cfg(wgpu_core)] +pub(crate) use wgpu_core::ContextWgpuCore; + +#[cfg(custom)] +pub mod custom; diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu.rs new file mode 100644 index 00000000..4796bdb7 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu.rs @@ -0,0 +1,4057 @@ +#![allow(clippy::type_complexity)] + +mod defined_non_null_js_value; +mod ext_bindings; +#[allow(clippy::allow_attributes)] +mod webgpu_sys; + +use alloc::{ + boxed::Box, + format, + rc::Rc, + string::{String, ToString as _}, + sync::Arc, + vec, + vec::Vec, +}; +use core::{ + cell::{Cell, OnceCell, RefCell}, + fmt, + future::Future, + ops::Range, + pin::Pin, + task::{self, Poll}, +}; +use wgt::Backends; + +use js_sys::Promise; +use wasm_bindgen::{prelude::*, JsCast}; + +use crate::{ + dispatch::{self, BlasCompactCallback}, + Blas, SurfaceTargetUnsafe, Tlas, WriteOnly, +}; + +use defined_non_null_js_value::DefinedNonNullJsValue; + +// We need to mark various types as Send and Sync to satisfy the Rust type system. +// +// SAFETY: All webgpu handle types in wasm32 are internally a `JsValue`, and `JsValue` is neither +// Send nor Sync. Currently, wasm32 has no threading support by default, so implementing `Send` or +// `Sync` for a type is harmless. However, nightly Rust supports compiling wasm with experimental +// threading support via `--target-features`. If `wgpu` is being compiled with those features, we do +// not implement `Send` and `Sync` on the webgpu handle types. +macro_rules! impl_send_sync { + ($name:ty) => { + #[cfg(send_sync)] + unsafe impl Send for $name {} + #[cfg(send_sync)] + unsafe impl Sync for $name {} + }; +} + +#[derive(Clone)] +pub struct ContextWebGpu { + /// `None` if browser does not advertise support for WebGPU. + gpu: Option>, + /// Unique identifier for this context. + ident: crate::cmp::Identifier, + /// Backends requested in the [`crate::InstanceDescriptor`]. + /// Remembered for error reporting even though this itself is strictly + /// [`Backends::BROWSER_WEBGPU`]. + requested_backends: Backends, +} + +impl fmt::Debug for ContextWebGpu { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ContextWebGpu") + .field("type", &"Web") + .finish() + } +} + +impl crate::Error { + fn from_js(js_error: js_sys::Object) -> Self { + let source = Box::::from(""); + if let Some(js_error) = js_error.dyn_ref::() { + crate::Error::Validation { + source, + description: js_error.message(), + } + } else if js_error.has_type::() { + crate::Error::OutOfMemory { source } + } else { + panic!("Unexpected error"); + } + } +} + +#[derive(Debug, Clone)] +pub struct WebShaderModule { + module: webgpu_sys::GpuShaderModule, + compilation_info: WebShaderCompilationInfo, + /// Unique identifier for this shader module. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +enum WebShaderCompilationInfo { + /// WGSL shaders get their compilation info from a native WebGPU function. + /// We need the source to be able to do UTF16 to UTF8 location remapping. + Wgsl { source: String }, + /// Transformed shaders get their compilation info from the transformer. + /// Further compilation errors are reported without a span. + Transformed { + compilation_info: crate::CompilationInfo, + }, +} + +fn map_utf16_to_utf8_offset(utf16_offset: u32, text: &str) -> u32 { + let mut utf16_i = 0; + for (utf8_index, c) in text.char_indices() { + if utf16_i >= utf16_offset { + return utf8_index as u32; + } + utf16_i += c.len_utf16() as u32; + } + if utf16_i >= utf16_offset { + text.len() as u32 + } else { + log::error!("UTF16 offset {utf16_offset} is out of bounds for string {text}"); + u32::MAX + } +} + +impl crate::CompilationMessage { + fn from_js( + js_message: webgpu_sys::GpuCompilationMessage, + compilation_info: &WebShaderCompilationInfo, + ) -> Self { + let message_type = match js_message.type_() { + webgpu_sys::GpuCompilationMessageType::Error => crate::CompilationMessageType::Error, + webgpu_sys::GpuCompilationMessageType::Warning => { + crate::CompilationMessageType::Warning + } + webgpu_sys::GpuCompilationMessageType::Info => crate::CompilationMessageType::Info, + _ => crate::CompilationMessageType::Error, + }; + let utf16_offset = js_message.offset() as u32; + let utf16_length = js_message.length() as u32; + let span = match compilation_info { + WebShaderCompilationInfo::Wgsl { .. } if utf16_offset == 0 && utf16_length == 0 => None, + WebShaderCompilationInfo::Wgsl { source } => { + let offset = map_utf16_to_utf8_offset(utf16_offset, source); + let length = map_utf16_to_utf8_offset(utf16_length, &source[offset as usize..]); + let line_number = js_message.line_num() as u32; // That's legal, because we're counting lines the same way + + let prefix = &source[..offset as usize]; + let line_start = prefix.rfind('\n').map(|pos| pos + 1).unwrap_or(0) as u32; + let line_position = offset - line_start + 1; // Counting UTF-8 byte indices + + Some(crate::SourceLocation { + offset, + length, + line_number, + line_position, + }) + } + WebShaderCompilationInfo::Transformed { .. } => None, + }; + + crate::CompilationMessage { + message: js_message.message(), + message_type, + location: span, + } + } +} + +// We need to assert that any future we return is Send to match the native API. +// +// This is safe on wasm32 *for now*, but similarly to the unsafe Send impls for the handle type +// wrappers, the full story for threading on wasm32 is still unfolding. + +pub(crate) struct MakeSendFuture { + future: F, + map: M, +} + +impl T, T> Future for MakeSendFuture { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll { + // This is safe because we have no Drop implementation to violate the Pin requirements and + // do not provide any means of moving the inner future. + unsafe { + let this = self.get_unchecked_mut(); + match Pin::new_unchecked(&mut this.future).poll(cx) { + task::Poll::Ready(value) => task::Poll::Ready((this.map)(value)), + task::Poll::Pending => task::Poll::Pending, + } + } + } +} + +impl MakeSendFuture { + fn new(future: F, map: M) -> Self { + Self { future, map } + } +} + +#[cfg(send_sync)] +unsafe impl Send for MakeSendFuture {} + +fn map_texture_format(texture_format: wgt::TextureFormat) -> webgpu_sys::GpuTextureFormat { + use webgpu_sys::GpuTextureFormat as tf; + use wgt::TextureFormat; + match texture_format { + // 8-bit formats + TextureFormat::R8Unorm => tf::R8unorm, + TextureFormat::R8Snorm => tf::R8snorm, + TextureFormat::R8Uint => tf::R8uint, + TextureFormat::R8Sint => tf::R8sint, + // 16-bit formats + TextureFormat::R16Uint => tf::R16uint, + TextureFormat::R16Sint => tf::R16sint, + TextureFormat::R16Float => tf::R16float, + TextureFormat::Rg8Unorm => tf::Rg8unorm, + TextureFormat::Rg8Snorm => tf::Rg8snorm, + TextureFormat::Rg8Uint => tf::Rg8uint, + TextureFormat::Rg8Sint => tf::Rg8sint, + // 32-bit formats + TextureFormat::R32Uint => tf::R32uint, + TextureFormat::R32Sint => tf::R32sint, + TextureFormat::R32Float => tf::R32float, + TextureFormat::Rg16Uint => tf::Rg16uint, + TextureFormat::Rg16Sint => tf::Rg16sint, + TextureFormat::Rg16Float => tf::Rg16float, + TextureFormat::Rgba8Unorm => tf::Rgba8unorm, + TextureFormat::Rgba8UnormSrgb => tf::Rgba8unormSrgb, + TextureFormat::Rgba8Snorm => tf::Rgba8snorm, + TextureFormat::Rgba8Uint => tf::Rgba8uint, + TextureFormat::Rgba8Sint => tf::Rgba8sint, + TextureFormat::Bgra8Unorm => tf::Bgra8unorm, + TextureFormat::Bgra8UnormSrgb => tf::Bgra8unormSrgb, + // Packed 32-bit formats + TextureFormat::Rgb9e5Ufloat => tf::Rgb9e5ufloat, + TextureFormat::Rgb10a2Uint => tf::Rgb10a2uint, + TextureFormat::Rgb10a2Unorm => tf::Rgb10a2unorm, + TextureFormat::Rg11b10Ufloat => tf::Rg11b10ufloat, + // 64-bit formats + TextureFormat::Rg32Uint => tf::Rg32uint, + TextureFormat::Rg32Sint => tf::Rg32sint, + TextureFormat::Rg32Float => tf::Rg32float, + TextureFormat::Rgba16Uint => tf::Rgba16uint, + TextureFormat::Rgba16Sint => tf::Rgba16sint, + TextureFormat::Rgba16Float => tf::Rgba16float, + // 128-bit formats + TextureFormat::Rgba32Uint => tf::Rgba32uint, + TextureFormat::Rgba32Sint => tf::Rgba32sint, + TextureFormat::Rgba32Float => tf::Rgba32float, + // Depth/stencil formats + TextureFormat::Stencil8 => tf::Stencil8, + TextureFormat::Depth16Unorm => tf::Depth16unorm, + TextureFormat::Depth24Plus => tf::Depth24plus, + TextureFormat::Depth24PlusStencil8 => tf::Depth24plusStencil8, + TextureFormat::Depth32Float => tf::Depth32float, + // "depth32float-stencil8" feature + TextureFormat::Depth32FloatStencil8 => tf::Depth32floatStencil8, + + TextureFormat::Bc1RgbaUnorm => tf::Bc1RgbaUnorm, + TextureFormat::Bc1RgbaUnormSrgb => tf::Bc1RgbaUnormSrgb, + TextureFormat::Bc2RgbaUnorm => tf::Bc2RgbaUnorm, + TextureFormat::Bc2RgbaUnormSrgb => tf::Bc2RgbaUnormSrgb, + TextureFormat::Bc3RgbaUnorm => tf::Bc3RgbaUnorm, + TextureFormat::Bc3RgbaUnormSrgb => tf::Bc3RgbaUnormSrgb, + TextureFormat::Bc4RUnorm => tf::Bc4RUnorm, + TextureFormat::Bc4RSnorm => tf::Bc4RSnorm, + TextureFormat::Bc5RgUnorm => tf::Bc5RgUnorm, + TextureFormat::Bc5RgSnorm => tf::Bc5RgSnorm, + TextureFormat::Bc6hRgbUfloat => tf::Bc6hRgbUfloat, + TextureFormat::Bc6hRgbFloat => tf::Bc6hRgbFloat, + TextureFormat::Bc7RgbaUnorm => tf::Bc7RgbaUnorm, + TextureFormat::Bc7RgbaUnormSrgb => tf::Bc7RgbaUnormSrgb, + TextureFormat::Etc2Rgb8Unorm => tf::Etc2Rgb8unorm, + TextureFormat::Etc2Rgb8UnormSrgb => tf::Etc2Rgb8unormSrgb, + TextureFormat::Etc2Rgb8A1Unorm => tf::Etc2Rgb8a1unorm, + TextureFormat::Etc2Rgb8A1UnormSrgb => tf::Etc2Rgb8a1unormSrgb, + TextureFormat::Etc2Rgba8Unorm => tf::Etc2Rgba8unorm, + TextureFormat::Etc2Rgba8UnormSrgb => tf::Etc2Rgba8unormSrgb, + TextureFormat::EacR11Unorm => tf::EacR11unorm, + TextureFormat::EacR11Snorm => tf::EacR11snorm, + TextureFormat::EacRg11Unorm => tf::EacRg11unorm, + TextureFormat::EacRg11Snorm => tf::EacRg11snorm, + TextureFormat::Astc { block, channel } => match channel { + wgt::AstcChannel::Unorm => match block { + wgt::AstcBlock::B4x4 => tf::Astc4x4Unorm, + wgt::AstcBlock::B5x4 => tf::Astc5x4Unorm, + wgt::AstcBlock::B5x5 => tf::Astc5x5Unorm, + wgt::AstcBlock::B6x5 => tf::Astc6x5Unorm, + wgt::AstcBlock::B6x6 => tf::Astc6x6Unorm, + wgt::AstcBlock::B8x5 => tf::Astc8x5Unorm, + wgt::AstcBlock::B8x6 => tf::Astc8x6Unorm, + wgt::AstcBlock::B8x8 => tf::Astc8x8Unorm, + wgt::AstcBlock::B10x5 => tf::Astc10x5Unorm, + wgt::AstcBlock::B10x6 => tf::Astc10x6Unorm, + wgt::AstcBlock::B10x8 => tf::Astc10x8Unorm, + wgt::AstcBlock::B10x10 => tf::Astc10x10Unorm, + wgt::AstcBlock::B12x10 => tf::Astc12x10Unorm, + wgt::AstcBlock::B12x12 => tf::Astc12x12Unorm, + }, + wgt::AstcChannel::UnormSrgb => match block { + wgt::AstcBlock::B4x4 => tf::Astc4x4UnormSrgb, + wgt::AstcBlock::B5x4 => tf::Astc5x4UnormSrgb, + wgt::AstcBlock::B5x5 => tf::Astc5x5UnormSrgb, + wgt::AstcBlock::B6x5 => tf::Astc6x5UnormSrgb, + wgt::AstcBlock::B6x6 => tf::Astc6x6UnormSrgb, + wgt::AstcBlock::B8x5 => tf::Astc8x5UnormSrgb, + wgt::AstcBlock::B8x6 => tf::Astc8x6UnormSrgb, + wgt::AstcBlock::B8x8 => tf::Astc8x8UnormSrgb, + wgt::AstcBlock::B10x5 => tf::Astc10x5UnormSrgb, + wgt::AstcBlock::B10x6 => tf::Astc10x6UnormSrgb, + wgt::AstcBlock::B10x8 => tf::Astc10x8UnormSrgb, + wgt::AstcBlock::B10x10 => tf::Astc10x10UnormSrgb, + wgt::AstcBlock::B12x10 => tf::Astc12x10UnormSrgb, + wgt::AstcBlock::B12x12 => tf::Astc12x12UnormSrgb, + }, + wgt::AstcChannel::Hdr => { + unimplemented!("Format {texture_format:?} has no WebGPU equivalent") + } + }, + _ => unimplemented!("Format {texture_format:?} has no WebGPU equivalent"), + } +} + +fn map_texture_component_type( + sample_type: wgt::TextureSampleType, +) -> webgpu_sys::GpuTextureSampleType { + use webgpu_sys::GpuTextureSampleType as ts; + use wgt::TextureSampleType; + match sample_type { + TextureSampleType::Float { filterable: true } => ts::Float, + TextureSampleType::Float { filterable: false } => ts::UnfilterableFloat, + TextureSampleType::Sint => ts::Sint, + TextureSampleType::Uint => ts::Uint, + TextureSampleType::Depth => ts::Depth, + } +} + +fn map_cull_mode(cull_mode: Option) -> webgpu_sys::GpuCullMode { + use webgpu_sys::GpuCullMode as cm; + use wgt::Face; + match cull_mode { + None => cm::None, + Some(Face::Front) => cm::Front, + Some(Face::Back) => cm::Back, + } +} + +fn map_front_face(front_face: wgt::FrontFace) -> webgpu_sys::GpuFrontFace { + use webgpu_sys::GpuFrontFace as ff; + use wgt::FrontFace; + match front_face { + FrontFace::Ccw => ff::Ccw, + FrontFace::Cw => ff::Cw, + } +} + +fn map_primitive_state(primitive: &wgt::PrimitiveState) -> webgpu_sys::GpuPrimitiveState { + use webgpu_sys::GpuPrimitiveTopology as pt; + use wgt::PrimitiveTopology; + + let mapped = webgpu_sys::GpuPrimitiveState::new(); + mapped.set_cull_mode(map_cull_mode(primitive.cull_mode)); + mapped.set_front_face(map_front_face(primitive.front_face)); + + if let Some(format) = primitive.strip_index_format { + mapped.set_strip_index_format(map_index_format(format)); + } + + mapped.set_topology(match primitive.topology { + PrimitiveTopology::PointList => pt::PointList, + PrimitiveTopology::LineList => pt::LineList, + PrimitiveTopology::LineStrip => pt::LineStrip, + PrimitiveTopology::TriangleList => pt::TriangleList, + PrimitiveTopology::TriangleStrip => pt::TriangleStrip, + }); + + mapped.set_unclipped_depth(primitive.unclipped_depth); + + match primitive.polygon_mode { + wgt::PolygonMode::Fill => {} + wgt::PolygonMode::Line => panic!( + "{:?} is not enabled for this backend", + wgt::Features::POLYGON_MODE_LINE + ), + wgt::PolygonMode::Point => panic!( + "{:?} is not enabled for this backend", + wgt::Features::POLYGON_MODE_POINT + ), + } + + mapped +} + +fn map_compare_function(compare_fn: wgt::CompareFunction) -> webgpu_sys::GpuCompareFunction { + use webgpu_sys::GpuCompareFunction as cf; + use wgt::CompareFunction; + match compare_fn { + CompareFunction::Never => cf::Never, + CompareFunction::Less => cf::Less, + CompareFunction::Equal => cf::Equal, + CompareFunction::LessEqual => cf::LessEqual, + CompareFunction::Greater => cf::Greater, + CompareFunction::NotEqual => cf::NotEqual, + CompareFunction::GreaterEqual => cf::GreaterEqual, + CompareFunction::Always => cf::Always, + } +} + +fn map_stencil_operation(op: wgt::StencilOperation) -> webgpu_sys::GpuStencilOperation { + use webgpu_sys::GpuStencilOperation as so; + use wgt::StencilOperation; + match op { + StencilOperation::Keep => so::Keep, + StencilOperation::Zero => so::Zero, + StencilOperation::Replace => so::Replace, + StencilOperation::Invert => so::Invert, + StencilOperation::IncrementClamp => so::IncrementClamp, + StencilOperation::DecrementClamp => so::DecrementClamp, + StencilOperation::IncrementWrap => so::IncrementWrap, + StencilOperation::DecrementWrap => so::DecrementWrap, + } +} + +fn map_stencil_state_face(desc: &wgt::StencilFaceState) -> webgpu_sys::GpuStencilFaceState { + let mapped = webgpu_sys::GpuStencilFaceState::new(); + mapped.set_compare(map_compare_function(desc.compare)); + mapped.set_depth_fail_op(map_stencil_operation(desc.depth_fail_op)); + mapped.set_fail_op(map_stencil_operation(desc.fail_op)); + mapped.set_pass_op(map_stencil_operation(desc.pass_op)); + mapped +} + +fn map_depth_stencil_state(desc: &wgt::DepthStencilState) -> webgpu_sys::GpuDepthStencilState { + let mapped = webgpu_sys::GpuDepthStencilState::new(map_texture_format(desc.format)); + if let Some(compare) = desc.depth_compare { + mapped.set_depth_compare(map_compare_function(compare)); + } + if let Some(write_enabled) = desc.depth_write_enabled { + mapped.set_depth_write_enabled(write_enabled); + } + mapped.set_depth_bias(desc.bias.constant); + mapped.set_depth_bias_clamp(desc.bias.clamp); + mapped.set_depth_bias_slope_scale(desc.bias.slope_scale); + mapped.set_stencil_back(&map_stencil_state_face(&desc.stencil.back)); + mapped.set_stencil_front(&map_stencil_state_face(&desc.stencil.front)); + mapped.set_stencil_read_mask(desc.stencil.read_mask); + mapped.set_stencil_write_mask(desc.stencil.write_mask); + mapped +} + +fn map_blend_component(desc: &wgt::BlendComponent) -> webgpu_sys::GpuBlendComponent { + let mapped = webgpu_sys::GpuBlendComponent::new(); + mapped.set_dst_factor(map_blend_factor(desc.dst_factor)); + mapped.set_operation(map_blend_operation(desc.operation)); + mapped.set_src_factor(map_blend_factor(desc.src_factor)); + mapped +} + +fn map_blend_factor(factor: wgt::BlendFactor) -> webgpu_sys::GpuBlendFactor { + use webgpu_sys::GpuBlendFactor as bf; + use wgt::BlendFactor; + match factor { + BlendFactor::Zero => bf::Zero, + BlendFactor::One => bf::One, + BlendFactor::Src => bf::Src, + BlendFactor::OneMinusSrc => bf::OneMinusSrc, + BlendFactor::SrcAlpha => bf::SrcAlpha, + BlendFactor::OneMinusSrcAlpha => bf::OneMinusSrcAlpha, + BlendFactor::Dst => bf::Dst, + BlendFactor::OneMinusDst => bf::OneMinusDst, + BlendFactor::DstAlpha => bf::DstAlpha, + BlendFactor::OneMinusDstAlpha => bf::OneMinusDstAlpha, + BlendFactor::SrcAlphaSaturated => bf::SrcAlphaSaturated, + BlendFactor::Constant => bf::Constant, + BlendFactor::OneMinusConstant => bf::OneMinusConstant, + BlendFactor::Src1 => bf::Src1, + BlendFactor::OneMinusSrc1 => bf::OneMinusSrc1, + BlendFactor::Src1Alpha => bf::Src1Alpha, + BlendFactor::OneMinusSrc1Alpha => bf::OneMinusSrc1Alpha, + } +} + +fn map_blend_operation(op: wgt::BlendOperation) -> webgpu_sys::GpuBlendOperation { + use webgpu_sys::GpuBlendOperation as bo; + use wgt::BlendOperation; + match op { + BlendOperation::Add => bo::Add, + BlendOperation::Subtract => bo::Subtract, + BlendOperation::ReverseSubtract => bo::ReverseSubtract, + BlendOperation::Min => bo::Min, + BlendOperation::Max => bo::Max, + } +} + +fn map_index_format(format: wgt::IndexFormat) -> webgpu_sys::GpuIndexFormat { + use webgpu_sys::GpuIndexFormat as f; + use wgt::IndexFormat; + match format { + IndexFormat::Uint16 => f::Uint16, + IndexFormat::Uint32 => f::Uint32, + } +} + +fn map_vertex_format(format: wgt::VertexFormat) -> webgpu_sys::GpuVertexFormat { + use webgpu_sys::GpuVertexFormat as vf; + use wgt::VertexFormat; + match format { + VertexFormat::Uint8 => vf::Uint8, + VertexFormat::Uint8x2 => vf::Uint8x2, + VertexFormat::Uint8x4 => vf::Uint8x4, + VertexFormat::Sint8 => vf::Sint8, + VertexFormat::Sint8x2 => vf::Sint8x2, + VertexFormat::Sint8x4 => vf::Sint8x4, + VertexFormat::Unorm8 => vf::Unorm8, + VertexFormat::Unorm8x2 => vf::Unorm8x2, + VertexFormat::Unorm8x4 => vf::Unorm8x4, + VertexFormat::Snorm8 => vf::Snorm8, + VertexFormat::Snorm8x2 => vf::Snorm8x2, + VertexFormat::Snorm8x4 => vf::Snorm8x4, + VertexFormat::Uint16 => vf::Uint16, + VertexFormat::Uint16x2 => vf::Uint16x2, + VertexFormat::Uint16x4 => vf::Uint16x4, + VertexFormat::Sint16 => vf::Sint16, + VertexFormat::Sint16x2 => vf::Sint16x2, + VertexFormat::Sint16x4 => vf::Sint16x4, + VertexFormat::Unorm16 => vf::Unorm16, + VertexFormat::Unorm16x2 => vf::Unorm16x2, + VertexFormat::Unorm16x4 => vf::Unorm16x4, + VertexFormat::Snorm16 => vf::Snorm16, + VertexFormat::Snorm16x2 => vf::Snorm16x2, + VertexFormat::Snorm16x4 => vf::Snorm16x4, + VertexFormat::Float16 => vf::Float16, + VertexFormat::Float16x2 => vf::Float16x2, + VertexFormat::Float16x4 => vf::Float16x4, + VertexFormat::Float32 => vf::Float32, + VertexFormat::Float32x2 => vf::Float32x2, + VertexFormat::Float32x3 => vf::Float32x3, + VertexFormat::Float32x4 => vf::Float32x4, + VertexFormat::Uint32 => vf::Uint32, + VertexFormat::Uint32x2 => vf::Uint32x2, + VertexFormat::Uint32x3 => vf::Uint32x3, + VertexFormat::Uint32x4 => vf::Uint32x4, + VertexFormat::Sint32 => vf::Sint32, + VertexFormat::Sint32x2 => vf::Sint32x2, + VertexFormat::Sint32x3 => vf::Sint32x3, + VertexFormat::Sint32x4 => vf::Sint32x4, + VertexFormat::Unorm10_10_10_2 => vf::Unorm1010102, + VertexFormat::Unorm8x4Bgra => vf::Unorm8x4Bgra, + VertexFormat::Float64 + | VertexFormat::Float64x2 + | VertexFormat::Float64x3 + | VertexFormat::Float64x4 => { + panic!("VERTEX_ATTRIBUTE_64BIT feature must be enabled to use Double formats") + } + } +} + +fn map_vertex_step_mode(mode: wgt::VertexStepMode) -> webgpu_sys::GpuVertexStepMode { + use webgpu_sys::GpuVertexStepMode as sm; + use wgt::VertexStepMode; + match mode { + VertexStepMode::Vertex => sm::Vertex, + VertexStepMode::Instance => sm::Instance, + } +} + +fn map_extent_3d(extent: wgt::Extent3d) -> webgpu_sys::GpuExtent3dDict { + let mapped = webgpu_sys::GpuExtent3dDict::new(extent.width); + mapped.set_height(extent.height); + mapped.set_depth_or_array_layers(extent.depth_or_array_layers); + mapped +} + +fn map_origin_2d(extent: wgt::Origin2d) -> webgpu_sys::GpuOrigin2dDict { + let mapped = webgpu_sys::GpuOrigin2dDict::new(); + mapped.set_x(extent.x); + mapped.set_y(extent.y); + mapped +} + +fn map_origin_3d(origin: wgt::Origin3d) -> webgpu_sys::GpuOrigin3dDict { + let mapped = webgpu_sys::GpuOrigin3dDict::new(); + mapped.set_x(origin.x); + mapped.set_y(origin.y); + mapped.set_z(origin.z); + mapped +} + +fn map_texture_dimension( + texture_dimension: wgt::TextureDimension, +) -> webgpu_sys::GpuTextureDimension { + match texture_dimension { + wgt::TextureDimension::D1 => webgpu_sys::GpuTextureDimension::N1d, + wgt::TextureDimension::D2 => webgpu_sys::GpuTextureDimension::N2d, + wgt::TextureDimension::D3 => webgpu_sys::GpuTextureDimension::N3d, + } +} + +fn map_texture_view_dimension( + texture_view_dimension: wgt::TextureViewDimension, +) -> webgpu_sys::GpuTextureViewDimension { + use webgpu_sys::GpuTextureViewDimension as tvd; + match texture_view_dimension { + wgt::TextureViewDimension::D1 => tvd::N1d, + wgt::TextureViewDimension::D2 => tvd::N2d, + wgt::TextureViewDimension::D2Array => tvd::N2dArray, + wgt::TextureViewDimension::Cube => tvd::Cube, + wgt::TextureViewDimension::CubeArray => tvd::CubeArray, + wgt::TextureViewDimension::D3 => tvd::N3d, + } +} + +fn map_buffer_copy_view( + view: crate::TexelCopyBufferInfo<'_>, +) -> webgpu_sys::GpuTexelCopyBufferInfo { + let buffer = view.buffer.inner.as_webgpu(); + let mapped = webgpu_sys::GpuTexelCopyBufferInfo::new(&buffer.inner); + if let Some(bytes_per_row) = view.layout.bytes_per_row { + mapped.set_bytes_per_row(bytes_per_row); + } + if let Some(rows_per_image) = view.layout.rows_per_image { + mapped.set_rows_per_image(rows_per_image); + } + mapped.set_offset(view.layout.offset as f64); + mapped +} + +fn map_texture_copy_view( + view: crate::TexelCopyTextureInfo<'_>, +) -> webgpu_sys::GpuTexelCopyTextureInfo { + let texture = view.texture.inner.as_webgpu(); + let mapped = webgpu_sys::GpuTexelCopyTextureInfo::new(&texture.inner); + mapped.set_mip_level(view.mip_level); + mapped.set_origin(&map_origin_3d(view.origin)); + mapped.set_aspect(map_texture_aspect(view.aspect)); + mapped +} + +fn map_tagged_texture_copy_view( + view: crate::CopyExternalImageDestInfo<&crate::api::Texture>, +) -> webgpu_sys::GpuCopyExternalImageDestInfo { + let texture = view.texture.inner.as_webgpu(); + let mapped = webgpu_sys::GpuCopyExternalImageDestInfo::new(&texture.inner); + mapped.set_mip_level(view.mip_level); + mapped.set_origin(&map_origin_3d(view.origin)); + mapped.set_aspect(map_texture_aspect(view.aspect)); + // mapped.set_color_space(map_color_space(view.color_space)); + mapped.set_premultiplied_alpha(view.premultiplied_alpha); + mapped +} + +fn map_external_texture_copy_view( + view: &crate::CopyExternalImageSourceInfo, +) -> webgpu_sys::GpuCopyExternalImageSourceInfo { + let mapped = webgpu_sys::GpuCopyExternalImageSourceInfo::new(&view.source); + mapped.set_origin(&map_origin_2d(view.origin)); + mapped.set_flip_y(view.flip_y); + mapped +} + +fn map_texture_aspect(aspect: wgt::TextureAspect) -> webgpu_sys::GpuTextureAspect { + match aspect { + wgt::TextureAspect::All => webgpu_sys::GpuTextureAspect::All, + wgt::TextureAspect::StencilOnly => webgpu_sys::GpuTextureAspect::StencilOnly, + wgt::TextureAspect::DepthOnly => webgpu_sys::GpuTextureAspect::DepthOnly, + wgt::TextureAspect::Plane0 | wgt::TextureAspect::Plane1 | wgt::TextureAspect::Plane2 => { + panic!("multi-plane textures are not supported") + } + } +} + +fn map_filter_mode(mode: wgt::FilterMode) -> webgpu_sys::GpuFilterMode { + match mode { + wgt::FilterMode::Nearest => webgpu_sys::GpuFilterMode::Nearest, + wgt::FilterMode::Linear => webgpu_sys::GpuFilterMode::Linear, + } +} + +fn map_mipmap_filter_mode(mode: wgt::MipmapFilterMode) -> webgpu_sys::GpuMipmapFilterMode { + match mode { + wgt::MipmapFilterMode::Nearest => webgpu_sys::GpuMipmapFilterMode::Nearest, + wgt::MipmapFilterMode::Linear => webgpu_sys::GpuMipmapFilterMode::Linear, + } +} + +fn map_address_mode(mode: wgt::AddressMode) -> webgpu_sys::GpuAddressMode { + match mode { + wgt::AddressMode::ClampToEdge => webgpu_sys::GpuAddressMode::ClampToEdge, + wgt::AddressMode::Repeat => webgpu_sys::GpuAddressMode::Repeat, + wgt::AddressMode::MirrorRepeat => webgpu_sys::GpuAddressMode::MirrorRepeat, + wgt::AddressMode::ClampToBorder => panic!("Clamp to border is not supported"), + } +} + +fn map_color(color: wgt::Color) -> webgpu_sys::GpuColorDict { + webgpu_sys::GpuColorDict::new(color.a, color.b, color.g, color.r) +} + +fn map_store_op(store: crate::StoreOp) -> webgpu_sys::GpuStoreOp { + match store { + crate::StoreOp::Store => webgpu_sys::GpuStoreOp::Store, + crate::StoreOp::Discard => webgpu_sys::GpuStoreOp::Discard, + } +} + +fn map_map_mode(mode: crate::MapMode) -> u32 { + match mode { + crate::MapMode::Read => webgpu_sys::gpu_map_mode::READ, + crate::MapMode::Write => webgpu_sys::gpu_map_mode::WRITE, + } +} + +const FEATURES_MAPPING: [(wgt::Features, webgpu_sys::GpuFeatureName); 16] = [ + ( + wgt::Features::DEPTH_CLIP_CONTROL, + webgpu_sys::GpuFeatureName::DepthClipControl, + ), + ( + wgt::Features::DEPTH32FLOAT_STENCIL8, + webgpu_sys::GpuFeatureName::Depth32floatStencil8, + ), + ( + wgt::Features::TEXTURE_COMPRESSION_BC, + webgpu_sys::GpuFeatureName::TextureCompressionBc, + ), + ( + wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D, + webgpu_sys::GpuFeatureName::TextureCompressionBcSliced3d, + ), + ( + wgt::Features::TEXTURE_COMPRESSION_ETC2, + webgpu_sys::GpuFeatureName::TextureCompressionEtc2, + ), + ( + wgt::Features::TEXTURE_COMPRESSION_ASTC, + webgpu_sys::GpuFeatureName::TextureCompressionAstc, + ), + ( + wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D, + webgpu_sys::GpuFeatureName::TextureCompressionAstcSliced3d, + ), + ( + wgt::Features::TIMESTAMP_QUERY, + webgpu_sys::GpuFeatureName::TimestampQuery, + ), + ( + wgt::Features::INDIRECT_FIRST_INSTANCE, + webgpu_sys::GpuFeatureName::IndirectFirstInstance, + ), + ( + wgt::Features::SHADER_F16, + webgpu_sys::GpuFeatureName::ShaderF16, + ), + ( + wgt::Features::RG11B10UFLOAT_RENDERABLE, + webgpu_sys::GpuFeatureName::Rg11b10ufloatRenderable, + ), + ( + wgt::Features::BGRA8UNORM_STORAGE, + webgpu_sys::GpuFeatureName::Bgra8unormStorage, + ), + ( + wgt::Features::FLOAT32_FILTERABLE, + webgpu_sys::GpuFeatureName::Float32Filterable, + ), + ( + wgt::Features::FLOAT32_BLENDABLE, + webgpu_sys::GpuFeatureName::Float32Blendable, + ), + ( + wgt::Features::DUAL_SOURCE_BLENDING, + webgpu_sys::GpuFeatureName::DualSourceBlending, + ), + ( + wgt::Features::CLIP_DISTANCES, + webgpu_sys::GpuFeatureName::ClipDistances, + ), +]; + +fn map_wgt_features(supported_features: webgpu_sys::GpuSupportedFeatures) -> wgt::Features { + let mut features = wgt::Features::empty(); + for (wgpu_feat, web_feat) in FEATURES_MAPPING { + match wasm_bindgen::JsValue::from(web_feat).as_string() { + Some(value) if supported_features.has(&value) => features |= wgpu_feat, + _ => {} + } + } + features +} + +fn map_wgt_limits(limits: webgpu_sys::GpuSupportedLimits) -> wgt::Limits { + wgt::Limits { + max_texture_dimension_1d: limits.max_texture_dimension_1d(), + max_texture_dimension_2d: limits.max_texture_dimension_2d(), + max_texture_dimension_3d: limits.max_texture_dimension_3d(), + max_texture_array_layers: limits.max_texture_array_layers(), + max_bind_groups: limits.max_bind_groups(), + max_bindings_per_bind_group: limits.max_bindings_per_bind_group(), + max_dynamic_uniform_buffers_per_pipeline_layout: limits + .max_dynamic_uniform_buffers_per_pipeline_layout(), + max_dynamic_storage_buffers_per_pipeline_layout: limits + .max_dynamic_storage_buffers_per_pipeline_layout(), + max_sampled_textures_per_shader_stage: limits.max_sampled_textures_per_shader_stage(), + max_samplers_per_shader_stage: limits.max_samplers_per_shader_stage(), + max_storage_buffers_per_shader_stage: limits.max_storage_buffers_per_shader_stage(), + max_storage_textures_per_shader_stage: limits.max_storage_textures_per_shader_stage(), + max_uniform_buffers_per_shader_stage: limits.max_uniform_buffers_per_shader_stage(), + max_binding_array_elements_per_shader_stage: 0, + max_binding_array_sampler_elements_per_shader_stage: 0, + max_binding_array_acceleration_structure_elements_per_shader_stage: 0, + max_uniform_buffer_binding_size: limits.max_uniform_buffer_binding_size() as u64, + max_storage_buffer_binding_size: limits.max_storage_buffer_binding_size() as u64, + max_vertex_buffers: limits.max_vertex_buffers(), + max_buffer_size: limits.max_buffer_size() as u64, + max_vertex_attributes: limits.max_vertex_attributes(), + max_vertex_buffer_array_stride: limits.max_vertex_buffer_array_stride(), + max_inter_stage_shader_variables: limits.max_inter_stage_shader_variables(), + min_uniform_buffer_offset_alignment: limits.min_uniform_buffer_offset_alignment(), + min_storage_buffer_offset_alignment: limits.min_storage_buffer_offset_alignment(), + max_color_attachments: limits.max_color_attachments(), + max_color_attachment_bytes_per_sample: limits.max_color_attachment_bytes_per_sample(), + max_compute_workgroup_storage_size: limits.max_compute_workgroup_storage_size(), + max_compute_invocations_per_workgroup: limits.max_compute_invocations_per_workgroup(), + max_compute_workgroup_size_x: limits.max_compute_workgroup_size_x(), + max_compute_workgroup_size_y: limits.max_compute_workgroup_size_y(), + max_compute_workgroup_size_z: limits.max_compute_workgroup_size_z(), + max_compute_workgroups_per_dimension: limits.max_compute_workgroups_per_dimension(), + max_immediate_size: wgt::Limits::default().max_immediate_size, + max_non_sampler_bindings: wgt::Limits::default().max_non_sampler_bindings, + + max_task_mesh_workgroup_total_count: wgt::Limits::default() + .max_task_mesh_workgroup_total_count, + max_task_mesh_workgroups_per_dimension: wgt::Limits::default() + .max_task_mesh_workgroups_per_dimension, + max_task_invocations_per_workgroup: wgt::Limits::default() + .max_task_invocations_per_workgroup, + max_task_invocations_per_dimension: wgt::Limits::default() + .max_task_invocations_per_dimension, + max_mesh_invocations_per_workgroup: wgt::Limits::default() + .max_mesh_invocations_per_workgroup, + max_mesh_invocations_per_dimension: wgt::Limits::default() + .max_mesh_invocations_per_dimension, + max_task_payload_size: wgt::Limits::default().max_task_payload_size, + max_mesh_output_vertices: wgt::Limits::default().max_mesh_output_vertices, + max_mesh_output_primitives: wgt::Limits::default().max_mesh_output_primitives, + max_mesh_output_layers: wgt::Limits::default().max_mesh_output_layers, + max_mesh_multiview_view_count: wgt::Limits::default().max_mesh_multiview_view_count, + + max_blas_primitive_count: wgt::Limits::default().max_blas_primitive_count, + max_blas_geometry_count: wgt::Limits::default().max_blas_geometry_count, + max_tlas_instance_count: wgt::Limits::default().max_tlas_instance_count, + max_acceleration_structures_per_shader_stage: wgt::Limits::default() + .max_acceleration_structures_per_shader_stage, + + max_multiview_view_count: wgt::Limits::default().max_multiview_view_count, + } +} + +fn map_adapter_info(adapter_info: &webgpu_sys::GpuAdapterInfo) -> wgt::AdapterInfo { + // TODO(https://github.com/gfx-rs/wgpu/issues/8819): populate more fields if/when possible + wgt::AdapterInfo { + name: adapter_info.description().to_string(), + vendor: 0, + device: 0, + device_type: wgt::DeviceType::Other, + device_pci_bus_id: String::new(), + driver: String::new(), + driver_info: String::new(), + backend: wgt::Backend::BrowserWebGpu, + subgroup_min_size: wgt::MINIMUM_SUBGROUP_MIN_SIZE, + subgroup_max_size: wgt::MAXIMUM_SUBGROUP_MAX_SIZE, + transient_saves_memory: false, + } +} + +fn map_js_sys_limits(limits: &wgt::Limits) -> js_sys::Object { + let object = js_sys::Object::new(); + + macro_rules! set_properties { + (($from:expr) => ($on:expr) : $(($js_ident:ident, $rs_ident:ident)),* $(,)?) => { + $( + ::js_sys::Reflect::set( + &$on, + &::wasm_bindgen::JsValue::from(stringify!($js_ident)), + // Numbers may be u64, however using `from` on a u64 yields + // errors on the wasm side, since it uses an unsupported api. + // Wasm sends us things that need to fit into u64s by sending + // us f64s instead. So we just send them f64s back. + &::wasm_bindgen::JsValue::from($from.$rs_ident as f64) + ) + .expect("Setting Object properties should never fail."); + )* + } + } + + // https://gpuweb.github.io/gpuweb/#gpusupportedlimits + set_properties![ + (limits) => (object): + (maxTextureDimension1D, max_texture_dimension_1d), + (maxTextureDimension2D, max_texture_dimension_2d), + (maxTextureDimension3D, max_texture_dimension_3d), + (maxTextureArrayLayers, max_texture_array_layers), + (maxBindGroups, max_bind_groups), + // TODO: (maxBindGroupsPlusVertexBuffers, max_bind_groups_plus_vertex_buffers), + (maxBindingsPerBindGroup, max_bindings_per_bind_group), + (maxDynamicUniformBuffersPerPipelineLayout, max_dynamic_uniform_buffers_per_pipeline_layout), + (maxDynamicStorageBuffersPerPipelineLayout, max_dynamic_storage_buffers_per_pipeline_layout), + (maxSampledTexturesPerShaderStage, max_sampled_textures_per_shader_stage), + (maxSamplersPerShaderStage, max_samplers_per_shader_stage), + (maxStorageBuffersPerShaderStage, max_storage_buffers_per_shader_stage), + (maxStorageTexturesPerShaderStage, max_storage_textures_per_shader_stage), + (maxUniformBuffersPerShaderStage, max_uniform_buffers_per_shader_stage), + (maxUniformBufferBindingSize, max_uniform_buffer_binding_size), + (maxStorageBufferBindingSize, max_storage_buffer_binding_size), + (minUniformBufferOffsetAlignment, min_uniform_buffer_offset_alignment), + (minStorageBufferOffsetAlignment, min_storage_buffer_offset_alignment), + (maxVertexBuffers, max_vertex_buffers), + (maxBufferSize, max_buffer_size), + (maxVertexAttributes, max_vertex_attributes), + (maxVertexBufferArrayStride, max_vertex_buffer_array_stride), + (maxInterStageShaderVariables, max_inter_stage_shader_variables), + (maxColorAttachments, max_color_attachments), + (maxColorAttachmentBytesPerSample, max_color_attachment_bytes_per_sample), + (maxComputeWorkgroupStorageSize, max_compute_workgroup_storage_size), + (maxComputeInvocationsPerWorkgroup, max_compute_invocations_per_workgroup), + (maxComputeWorkgroupSizeX, max_compute_workgroup_size_x), + (maxComputeWorkgroupSizeY, max_compute_workgroup_size_y), + (maxComputeWorkgroupSizeZ, max_compute_workgroup_size_z), + (maxComputeWorkgroupsPerDimension, max_compute_workgroups_per_dimension), + ]; + + object +} + +type JsFutureResult = Result; + +fn future_request_adapter( + result: JsFutureResult, + requested_backends: Backends, +) -> Result { + let web_adapter: Option = + result.and_then(wasm_bindgen::JsCast::dyn_into).ok(); + web_adapter + .map(|adapter| { + WebAdapter { + inner: adapter, + ident: crate::cmp::Identifier::create(), + } + .into() + }) + .ok_or_else(|| request_adapter_null_error(requested_backends)) +} + +// Translate WebGPU’s null return into our error. +fn request_adapter_null_error(requested_backends: Backends) -> wgt::RequestAdapterError { + wgt::RequestAdapterError::NotFound { + active_backends: Backends::BROWSER_WEBGPU, + requested_backends, + // TODO: supported_backends should also include wgpu-core-based backends, + // if they were compiled in. + supported_backends: Backends::BROWSER_WEBGPU, + no_fallback_backends: Backends::empty(), + no_adapter_backends: Backends::BROWSER_WEBGPU, + incompatible_surface_backends: Backends::empty(), + } +} + +fn future_request_device( + result: JsFutureResult, +) -> Result<(dispatch::DispatchDevice, dispatch::DispatchQueue), crate::RequestDeviceError> { + result + .map(|js_value| { + let device = webgpu_sys::GpuDevice::from(js_value); + let queue = device.queue(); + + ( + WebDevice { + inner: device, + ident: crate::cmp::Identifier::create(), + error_scope_count: Rc::new(Cell::new(0)), + } + .into(), + WebQueue { + inner: queue, + ident: crate::cmp::Identifier::create(), + } + .into(), + ) + }) + .map_err(|error_value| crate::RequestDeviceError { + // wasm-bindgen provides a reasonable error stringification via `Debug` impl + inner: crate::RequestDeviceErrorKind::WebGpu(format!("{error_value:?}")), + }) +} + +fn future_pop_error_scope(result: JsFutureResult) -> Option { + match result { + Ok(js_value) if js_value.is_object() => { + let js_error = wasm_bindgen::JsCast::dyn_into(js_value).unwrap(); + Some(crate::Error::from_js(js_error)) + } + _ => None, + } +} + +fn future_compilation_info( + result: JsFutureResult, + base_compilation_info: &WebShaderCompilationInfo, +) -> crate::CompilationInfo { + let base_messages = match base_compilation_info { + WebShaderCompilationInfo::Transformed { compilation_info } => { + compilation_info.messages.iter().cloned() + } + _ => [].iter().cloned(), + }; + + let messages = match result { + Ok(js_value) => { + let info = webgpu_sys::GpuCompilationInfo::from(js_value); + base_messages + .chain(info.messages().into_iter().map(|message| { + crate::CompilationMessage::from_js( + webgpu_sys::GpuCompilationMessage::from(message), + base_compilation_info, + ) + })) + .collect() + } + Err(_v) => base_messages + .chain(core::iter::once(crate::CompilationMessage { + message: "Getting compilation info failed".to_string(), + message_type: crate::CompilationMessageType::Error, + location: None, + })) + .collect(), + }; + + crate::CompilationInfo { messages } +} + +/// Calls `callback(success_value)` when the promise completes successfully, calls `callback(failure_value)` +/// when the promise completes unsuccessfully. +fn register_then_closures(promise: &Promise, callback: F, success_value: T, failure_value: T) +where + F: FnOnce(T) + 'static, + T: 'static, +{ + // Both the 'success' and 'rejected' closures need access to callback, but only one + // of them will ever run. We have them both hold a reference to a `Rc>>`, + // and then take ownership of callback when invoked. + // + // We also only need Rc's because these will only ever be called on our thread. + // + // We also store the actual closure types inside this Rc, as the closures need to be kept alive + // until they are actually called by the callback. It is valid to drop a closure inside of a callback. + // This allows us to keep the closures alive without leaking them. + let rc_callback: Rc>> = Rc::new(RefCell::new(None)); + + let rc_callback_clone1 = rc_callback.clone(); + let rc_callback_clone2 = rc_callback.clone(); + let closure_success = wasm_bindgen::closure::Closure::once(move |_| { + let (success_closure, rejection_closure, callback) = + rc_callback_clone1.borrow_mut().take().unwrap(); + callback(success_value); + // drop the closures, including ourselves, which will free any captured memory. + drop((success_closure, rejection_closure)); + }); + let closure_rejected = wasm_bindgen::closure::Closure::once(move |_| { + let (success_closure, rejection_closure, callback) = + rc_callback_clone2.borrow_mut().take().unwrap(); + callback(failure_value); + // drop the closures, including ourselves, which will free any captured memory. + drop((success_closure, rejection_closure)); + }); + + // Calling then before setting the value in the Rc seems like a race, but it isn't + // because the promise callback will run on this thread, so there is no race. + let _ = promise.then2(&closure_success, &closure_rejected); + + *rc_callback.borrow_mut() = Some((closure_success, closure_rejected, callback)); +} + +impl ContextWebGpu { + /// Common portion of the internal branches of the public `instance_create_surface` function. + /// + /// Note: Analogous code also exists in the WebGL2 backend at + /// `wgpu_hal::gles::web::Instance`. + fn create_surface_from_context( + &self, + canvas: Canvas, + context_result: Result, wasm_bindgen::JsValue>, + ) -> Result { + let context: js_sys::Object = match context_result { + Ok(Some(context)) => context, + Ok(None) => { + // + // A getContext() call “returns null if contextId is not supported, or if the + // canvas has already been initialized with another context type”. Additionally, + // “not supported” could include “insufficient GPU resources” or “the GPU process + // previously crashed”. So, we must return it as an `Err` since it could occur + // for circumstances outside the application author's control. + return Err(crate::CreateSurfaceError { + inner: crate::CreateSurfaceErrorKind::Web( + String::from( + "canvas.getContext() returned null; webgpu not available or canvas already in use" + ) + ) + }); + } + Err(js_error) => { + // + // A thrown exception indicates misuse of the canvas state. + return Err(crate::CreateSurfaceError { + inner: crate::CreateSurfaceErrorKind::Web(format!( + "canvas.getContext() threw exception {js_error:?}", + )), + }); + } + }; + + // Not returning this error because it is a type error that shouldn't happen unless + // the browser, JS builtin objects, or wasm bindings are misbehaving somehow. + let context: webgpu_sys::GpuCanvasContext = context + .dyn_into() + .expect("canvas context is not a GPUCanvasContext"); + + Ok(WebSurface { + gpu: self.gpu.clone(), + context, + canvas, + ident: crate::cmp::Identifier::create(), + } + .into()) + } +} + +// Represents the global object in the JavaScript context. +// It can be cast to from `webgpu_sys::global` and exposes two getters `window` and `worker` of which only one is defined depending on the caller's context. +// When called from the UI thread only `window` is defined whereas `worker` is only defined within a web worker context. +// See: https://github.com/rustwasm/gloo/blob/2c9e776701ecb90c53e62dec1abd19c2b70e47c7/crates/timers/src/callback.rs#L8-L40 +#[wasm_bindgen] +extern "C" { + type Global; + + #[wasm_bindgen(method, getter, js_name = Window)] + fn window(this: &Global) -> JsValue; + + #[wasm_bindgen(method, getter, js_name = WorkerGlobalScope)] + fn worker(this: &Global) -> JsValue; +} + +#[derive(Debug, Clone)] +pub enum Canvas { + Canvas(web_sys::HtmlCanvasElement), + Offscreen(web_sys::OffscreenCanvas), +} + +#[derive(Debug, Clone, Copy)] +pub struct BrowserGpuPropertyInaccessible; + +/// Returns the browser's gpu object or `Err(BrowserGpuPropertyInaccessible)` if +/// the current context is neither the main thread nor a dedicated worker. +/// +/// If WebGPU is not supported, the Gpu property may (!) be `undefined`, +/// and so this function will return `Ok(None)`. +/// Note that this check is insufficient to determine whether WebGPU is +/// supported, as the browser may define the Gpu property, but be unable to +/// create any WebGPU adapters. +/// To detect whether WebGPU is supported, use the [`crate::utils::is_browser_webgpu_supported`] function. +/// +/// See: +/// * +/// * +pub fn get_browser_gpu_property( +) -> Result>, BrowserGpuPropertyInaccessible> { + let global: Global = js_sys::global().unchecked_into(); + + let maybe_undefined_gpu: webgpu_sys::Gpu = if !global.window().is_undefined() { + let navigator = global.unchecked_into::().navigator(); + ext_bindings::NavigatorGpu::gpu(&navigator) + } else if !global.worker().is_undefined() { + let navigator = global + .unchecked_into::() + .navigator(); + ext_bindings::NavigatorGpu::gpu(&navigator) + } else { + return Err(BrowserGpuPropertyInaccessible); + }; + Ok(DefinedNonNullJsValue::new(maybe_undefined_gpu)) +} + +#[derive(Debug, Clone)] +pub struct WebAdapter { + pub(crate) inner: webgpu_sys::GpuAdapter, + /// Unique identifier for this Adapter. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebDevice { + pub(crate) inner: webgpu_sys::GpuDevice, + /// Unique identifier for this Device. + ident: crate::cmp::Identifier, + /// Current number of error scopes that have been pushed on the device. + error_scope_count: Rc>, +} + +#[derive(Debug, Clone)] +pub struct WebQueue { + pub(crate) inner: webgpu_sys::GpuQueue, + /// Unique identifier for this Queue. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebBindGroupLayout { + pub(crate) inner: webgpu_sys::GpuBindGroupLayout, + /// Unique identifier for this BindGroupLayout. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebBindGroup { + pub(crate) inner: webgpu_sys::GpuBindGroup, + /// Unique identifier for this BindGroup. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebTextureView { + pub(crate) inner: webgpu_sys::GpuTextureView, + /// Unique identifier for this TextureView. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebSampler { + pub(crate) inner: webgpu_sys::GpuSampler, + /// Unique identifier for this Sampler. + ident: crate::cmp::Identifier, +} + +/// Remembers which portion of a buffer has been mapped, along with a reference +/// to the mapped portion. +#[derive(Debug, Clone)] +struct WebBufferMapState { + /// The mapped memory of the buffer. + pub mapped_buffer: Option, + /// The total range which has been mapped in the buffer overall. + pub range: Range, +} + +/// Stores the state of a GPU buffer and a reference to its mapped `ArrayBuffer` (if any). +/// The WebGPU specification forbids calling `getMappedRange` on a `webgpu_sys::GpuBuffer` more than +/// once, so this struct stores the initial mapped range and re-uses it, allowing for multiple `get_mapped_range` +/// calls on the Rust-side. +#[derive(Debug, Clone)] +pub struct WebBuffer { + /// The associated GPU buffer. + inner: webgpu_sys::GpuBuffer, + /// The mapped array buffer and mapped range. + mapping: Rc>, + /// Unique identifier for this Buffer. + ident: crate::cmp::Identifier, +} + +impl WebBuffer { + /// Creates a new web buffer for the given Javascript object and description. + fn new(inner: webgpu_sys::GpuBuffer, desc: &crate::BufferDescriptor<'_>) -> Self { + Self { + inner, + mapping: Rc::new(RefCell::new(WebBufferMapState { + mapped_buffer: None, + range: 0..desc.size, + })), + ident: crate::cmp::Identifier::create(), + } + } + + /// Obtains a reference to the re-usable buffer mapping as a Javascript array view. + fn get_mapped_range(&self, sub_range: Range) -> js_sys::Uint8Array { + let mut mapping = self.mapping.borrow_mut(); + let range = mapping.range.clone(); + let array_buffer = mapping.mapped_buffer.get_or_insert_with(|| { + self.inner + .get_mapped_range_with_f64_and_f64( + range.start as f64, + (range.end - range.start) as f64, + ) + .unwrap() + }); + js_sys::Uint8Array::new_with_byte_offset_and_length( + array_buffer, + (sub_range.start - range.start) as u32, + (sub_range.end - sub_range.start) as u32, + ) + } + + /// Sets the range of the buffer which is presently mapped. + fn set_mapped_range(&self, range: Range) { + self.mapping.borrow_mut().range = range; + } +} + +#[derive(Debug, Clone)] +pub struct WebTexture { + pub(crate) inner: webgpu_sys::GpuTexture, + /// Unique identifier for this Texture. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebExternalTexture { + /// Unique identifier for this ExternalTexture. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebBlas { + /// Unique identifier for this Blas. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebTlas { + /// Unique identifier for this Blas. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebQuerySet { + pub(crate) inner: webgpu_sys::GpuQuerySet, + /// Unique identifier for this QuerySet. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebPipelineLayout { + pub(crate) inner: webgpu_sys::GpuPipelineLayout, + /// Unique identifier for this PipelineLayout. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebRenderPipeline { + pub(crate) inner: webgpu_sys::GpuRenderPipeline, + /// Unique identifier for this RenderPipeline. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebComputePipeline { + pub(crate) inner: webgpu_sys::GpuComputePipeline, + /// Unique identifier for this ComputePipeline. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebPipelineCache { + /// Unique identifier for this PipelineCache. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebCommandEncoder { + pub(crate) inner: webgpu_sys::GpuCommandEncoder, + /// Unique identifier for this CommandEncoder. + ident: crate::cmp::Identifier, +} + +#[derive(Debug)] +pub struct WebComputePassEncoder { + pub(crate) inner: webgpu_sys::GpuComputePassEncoder, + /// Unique identifier for this ComputePassEncoder. + ident: crate::cmp::Identifier, +} + +#[derive(Debug)] +pub struct WebRenderPassEncoder { + pub(crate) inner: webgpu_sys::GpuRenderPassEncoder, + /// Unique identifier for this RenderPassEncoder. + ident: crate::cmp::Identifier, +} + +#[derive(Debug)] +pub struct WebCommandBuffer { + pub(crate) inner: webgpu_sys::GpuCommandBuffer, + /// Unique identifier for this CommandBuffer. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebRenderBundleEncoder { + pub(crate) inner: webgpu_sys::GpuRenderBundleEncoder, + /// Unique identifier for this RenderBundleEncoder. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebRenderBundle { + pub(crate) inner: webgpu_sys::GpuRenderBundle, + /// Unique identifier for this RenderBundle. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebSurface { + gpu: Option>, + canvas: Canvas, + context: webgpu_sys::GpuCanvasContext, + /// Unique identifier for this Surface. + ident: crate::cmp::Identifier, +} + +#[derive(Debug, Clone)] +pub struct WebSurfaceOutputDetail { + /// Unique identifier for this SurfaceOutputDetail. + ident: crate::cmp::Identifier, +} + +#[derive(Debug)] +pub struct WebQueueWriteBuffer { + inner: Box<[u8]>, + /// Unique identifier for this QueueWriteBuffer. + ident: crate::cmp::Identifier, +} + +#[derive(Debug)] +pub struct WebBufferMappedRange { + actual_mapping: js_sys::Uint8Array, + /// Copy of actual_mapping that lives in the Rust/Wasm heap instead of JS. This + /// is done only when accessed for the first time to avoid unnecessary allocations. + temporary_mapping: OnceCell>, + /// Whether `temporary_mapping` has possibly been written to and needs to be written back to JS. + temporary_mapping_modified: bool, + /// Unique identifier for this BufferMappedRange. + ident: crate::cmp::Identifier, +} + +impl_send_sync!(ContextWebGpu); +impl_send_sync!(WebAdapter); +impl_send_sync!(WebDevice); +impl_send_sync!(WebQueue); +impl_send_sync!(WebShaderModule); +impl_send_sync!(WebBindGroupLayout); +impl_send_sync!(WebBindGroup); +impl_send_sync!(WebTextureView); +impl_send_sync!(WebSampler); +impl_send_sync!(WebBuffer); +impl_send_sync!(WebTexture); +impl_send_sync!(WebExternalTexture); +impl_send_sync!(WebBlas); +impl_send_sync!(WebTlas); +impl_send_sync!(WebQuerySet); +impl_send_sync!(WebPipelineLayout); +impl_send_sync!(WebRenderPipeline); +impl_send_sync!(WebComputePipeline); +impl_send_sync!(WebPipelineCache); +impl_send_sync!(WebCommandEncoder); +impl_send_sync!(WebComputePassEncoder); +impl_send_sync!(WebRenderPassEncoder); +impl_send_sync!(WebCommandBuffer); +impl_send_sync!(WebRenderBundleEncoder); +impl_send_sync!(WebRenderBundle); +impl_send_sync!(WebSurface); +impl_send_sync!(WebSurfaceOutputDetail); +impl_send_sync!(WebQueueWriteBuffer); +impl_send_sync!(WebBufferMappedRange); + +crate::cmp::impl_eq_ord_hash_proxy!(ContextWebGpu => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebAdapter => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebDevice => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebQueue => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebShaderModule => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebBindGroupLayout => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebBindGroup => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebTextureView => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebSampler => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebBuffer => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebTexture => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebExternalTexture => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebBlas => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebTlas => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebQuerySet => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebPipelineLayout => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebRenderPipeline => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebComputePipeline => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebPipelineCache => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebCommandEncoder => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebComputePassEncoder => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebRenderPassEncoder => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebCommandBuffer => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebRenderBundleEncoder => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebRenderBundle => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebSurface => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebSurfaceOutputDetail => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebQueueWriteBuffer => .ident); +crate::cmp::impl_eq_ord_hash_proxy!(WebBufferMappedRange => .ident); + +impl dispatch::InstanceInterface for ContextWebGpu { + fn new(desc: crate::InstanceDescriptor) -> Self + where + Self: Sized, + { + let Ok(gpu) = get_browser_gpu_property() else { + panic!( + "Accessing the GPU is only supported on the main thread or from a dedicated worker" + ); + }; + + ContextWebGpu { + gpu, + requested_backends: desc.backends, + ident: crate::cmp::Identifier::create(), + } + } + + unsafe fn create_surface( + &self, + target: crate::SurfaceTargetUnsafe, + ) -> Result { + match target { + SurfaceTargetUnsafe::RawHandle { + raw_display_handle: _, + raw_window_handle, + } => { + let canvas_element: web_sys::HtmlCanvasElement = match raw_window_handle { + raw_window_handle::RawWindowHandle::Web(handle) => { + let canvas_node: wasm_bindgen::JsValue = web_sys::window() + .and_then(|win| win.document()) + .and_then(|doc| { + doc.query_selector_all(&format!( + "[data-raw-handle=\"{}\"]", + handle.id + )) + .ok() + }) + .and_then(|nodes| nodes.get(0)) + .expect("expected to find single canvas") + .into(); + canvas_node.into() + } + raw_window_handle::RawWindowHandle::WebCanvas(handle) => { + let value: &JsValue = unsafe { handle.obj.cast().as_ref() }; + value.clone().unchecked_into() + } + raw_window_handle::RawWindowHandle::WebOffscreenCanvas(handle) => { + let value: &JsValue = unsafe { handle.obj.cast().as_ref() }; + let canvas: web_sys::OffscreenCanvas = value.clone().unchecked_into(); + let context_result = canvas.get_context("webgpu"); + + return self.create_surface_from_context( + Canvas::Offscreen(canvas), + context_result, + ); + } + _ => panic!("expected valid handle for canvas"), + }; + + let context_result = canvas_element.get_context("webgpu"); + self.create_surface_from_context(Canvas::Canvas(canvas_element), context_result) + } + } + } + + fn request_adapter( + &self, + options: &crate::RequestAdapterOptions<'_, '_>, + ) -> Pin> { + let requested_backends = self.requested_backends; + + //TODO: support this check, return `None` if the flag is not set. + // It's not trivial, since we need the Future logic to have this check, + // and currently the Future here has no room for extra parameter `backends`. + if !(requested_backends.contains(wgt::Backends::BROWSER_WEBGPU)) { + return Box::pin(core::future::ready(Err( + wgt::RequestAdapterError::NotFound { + active_backends: Backends::BROWSER_WEBGPU, + requested_backends, + // TODO: supported_backends should also include wgpu-core-based backends, + // if they were compiled in. + supported_backends: Backends::BROWSER_WEBGPU, + no_fallback_backends: Backends::default(), + no_adapter_backends: Backends::default(), + incompatible_surface_backends: Backends::default(), + }, + ))); + } + let mapped_options = webgpu_sys::GpuRequestAdapterOptions::new(); + let mapped_power_preference = match options.power_preference { + wgt::PowerPreference::None => None, + wgt::PowerPreference::LowPower => Some(webgpu_sys::GpuPowerPreference::LowPower), + wgt::PowerPreference::HighPerformance => { + Some(webgpu_sys::GpuPowerPreference::HighPerformance) + } + }; + if let Some(mapped_pref) = mapped_power_preference { + mapped_options.set_power_preference(mapped_pref); + } + + if let Some(gpu) = &self.gpu { + let adapter_promise = gpu.request_adapter_with_options(&mapped_options); + Box::pin(MakeSendFuture::new( + wasm_bindgen_futures::JsFuture::from(adapter_promise), + move |result| future_request_adapter(result, requested_backends), + )) + } else { + // Gpu is undefined; WebGPU is not supported in this browser. + // Treat this exactly like requestAdapter() returned null. + Box::pin(core::future::ready(Err(request_adapter_null_error( + requested_backends, + )))) + } + } + fn enumerate_adapters( + &self, + _backends: crate::Backends, + ) -> Pin> { + let future = self.request_adapter(&crate::RequestAdapterOptions::default()); + let enumerate_future = async move { + let adapter = future.await; + match adapter { + Ok(a) => vec![a], + Err(_) => vec![], + } + }; + Box::pin(enumerate_future) + } + + fn poll_all_devices(&self, _force_wait: bool) -> bool { + // Devices are automatically polled. + true + } + + #[cfg(feature = "wgsl")] + fn wgsl_language_features(&self) -> crate::WgslLanguageFeatures { + let mut wgsl_language_features = crate::WgslLanguageFeatures::empty(); + if let Some(gpu) = &self.gpu { + gpu.wgsl_language_features() + .keys() + .into_iter() + .map(|wlf| wlf.expect("`WgslLanguageFeatures` elements should be valid")) + .map(|wlf| { + wlf.as_string() + .expect("`WgslLanguageFeatures` should be string set") + }) + .filter_map(|wlf| match wlf.as_str() { + "readonly_and_readwrite_storage_textures" => { + Some(crate::WgslLanguageFeatures::ReadOnlyAndReadWriteStorageTextures) + } + "packed_4x8_integer_dot_product" => { + Some(crate::WgslLanguageFeatures::Packed4x8IntegerDotProduct) + } + "unrestricted_pointer_parameters" => { + Some(crate::WgslLanguageFeatures::UnrestrictedPointerParameters) + } + "pointer_composite_access" => { + Some(crate::WgslLanguageFeatures::PointerCompositeAccess) + } + _ => None, + }) + .for_each(|wlf| { + wgsl_language_features |= wlf; + }) + } + wgsl_language_features + } +} + +impl Drop for ContextWebGpu { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::AdapterInterface for WebAdapter { + fn request_device( + &self, + desc: &crate::DeviceDescriptor<'_>, + ) -> Pin> { + if !matches!(desc.trace, wgt::Trace::Off) { + log::warn!("The `trace` parameter is not supported on the WebGPU backend."); + } + + let mapped_desc = webgpu_sys::GpuDeviceDescriptor::new(); + + let required_limits = map_js_sys_limits(&desc.required_limits); + mapped_desc.set_required_limits(&required_limits); + + let required_features = FEATURES_MAPPING + .iter() + .copied() + .flat_map(|(flag, value)| { + if desc.required_features.contains(flag) { + Some(JsValue::from(value)) + } else { + None + } + }) + .collect::(); + mapped_desc.set_required_features(&required_features); + + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + + let device_promise = self.inner.request_device_with_descriptor(&mapped_desc); + + Box::pin(MakeSendFuture::new( + wasm_bindgen_futures::JsFuture::from(device_promise), + future_request_device, + )) + } + + fn is_surface_supported(&self, _surface: &dispatch::DispatchSurface) -> bool { + // All surfaces are inherently supported. + true + } + + fn features(&self) -> crate::Features { + map_wgt_features(self.inner.features()) + } + + fn limits(&self) -> crate::Limits { + map_wgt_limits(self.inner.limits()) + } + + fn downlevel_capabilities(&self) -> crate::DownlevelCapabilities { + // WebGPU is assumed to be fully compliant + crate::DownlevelCapabilities::default() + } + + fn get_info(&self) -> crate::AdapterInfo { + map_adapter_info(&self.inner.info()) + } + + fn get_texture_format_features( + &self, + format: crate::TextureFormat, + ) -> crate::TextureFormatFeatures { + format.guaranteed_format_features(dispatch::AdapterInterface::features(self)) + } + + fn get_presentation_timestamp(&self) -> crate::PresentationTimestamp { + crate::PresentationTimestamp::INVALID_TIMESTAMP + } + + fn cooperative_matrix_properties(&self) -> Vec { + Vec::new() + } +} +impl Drop for WebAdapter { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::DeviceInterface for WebDevice { + fn features(&self) -> crate::Features { + map_wgt_features(self.inner.features()) + } + + fn limits(&self) -> crate::Limits { + map_wgt_limits(self.inner.limits()) + } + + fn adapter_info(&self) -> crate::AdapterInfo { + map_adapter_info(&self.inner.adapter_info()) + } + + fn create_shader_module( + &self, + desc: crate::ShaderModuleDescriptor<'_>, + _shader_runtime_checks: crate::ShaderRuntimeChecks, + ) -> dispatch::DispatchShaderModule { + let shader_module_result = match desc.source { + #[cfg(feature = "spirv")] + crate::ShaderSource::SpirV(ref spv) => { + use naga::front; + + let options = naga::front::spv::Options { + adjust_coordinate_space: false, + strict_capabilities: true, + block_ctx_dump_prefix: None, + }; + let spv_parser = front::spv::Frontend::new(spv.iter().cloned(), &options); + spv_parser + .parse() + .map_err(|inner| { + crate::CompilationInfo::from(naga::error::ShaderError { + source: String::new(), + label: desc.label.map(|s| s.to_string()), + inner: Box::new(inner), + }) + }) + .and_then(|spv_module| { + validate_transformed_shader_module(&spv_module, "", &desc).map(|v| { + ( + v, + WebShaderCompilationInfo::Transformed { + compilation_info: crate::CompilationInfo { messages: vec![] }, + }, + ) + }) + }) + } + #[cfg(feature = "glsl")] + crate::ShaderSource::Glsl { + ref shader, + stage, + defines, + } => { + use naga::front; + + // Parse the given shader code and store its representation. + let options = front::glsl::Options { + stage, + defines: defines + .iter() + .map(|&(key, value)| (String::from(key), String::from(value))) + .collect(), + }; + let mut parser = front::glsl::Frontend::default(); + parser + .parse(&options, shader) + .map_err(|inner| { + crate::CompilationInfo::from(naga::error::ShaderError { + source: shader.to_string(), + label: desc.label.map(|s| s.to_string()), + inner: Box::new(inner), + }) + }) + .and_then(|glsl_module| { + validate_transformed_shader_module(&glsl_module, shader, &desc).map(|v| { + ( + v, + WebShaderCompilationInfo::Transformed { + compilation_info: crate::CompilationInfo { messages: vec![] }, + }, + ) + }) + }) + } + #[cfg(feature = "wgsl")] + crate::ShaderSource::Wgsl(ref code) => { + let shader_module = webgpu_sys::GpuShaderModuleDescriptor::new(code); + Ok(( + shader_module, + WebShaderCompilationInfo::Wgsl { + source: code.to_string(), + }, + )) + } + #[cfg(feature = "naga-ir")] + crate::ShaderSource::Naga(ref module) => { + validate_transformed_shader_module(module, "", &desc).map(|v| { + ( + v, + WebShaderCompilationInfo::Transformed { + compilation_info: crate::CompilationInfo { messages: vec![] }, + }, + ) + }) + } + crate::ShaderSource::Dummy(_) => { + panic!("found `ShaderSource::Dummy`") + } + }; + + #[cfg(naga)] + fn validate_transformed_shader_module( + module: &naga::Module, + source: &str, + desc: &crate::ShaderModuleDescriptor<'_>, + ) -> Result { + use naga::{back, valid}; + let mut validator = + valid::Validator::new(valid::ValidationFlags::all(), valid::Capabilities::all()); + let module_info = validator.validate(module).map_err(|err| { + crate::CompilationInfo::from(naga::error::ShaderError { + source: source.to_string(), + label: desc.label.map(|s| s.to_string()), + inner: Box::new(err), + }) + })?; + + let writer_flags = naga::back::wgsl::WriterFlags::empty(); + let wgsl_text = back::wgsl::write_string(module, &module_info, writer_flags).unwrap(); + Ok(webgpu_sys::GpuShaderModuleDescriptor::new( + wgsl_text.as_str(), + )) + } + let (descriptor, compilation_info) = match shader_module_result { + Ok(v) => v, + Err(compilation_info) => ( + webgpu_sys::GpuShaderModuleDescriptor::new(""), + WebShaderCompilationInfo::Transformed { compilation_info }, + ), + }; + if let Some(label) = desc.label { + descriptor.set_label(label); + } + WebShaderModule { + module: self.inner.create_shader_module(&descriptor), + compilation_info, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + unsafe fn create_shader_module_passthrough( + &self, + desc: &crate::ShaderModuleDescriptorPassthrough<'_>, + ) -> dispatch::DispatchShaderModule { + let shader_module_result = if let Some(ref code) = desc.wgsl { + let shader_module = webgpu_sys::GpuShaderModuleDescriptor::new(code); + Ok(( + shader_module, + WebShaderCompilationInfo::Wgsl { + source: code.to_string(), + }, + )) + } else { + Err(crate::CompilationInfo { + messages: vec![crate::CompilationMessage { + message: + "Passthrough shader not compiled for WGSL on WebGPU backend (wgpu error)" + .to_string(), + location: None, + message_type: crate::CompilationMessageType::Error, + }], + }) + }; + let (descriptor, compilation_info) = match shader_module_result { + Ok(v) => v, + Err(compilation_info) => ( + webgpu_sys::GpuShaderModuleDescriptor::new(""), + WebShaderCompilationInfo::Transformed { compilation_info }, + ), + }; + if let Some(label) = desc.label { + descriptor.set_label(label); + } + WebShaderModule { + module: self.inner.create_shader_module(&descriptor), + compilation_info, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_bind_group_layout( + &self, + desc: &crate::BindGroupLayoutDescriptor<'_>, + ) -> dispatch::DispatchBindGroupLayout { + let mapped_bindings = desc + .entries + .iter() + .map(|bind| { + let mapped_entry = + webgpu_sys::GpuBindGroupLayoutEntry::new(bind.binding, bind.visibility.bits()); + + match bind.ty { + wgt::BindingType::Buffer { + ty, + has_dynamic_offset, + min_binding_size, + } => { + let buffer = webgpu_sys::GpuBufferBindingLayout::new(); + buffer.set_has_dynamic_offset(has_dynamic_offset); + if let Some(size) = min_binding_size { + buffer.set_min_binding_size(size.get() as f64); + } + buffer.set_type(match ty { + wgt::BufferBindingType::Uniform => { + webgpu_sys::GpuBufferBindingType::Uniform + } + wgt::BufferBindingType::Storage { read_only: false } => { + webgpu_sys::GpuBufferBindingType::Storage + } + wgt::BufferBindingType::Storage { read_only: true } => { + webgpu_sys::GpuBufferBindingType::ReadOnlyStorage + } + }); + mapped_entry.set_buffer(&buffer); + } + wgt::BindingType::Sampler(ty) => { + let sampler = webgpu_sys::GpuSamplerBindingLayout::new(); + sampler.set_type(match ty { + wgt::SamplerBindingType::NonFiltering => { + webgpu_sys::GpuSamplerBindingType::NonFiltering + } + wgt::SamplerBindingType::Filtering => { + webgpu_sys::GpuSamplerBindingType::Filtering + } + wgt::SamplerBindingType::Comparison => { + webgpu_sys::GpuSamplerBindingType::Comparison + } + }); + mapped_entry.set_sampler(&sampler); + } + wgt::BindingType::Texture { + multisampled, + sample_type, + view_dimension, + } => { + let texture = webgpu_sys::GpuTextureBindingLayout::new(); + texture.set_multisampled(multisampled); + texture.set_sample_type(map_texture_component_type(sample_type)); + texture.set_view_dimension(map_texture_view_dimension(view_dimension)); + mapped_entry.set_texture(&texture); + } + wgt::BindingType::StorageTexture { + access, + format, + view_dimension, + } => { + let mapped_access = match access { + wgt::StorageTextureAccess::WriteOnly => { + webgpu_sys::GpuStorageTextureAccess::WriteOnly + } + wgt::StorageTextureAccess::ReadOnly => { + webgpu_sys::GpuStorageTextureAccess::ReadOnly + } + wgt::StorageTextureAccess::ReadWrite => { + webgpu_sys::GpuStorageTextureAccess::ReadWrite + } + wgt::StorageTextureAccess::Atomic => { + // Validated out by `BindGroupLayoutEntryError::StorageTextureAtomic` + unreachable!() + } + }; + let storage_texture = webgpu_sys::GpuStorageTextureBindingLayout::new( + map_texture_format(format), + ); + storage_texture.set_access(mapped_access); + storage_texture + .set_view_dimension(map_texture_view_dimension(view_dimension)); + mapped_entry.set_storage_texture(&storage_texture); + } + wgt::BindingType::AccelerationStructure { .. } => todo!(), + wgt::BindingType::ExternalTexture => { + mapped_entry.set_external_texture( + &webgpu_sys::GpuExternalTextureBindingLayout::new(), + ); + } + } + + mapped_entry + }) + .collect::(); + + let mapped_desc = webgpu_sys::GpuBindGroupLayoutDescriptor::new(&mapped_bindings); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + let bind_group_layout = self.inner.create_bind_group_layout(&mapped_desc).unwrap(); + + WebBindGroupLayout { + inner: bind_group_layout, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_bind_group( + &self, + desc: &crate::BindGroupDescriptor<'_>, + ) -> dispatch::DispatchBindGroup { + let mapped_entries = desc + .entries + .iter() + .map(|binding| { + let mapped_resource = match binding.resource { + crate::BindingResource::Buffer(crate::BufferBinding { + buffer, + offset, + size, + }) => { + let buffer = buffer.inner.as_webgpu(); + let mapped_buffer_binding = + webgpu_sys::GpuBufferBinding::new(&buffer.inner); + mapped_buffer_binding.set_offset(offset as f64); + if let Some(s) = size { + mapped_buffer_binding.set_size(s.get() as f64); + } + JsValue::from(mapped_buffer_binding) + } + crate::BindingResource::BufferArray(..) => { + panic!("Web backend does not support arrays of buffers") + } + crate::BindingResource::Sampler(sampler) => { + let sampler = &sampler.inner.as_webgpu().inner; + JsValue::from(sampler) + } + crate::BindingResource::SamplerArray(..) => { + panic!("Web backend does not support arrays of samplers") + } + crate::BindingResource::TextureView(texture_view) => { + let texture_view = &texture_view.inner.as_webgpu().inner; + JsValue::from(texture_view) + } + crate::BindingResource::TextureViewArray(..) => { + panic!("Web backend does not support BINDING_INDEXING extension") + } + crate::BindingResource::AccelerationStructure(_) => { + unimplemented!("Raytracing not implemented for web") + } + crate::BindingResource::AccelerationStructureArray(_) => { + unimplemented!("Raytracing not implemented for web") + } + crate::BindingResource::ExternalTexture(_) => { + unimplemented!("ExternalTexture not implemented for web") + } + }; + + webgpu_sys::GpuBindGroupEntry::new(binding.binding, &mapped_resource) + }) + .collect::(); + + let bgl = &desc.layout.inner.as_webgpu().inner; + let mapped_desc = webgpu_sys::GpuBindGroupDescriptor::new(&mapped_entries, bgl); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + let bind_group = self.inner.create_bind_group(&mapped_desc); + + WebBindGroup { + inner: bind_group, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_pipeline_layout( + &self, + desc: &crate::PipelineLayoutDescriptor<'_>, + ) -> dispatch::DispatchPipelineLayout { + let null = wasm_bindgen::JsValue::NULL; + let temp_layouts = desc + .bind_group_layouts + .iter() + .map(|bgl| match bgl { + Some(bgl) => bgl.inner.as_webgpu().inner.as_ref(), + None => &null, + }) + .collect::(); + let mapped_desc = webgpu_sys::GpuPipelineLayoutDescriptor::new(&temp_layouts); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + + let pipeline_layout = self.inner.create_pipeline_layout(&mapped_desc); + + WebPipelineLayout { + inner: pipeline_layout, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_render_pipeline( + &self, + desc: &crate::RenderPipelineDescriptor<'_>, + ) -> dispatch::DispatchRenderPipeline { + let module = desc.vertex.module.inner.as_webgpu(); + let mapped_vertex_state = webgpu_sys::GpuVertexState::new(&module.module); + insert_constants_map( + &mapped_vertex_state, + desc.vertex.compilation_options.constants, + ); + if let Some(ep) = desc.vertex.entry_point { + mapped_vertex_state.set_entry_point(ep); + } + + let buffers = desc + .vertex + .buffers + .iter() + .map(|vbuf| { + let mapped_attributes = vbuf + .attributes + .iter() + .map(|attr| { + webgpu_sys::GpuVertexAttribute::new( + map_vertex_format(attr.format), + attr.offset as f64, + attr.shader_location, + ) + }) + .collect::(); + + let mapped_vbuf = webgpu_sys::GpuVertexBufferLayout::new( + vbuf.array_stride as f64, + &mapped_attributes, + ); + mapped_vbuf.set_step_mode(map_vertex_step_mode(vbuf.step_mode)); + mapped_vbuf + }) + .collect::(); + + mapped_vertex_state.set_buffers(&buffers); + + let auto_layout = wasm_bindgen::JsValue::from(webgpu_sys::GpuAutoLayoutMode::Auto); + let mapped_desc = webgpu_sys::GpuRenderPipelineDescriptor::new( + &match desc.layout { + Some(layout) => { + let layout = &layout.inner.as_webgpu().inner; + JsValue::from(layout) + } + None => auto_layout, + }, + &mapped_vertex_state, + ); + + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + + if let Some(ref depth_stencil) = desc.depth_stencil { + mapped_desc.set_depth_stencil(&map_depth_stencil_state(depth_stencil)); + } + + if let Some(ref frag) = desc.fragment { + let targets = frag + .targets + .iter() + .map(|target| match target { + Some(target) => { + let mapped_format = map_texture_format(target.format); + let mapped_color_state = + webgpu_sys::GpuColorTargetState::new(mapped_format); + if let Some(ref bs) = target.blend { + let alpha = map_blend_component(&bs.alpha); + let color = map_blend_component(&bs.color); + let mapped_blend_state = webgpu_sys::GpuBlendState::new(&alpha, &color); + mapped_color_state.set_blend(&mapped_blend_state); + } + mapped_color_state.set_write_mask(target.write_mask.bits()); + wasm_bindgen::JsValue::from(mapped_color_state) + } + None => wasm_bindgen::JsValue::null(), + }) + .collect::(); + let module = frag.module.inner.as_webgpu(); + let mapped_fragment_desc = webgpu_sys::GpuFragmentState::new(&module.module, &targets); + insert_constants_map(&mapped_fragment_desc, frag.compilation_options.constants); + if let Some(ep) = frag.entry_point { + mapped_fragment_desc.set_entry_point(ep); + } + mapped_desc.set_fragment(&mapped_fragment_desc); + } + + let mapped_multisample = webgpu_sys::GpuMultisampleState::new(); + mapped_multisample.set_count(desc.multisample.count); + mapped_multisample.set_mask(desc.multisample.mask as u32); + mapped_multisample + .set_alpha_to_coverage_enabled(desc.multisample.alpha_to_coverage_enabled); + mapped_desc.set_multisample(&mapped_multisample); + + let mapped_primitive = map_primitive_state(&desc.primitive); + mapped_desc.set_primitive(&mapped_primitive); + + let render_pipeline = self.inner.create_render_pipeline(&mapped_desc).unwrap(); + + WebRenderPipeline { + inner: render_pipeline, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_mesh_pipeline( + &self, + _desc: &crate::MeshPipelineDescriptor<'_>, + ) -> dispatch::DispatchRenderPipeline { + panic!("MESH_SHADER feature must be enabled to call create_mesh_pipeline") + } + + fn create_compute_pipeline( + &self, + desc: &crate::ComputePipelineDescriptor<'_>, + ) -> dispatch::DispatchComputePipeline { + let shader_module = desc.module.inner.as_webgpu(); + let mapped_compute_stage = webgpu_sys::GpuProgrammableStage::new(&shader_module.module); + insert_constants_map(&mapped_compute_stage, desc.compilation_options.constants); + if let Some(ep) = desc.entry_point { + mapped_compute_stage.set_entry_point(ep); + } + let auto_layout = wasm_bindgen::JsValue::from(webgpu_sys::GpuAutoLayoutMode::Auto); + let mapped_desc = webgpu_sys::GpuComputePipelineDescriptor::new( + &match desc.layout { + Some(layout) => { + let layout = &layout.inner.as_webgpu().inner; + JsValue::from(layout) + } + None => auto_layout, + }, + &mapped_compute_stage, + ); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + + let compute_pipeline = self.inner.create_compute_pipeline(&mapped_desc); + + WebComputePipeline { + inner: compute_pipeline, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + unsafe fn create_pipeline_cache( + &self, + _desc: &crate::PipelineCacheDescriptor<'_>, + ) -> dispatch::DispatchPipelineCache { + WebPipelineCache { + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_buffer(&self, desc: &crate::BufferDescriptor<'_>) -> dispatch::DispatchBuffer { + let mapped_desc = webgpu_sys::GpuBufferDescriptor::new(desc.size as f64, desc.usage.bits()); + mapped_desc.set_mapped_at_creation(desc.mapped_at_creation); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + WebBuffer::new(self.inner.create_buffer(&mapped_desc).unwrap(), desc).into() + } + + fn create_texture(&self, desc: &crate::TextureDescriptor<'_>) -> dispatch::DispatchTexture { + let mapped_desc = webgpu_sys::GpuTextureDescriptor::new( + map_texture_format(desc.format), + &map_extent_3d(desc.size), + (desc.usage - crate::TextureUsages::TRANSIENT).bits(), + ); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + mapped_desc.set_dimension(map_texture_dimension(desc.dimension)); + mapped_desc.set_mip_level_count(desc.mip_level_count); + mapped_desc.set_sample_count(desc.sample_count); + let mapped_view_formats = desc + .view_formats + .iter() + .map(|format| JsValue::from(map_texture_format(*format))) + .collect::(); + mapped_desc.set_view_formats(&mapped_view_formats); + + let texture = self.inner.create_texture(&mapped_desc).unwrap(); + WebTexture { + inner: texture, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_external_texture( + &self, + _desc: &crate::ExternalTextureDescriptor<'_>, + _planes: &[&crate::TextureView], + ) -> dispatch::DispatchExternalTexture { + unimplemented!("ExternalTexture not implemented for web"); + } + + fn create_blas( + &self, + _desc: &crate::CreateBlasDescriptor<'_>, + _sizes: crate::BlasGeometrySizeDescriptors, + ) -> (Option, dispatch::DispatchBlas) { + unimplemented!("Raytracing not implemented for web"); + } + + fn create_tlas(&self, _desc: &crate::CreateTlasDescriptor<'_>) -> dispatch::DispatchTlas { + unimplemented!("Raytracing not implemented for web"); + } + + fn create_sampler(&self, desc: &crate::SamplerDescriptor<'_>) -> dispatch::DispatchSampler { + let mapped_desc = webgpu_sys::GpuSamplerDescriptor::new(); + mapped_desc.set_address_mode_u(map_address_mode(desc.address_mode_u)); + mapped_desc.set_address_mode_v(map_address_mode(desc.address_mode_v)); + mapped_desc.set_address_mode_w(map_address_mode(desc.address_mode_w)); + if let Some(compare) = desc.compare { + mapped_desc.set_compare(map_compare_function(compare)); + } + mapped_desc.set_lod_max_clamp(desc.lod_max_clamp); + mapped_desc.set_lod_min_clamp(desc.lod_min_clamp); + mapped_desc.set_mag_filter(map_filter_mode(desc.mag_filter)); + mapped_desc.set_min_filter(map_filter_mode(desc.min_filter)); + mapped_desc.set_mipmap_filter(map_mipmap_filter_mode(desc.mipmap_filter)); + mapped_desc.set_max_anisotropy(desc.anisotropy_clamp); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + + let sampler = self.inner.create_sampler_with_descriptor(&mapped_desc); + + WebSampler { + inner: sampler, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_query_set(&self, desc: &crate::QuerySetDescriptor<'_>) -> dispatch::DispatchQuerySet { + let ty = match desc.ty { + wgt::QueryType::Occlusion => webgpu_sys::GpuQueryType::Occlusion, + wgt::QueryType::Timestamp => webgpu_sys::GpuQueryType::Timestamp, + wgt::QueryType::PipelineStatistics(_) => unreachable!(), + }; + let mapped_desc = webgpu_sys::GpuQuerySetDescriptor::new(desc.count, ty); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + + let query_set = self.inner.create_query_set(&mapped_desc).unwrap(); + + WebQuerySet { + inner: query_set, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_command_encoder( + &self, + desc: &crate::CommandEncoderDescriptor<'_>, + ) -> dispatch::DispatchCommandEncoder { + let mapped_desc = webgpu_sys::GpuCommandEncoderDescriptor::new(); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + + let command_encoder = self + .inner + .create_command_encoder_with_descriptor(&mapped_desc); + + WebCommandEncoder { + inner: command_encoder, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn create_render_bundle_encoder( + &self, + desc: &crate::RenderBundleEncoderDescriptor<'_>, + ) -> dispatch::DispatchRenderBundleEncoder { + let mapped_color_formats = desc + .color_formats + .iter() + .map(|cf| match cf { + Some(cf) => wasm_bindgen::JsValue::from(map_texture_format(*cf)), + None => wasm_bindgen::JsValue::null(), + }) + .collect::(); + let mapped_desc = webgpu_sys::GpuRenderBundleEncoderDescriptor::new(&mapped_color_formats); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + if let Some(ds) = desc.depth_stencil { + mapped_desc.set_depth_stencil_format(map_texture_format(ds.format)); + mapped_desc.set_depth_read_only(ds.depth_read_only); + mapped_desc.set_stencil_read_only(ds.stencil_read_only); + } + mapped_desc.set_sample_count(desc.sample_count); + + let render_bundle_encoder = self + .inner + .create_render_bundle_encoder(&mapped_desc) + .unwrap(); + + WebRenderBundleEncoder { + inner: render_bundle_encoder, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn set_device_lost_callback(&self, device_lost_callback: dispatch::BoxDeviceLostCallback) { + let closure = Closure::once(move |info: JsValue| { + let info = info.dyn_into::().unwrap(); + device_lost_callback( + match info.reason() { + webgpu_sys::GpuDeviceLostReason::Destroyed => { + crate::DeviceLostReason::Destroyed + } + webgpu_sys::GpuDeviceLostReason::Unknown => crate::DeviceLostReason::Unknown, + _ => crate::DeviceLostReason::Unknown, + }, + info.message(), + ); + }); + let _ = self.inner.lost().then(&closure); + // Release memory management of this closure from Rust to the JS GC. + // TODO: This will leak if weak references is not supported. + closure.forget(); + } + + fn on_uncaptured_error(&self, handler: Arc) { + let f = Closure::wrap(Box::new(move |event: webgpu_sys::GpuUncapturedErrorEvent| { + let error = crate::Error::from_js(event.error().value_of()); + handler(error); + }) as Box); + self.inner + .set_onuncapturederror(Some(f.as_ref().unchecked_ref())); + // Release memory management of this closure from Rust to the JS GC. + // TODO: This will leak if weak references is not supported. + f.forget(); + } + + fn push_error_scope(&self, filter: crate::ErrorFilter) -> u32 { + let index = self.error_scope_count.get(); + self.error_scope_count.set( + index + .checked_add(1) + .expect("Greater than 2^32 nested error scopes"), + ); + self.inner.push_error_scope(match filter { + crate::ErrorFilter::OutOfMemory => webgpu_sys::GpuErrorFilter::OutOfMemory, + crate::ErrorFilter::Validation => webgpu_sys::GpuErrorFilter::Validation, + crate::ErrorFilter::Internal => webgpu_sys::GpuErrorFilter::Internal, + }); + index + } + + fn pop_error_scope(&self, index: u32) -> Pin> { + let current_scope_count = self.error_scope_count.get(); + let is_panicking = crate::util::is_panicking(); + if current_scope_count == 0 && !is_panicking { + panic!("Mismatched pop_error_scope call: no error scope for this thread. Error scopes are thread-local."); + } + if index + 1 != current_scope_count && !is_panicking { + panic!( + "Mismatched pop_error_scope call: error scopes must be popped in reverse order." + ); + } + // Decrement the error scope count. We've asserted that the current + // size is `index + 1` above. + self.error_scope_count.set(index); + + let error_promise = self.inner.pop_error_scope(); + Box::pin(MakeSendFuture::new( + wasm_bindgen_futures::JsFuture::from(error_promise), + future_pop_error_scope, + )) + } + + unsafe fn start_graphics_debugger_capture(&self) { + // No capturing api in webgpu + } + + unsafe fn stop_graphics_debugger_capture(&self) { + // No capturing api in webgpu + } + + fn poll(&self, _poll_type: wgt::PollType) -> Result { + // Device is polled automatically + Ok(crate::PollStatus::QueueEmpty) + } + + fn get_internal_counters(&self) -> crate::InternalCounters { + crate::InternalCounters::default() + } + + fn generate_allocator_report(&self) -> Option { + None + } + + fn destroy(&self) { + self.inner.destroy(); + } +} + +impl Drop for WebDevice { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::QueueInterface for WebQueue { + fn write_buffer( + &self, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + data: &[u8], + ) { + let buffer = buffer.as_webgpu(); + self.inner + .write_buffer_with_f64_and_u8_slice_and_f64_and_f64( + &buffer.inner, + offset as f64, + data, + 0f64, + data.len() as f64, + ) + .unwrap(); + } + + fn create_staging_buffer( + &self, + size: crate::BufferSize, + ) -> Option { + Some( + WebQueueWriteBuffer { + inner: vec![0; size.get() as usize].into_boxed_slice(), + ident: crate::cmp::Identifier::create(), + } + .into(), + ) + } + + fn validate_write_buffer( + &self, + buffer: &dispatch::DispatchBuffer, + offset: wgt::BufferAddress, + size: wgt::BufferSize, + ) -> Option<()> { + let buffer = buffer.as_webgpu(); + + let usage = wgt::BufferUsages::from_bits_truncate(buffer.inner.usage()); + // TODO: actually send this down the error scope + if !usage.contains(wgt::BufferUsages::COPY_DST) { + log::error!("Destination buffer is missing the `COPY_DST` usage flag"); + return None; + } + let write_size = u64::from(size); + if !write_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) { + log::error!("Copy size {size} does not respect `COPY_BUFFER_ALIGNMENT`"); + return None; + } + if !offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) { + log::error!( + "Buffer offset {offset} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`" + ); + return None; + } + if write_size + offset > buffer.inner.size() as u64 { + log::error!("copy of {}..{} would end up overrunning the bounds of the destination buffer of size {}", offset, offset + write_size, buffer.inner.size()); + return None; + } + Some(()) + } + + fn write_staging_buffer( + &self, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + staging_buffer: &dispatch::DispatchQueueWriteBuffer, + ) { + let staging_buffer = staging_buffer.as_webgpu(); + + dispatch::QueueInterface::write_buffer(self, buffer, offset, &staging_buffer.inner) + } + + fn write_texture( + &self, + texture: crate::TexelCopyTextureInfo<'_>, + data: &[u8], + data_layout: crate::TexelCopyBufferLayout, + size: crate::Extent3d, + ) { + let mapped_data_layout = webgpu_sys::GpuTexelCopyBufferLayout::new(); + if let Some(bytes_per_row) = data_layout.bytes_per_row { + mapped_data_layout.set_bytes_per_row(bytes_per_row); + } + if let Some(rows_per_image) = data_layout.rows_per_image { + mapped_data_layout.set_rows_per_image(rows_per_image); + } + mapped_data_layout.set_offset(data_layout.offset as f64); + + self.inner + .write_texture_with_u8_slice_and_gpu_extent_3d_dict( + &map_texture_copy_view(texture), + data, + &mapped_data_layout, + &map_extent_3d(size), + ) + .unwrap(); + } + + fn copy_external_image_to_texture( + &self, + source: &crate::CopyExternalImageSourceInfo, + dest: crate::CopyExternalImageDestInfo<&crate::api::Texture>, + size: crate::Extent3d, + ) { + self.inner + .copy_external_image_to_texture_with_gpu_extent_3d_dict( + &map_external_texture_copy_view(source), + &map_tagged_texture_copy_view(dest), + &map_extent_3d(size), + ) + .unwrap(); + } + + fn submit( + &self, + command_buffers: &mut dyn Iterator, + ) -> u64 { + let temp_command_buffers = command_buffers.collect::>(); + + let array = temp_command_buffers + .iter() + .map(|buffer| &buffer.as_webgpu().inner) + .collect::(); + + self.inner.submit(&array); + + 0 + } + + fn get_timestamp_period(&self) -> f32 { + // Timestamp values are always in nanoseconds, see https://gpuweb.github.io/gpuweb/#timestamp + 1.0 + } + + fn on_submitted_work_done(&self, callback: dispatch::BoxSubmittedWorkDoneCallback) { + let promise = self.inner.on_submitted_work_done(); + wasm_bindgen_futures::spawn_local(async move { + match wasm_bindgen_futures::JsFuture::from(promise).await { + Ok(_) => callback(), + Err(error) => { + log::error!("on_submitted_work_done promise failed: {error:?}"); + callback(); + } + } + }); + } + + fn compact_blas( + &self, + _blas: &dispatch::DispatchBlas, + ) -> (Option, dispatch::DispatchBlas) { + unimplemented!("Raytracing not implemented for web") + } +} +impl Drop for WebQueue { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::ShaderModuleInterface for WebShaderModule { + fn get_compilation_info(&self) -> Pin> { + let compilation_info_promise = self.module.get_compilation_info(); + let map_future = Box::new({ + let compilation_info = self.compilation_info.clone(); + move |result| future_compilation_info(result, &compilation_info) + }); + Box::pin(MakeSendFuture::new( + wasm_bindgen_futures::JsFuture::from(compilation_info_promise), + map_future, + )) + } +} +impl Drop for WebShaderModule { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::BindGroupLayoutInterface for WebBindGroupLayout {} +impl Drop for WebBindGroupLayout { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::BindGroupInterface for WebBindGroup {} +impl Drop for WebBindGroup { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::TextureViewInterface for WebTextureView {} +impl Drop for WebTextureView { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::SamplerInterface for WebSampler {} +impl Drop for WebSampler { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::BufferInterface for WebBuffer { + fn map_async( + &self, + mode: crate::MapMode, + range: Range, + callback: dispatch::BufferMapCallback, + ) { + let map_promise = self.inner.map_async_with_f64_and_f64( + map_map_mode(mode), + range.start as f64, + (range.end - range.start) as f64, + ); + + self.set_mapped_range(range); + + register_then_closures(&map_promise, callback, Ok(()), Err(crate::BufferAsyncError)); + } + + fn get_mapped_range( + &self, + sub_range: Range, + ) -> dispatch::DispatchBufferMappedRange { + let actual_mapping = self.get_mapped_range(sub_range); + WebBufferMappedRange { + actual_mapping, + temporary_mapping: OnceCell::new(), + temporary_mapping_modified: false, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn unmap(&self) { + self.inner.unmap(); + self.mapping.borrow_mut().mapped_buffer = None; + } + + fn destroy(&self) { + self.inner.destroy(); + } +} +impl Drop for WebBuffer { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::TextureInterface for WebTexture { + fn create_view( + &self, + desc: &crate::TextureViewDescriptor<'_>, + ) -> dispatch::DispatchTextureView { + let mapped = webgpu_sys::GpuTextureViewDescriptor::new(); + if let Some(dim) = desc.dimension { + mapped.set_dimension(map_texture_view_dimension(dim)); + } + if let Some(format) = desc.format { + mapped.set_format(map_texture_format(format)); + } + mapped.set_aspect(map_texture_aspect(desc.aspect)); + mapped.set_base_array_layer(desc.base_array_layer); + if let Some(count) = desc.array_layer_count { + mapped.set_array_layer_count(count); + } + mapped.set_base_mip_level(desc.base_mip_level); + if let Some(count) = desc.mip_level_count { + mapped.set_mip_level_count(count); + } + if let Some(label) = desc.label { + mapped.set_label(label); + } + mapped.set_usage(desc.usage.unwrap_or(wgt::TextureUsages::empty()).bits()); + + let view = self.inner.create_view_with_descriptor(&mapped).unwrap(); + + WebTextureView { + inner: view, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn destroy(&self) { + self.inner.destroy(); + } +} +impl Drop for WebTexture { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::ExternalTextureInterface for WebExternalTexture { + fn destroy(&self) { + unimplemented!("ExternalTexture not implemented for web"); + } +} +impl Drop for WebExternalTexture { + fn drop(&mut self) { + unimplemented!("ExternalTexture not implemented for web"); + } +} + +impl dispatch::BlasInterface for WebBlas { + fn prepare_compact_async(&self, _callback: BlasCompactCallback) { + unimplemented!("Raytracing not implemented for web") + } + fn ready_for_compaction(&self) -> bool { + unimplemented!("Raytracing not implemented for web") + } +} +impl Drop for WebBlas { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::TlasInterface for WebTlas {} +impl Drop for WebTlas { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::QuerySetInterface for WebQuerySet {} +impl Drop for WebQuerySet { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::PipelineLayoutInterface for WebPipelineLayout {} +impl Drop for WebPipelineLayout { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::RenderPipelineInterface for WebRenderPipeline { + fn get_bind_group_layout(&self, index: u32) -> dispatch::DispatchBindGroupLayout { + let bind_group_layout = self.inner.get_bind_group_layout(index); + + WebBindGroupLayout { + inner: bind_group_layout, + ident: crate::cmp::Identifier::create(), + } + .into() + } +} +impl Drop for WebRenderPipeline { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::ComputePipelineInterface for WebComputePipeline { + fn get_bind_group_layout(&self, index: u32) -> dispatch::DispatchBindGroupLayout { + let bind_group_layout = self.inner.get_bind_group_layout(index); + + WebBindGroupLayout { + inner: bind_group_layout, + ident: crate::cmp::Identifier::create(), + } + .into() + } +} +impl Drop for WebComputePipeline { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::CommandEncoderInterface for WebCommandEncoder { + fn copy_buffer_to_buffer( + &self, + source: &dispatch::DispatchBuffer, + source_offset: crate::BufferAddress, + destination: &dispatch::DispatchBuffer, + destination_offset: crate::BufferAddress, + copy_size: Option, + ) { + let source = source.as_webgpu(); + let destination = destination.as_webgpu(); + + if let Some(size) = copy_size { + self.inner + .copy_buffer_to_buffer_with_f64_and_f64_and_f64( + &source.inner, + source_offset as f64, + &destination.inner, + destination_offset as f64, + size as f64, + ) + .unwrap(); + } else { + self.inner + .copy_buffer_to_buffer_with_f64_and_f64( + &source.inner, + source_offset as f64, + &destination.inner, + destination_offset as f64, + ) + .unwrap(); + } + } + + fn copy_buffer_to_texture( + &self, + source: crate::TexelCopyBufferInfo<'_>, + destination: crate::TexelCopyTextureInfo<'_>, + copy_size: crate::Extent3d, + ) { + self.inner + .copy_buffer_to_texture_with_gpu_extent_3d_dict( + &map_buffer_copy_view(source), + &map_texture_copy_view(destination), + &map_extent_3d(copy_size), + ) + .unwrap(); + } + + fn copy_texture_to_buffer( + &self, + source: crate::TexelCopyTextureInfo<'_>, + destination: crate::TexelCopyBufferInfo<'_>, + copy_size: crate::Extent3d, + ) { + self.inner + .copy_texture_to_buffer_with_gpu_extent_3d_dict( + &map_texture_copy_view(source), + &map_buffer_copy_view(destination), + &map_extent_3d(copy_size), + ) + .unwrap(); + } + + fn copy_texture_to_texture( + &self, + source: crate::TexelCopyTextureInfo<'_>, + destination: crate::TexelCopyTextureInfo<'_>, + copy_size: crate::Extent3d, + ) { + self.inner + .copy_texture_to_texture_with_gpu_extent_3d_dict( + &map_texture_copy_view(source), + &map_texture_copy_view(destination), + &map_extent_3d(copy_size), + ) + .unwrap(); + } + + fn begin_compute_pass( + &self, + desc: &crate::ComputePassDescriptor<'_>, + ) -> dispatch::DispatchComputePass { + let mapped_desc = webgpu_sys::GpuComputePassDescriptor::new(); + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + + if let Some(ref timestamp_writes) = desc.timestamp_writes { + let query_set = timestamp_writes.query_set.inner.as_webgpu(); + let writes = webgpu_sys::GpuComputePassTimestampWrites::new(&query_set.inner); + if let Some(index) = timestamp_writes.beginning_of_pass_write_index { + writes.set_beginning_of_pass_write_index(index); + } + if let Some(index) = timestamp_writes.end_of_pass_write_index { + writes.set_end_of_pass_write_index(index); + } + mapped_desc.set_timestamp_writes(&writes); + } + + let compute_pass = self.inner.begin_compute_pass_with_descriptor(&mapped_desc); + + WebComputePassEncoder { + inner: compute_pass, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn begin_render_pass( + &self, + desc: &crate::RenderPassDescriptor<'_>, + ) -> dispatch::DispatchRenderPass { + let mapped_color_attachments = desc + .color_attachments + .iter() + .map(|attachment| match attachment { + Some(ca) => { + let mut clear_value: Option = None; + let load_value = match ca.ops.load { + crate::LoadOp::Clear(color) => { + clear_value = Some(wasm_bindgen::JsValue::from(map_color(color))); + webgpu_sys::GpuLoadOp::Clear + } + crate::LoadOp::DontCare(_token) => { + // WebGPU can't safely have a ClearOp::DontCare, so we clear to black + // which is ideal for most GPUs. + clear_value = + Some(wasm_bindgen::JsValue::from(map_color(crate::Color::BLACK))); + webgpu_sys::GpuLoadOp::Clear + } + crate::LoadOp::Load => webgpu_sys::GpuLoadOp::Load, + }; + + let view = &ca.view.inner.as_webgpu().inner; + + let mapped_color_attachment = webgpu_sys::GpuRenderPassColorAttachment::new( + load_value, + map_store_op(ca.ops.store), + view, + ); + if let Some(cv) = clear_value { + mapped_color_attachment.set_clear_value(&cv); + } + if let Some(rt) = ca.resolve_target { + let resolve_target_view = &rt.inner.as_webgpu().inner; + mapped_color_attachment.set_resolve_target(resolve_target_view); + } + mapped_color_attachment.set_store_op(map_store_op(ca.ops.store)); + + wasm_bindgen::JsValue::from(mapped_color_attachment) + } + None => wasm_bindgen::JsValue::null(), + }) + .collect::(); + + let mapped_desc = webgpu_sys::GpuRenderPassDescriptor::new(&mapped_color_attachments); + + if let Some(label) = desc.label { + mapped_desc.set_label(label); + } + + if let Some(dsa) = &desc.depth_stencil_attachment { + let depth_stencil_attachment = &dsa.view.inner.as_webgpu().inner; + let mapped_depth_stencil_attachment = + webgpu_sys::GpuRenderPassDepthStencilAttachment::new(depth_stencil_attachment); + if let Some(ref ops) = dsa.depth_ops { + let load_op = match ops.load { + crate::LoadOp::Clear(v) => { + mapped_depth_stencil_attachment.set_depth_clear_value(v); + webgpu_sys::GpuLoadOp::Clear + } + crate::LoadOp::DontCare(_token) => { + // WebGPU can't safely have a ClearOp::DontCare, so we clear to 1.0 + mapped_depth_stencil_attachment.set_depth_clear_value(1.0); + webgpu_sys::GpuLoadOp::Clear + } + crate::LoadOp::Load => webgpu_sys::GpuLoadOp::Load, + }; + mapped_depth_stencil_attachment.set_depth_load_op(load_op); + mapped_depth_stencil_attachment.set_depth_store_op(map_store_op(ops.store)); + } + mapped_depth_stencil_attachment.set_depth_read_only(dsa.depth_ops.is_none()); + if let Some(ref ops) = dsa.stencil_ops { + let load_op = match ops.load { + crate::LoadOp::Clear(v) => { + mapped_depth_stencil_attachment.set_stencil_clear_value(v); + webgpu_sys::GpuLoadOp::Clear + } + crate::LoadOp::DontCare(_token) => { + // WebGPU can't safely have a ClearOp::DontCare, so we clear to 0 + mapped_depth_stencil_attachment.set_stencil_clear_value(0); + webgpu_sys::GpuLoadOp::Clear + } + crate::LoadOp::Load => webgpu_sys::GpuLoadOp::Load, + }; + mapped_depth_stencil_attachment.set_stencil_load_op(load_op); + mapped_depth_stencil_attachment.set_stencil_store_op(map_store_op(ops.store)); + } + mapped_depth_stencil_attachment.set_stencil_read_only(dsa.stencil_ops.is_none()); + mapped_desc.set_depth_stencil_attachment(&mapped_depth_stencil_attachment); + } + + if let Some(ref timestamp_writes) = desc.timestamp_writes { + let query_set = ×tamp_writes.query_set.inner.as_webgpu().inner; + let writes = webgpu_sys::GpuRenderPassTimestampWrites::new(query_set); + if let Some(index) = timestamp_writes.beginning_of_pass_write_index { + writes.set_beginning_of_pass_write_index(index); + } + if let Some(index) = timestamp_writes.end_of_pass_write_index { + writes.set_end_of_pass_write_index(index); + } + mapped_desc.set_timestamp_writes(&writes); + } + + let render_pass = self.inner.begin_render_pass(&mapped_desc).unwrap(); + + WebRenderPassEncoder { + inner: render_pass, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn finish(&mut self) -> dispatch::DispatchCommandBuffer { + let label = self.inner.label(); + let buffer = if label.is_empty() { + self.inner.finish() + } else { + let mapped_desc = webgpu_sys::GpuCommandBufferDescriptor::new(); + mapped_desc.set_label(&label); + + self.inner.finish_with_descriptor(&mapped_desc) + }; + + WebCommandBuffer { + inner: buffer, + ident: crate::cmp::Identifier::create(), + } + .into() + } + + fn clear_texture( + &self, + _texture: &dispatch::DispatchTexture, + _subresource_range: &crate::ImageSubresourceRange, + ) { + unimplemented!("clear_texture is not yet implemented"); + } + + fn clear_buffer( + &self, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_webgpu(); + + match size { + Some(size) => { + self.inner + .clear_buffer_with_f64_and_f64(&buffer.inner, offset as f64, size as f64) + } + None => self + .inner + .clear_buffer_with_f64(&buffer.inner, offset as f64), + } + } + + fn insert_debug_marker(&self, label: &str) { + self.inner.insert_debug_marker(label) + } + + fn push_debug_group(&self, group_label: &str) { + self.inner.push_debug_group(group_label) + } + + fn pop_debug_group(&self) { + self.inner.pop_debug_group() + } + + fn write_timestamp(&self, _query_set: &dispatch::DispatchQuerySet, _query_index: u32) { + // Not available on WebGPU. + // This was part of the spec originally but got removed, see https://github.com/gpuweb/gpuweb/pull/4370 + panic!("TIMESTAMP_QUERY_INSIDE_ENCODERS feature must be enabled to call write_timestamp on a command encoder.") + } + + fn resolve_query_set( + &self, + query_set: &dispatch::DispatchQuerySet, + first_query: u32, + query_count: u32, + destination: &dispatch::DispatchBuffer, + destination_offset: crate::BufferAddress, + ) { + let query_set = &query_set.as_webgpu().inner; + let destination = &destination.as_webgpu().inner; + + self.inner.resolve_query_set_with_u32( + query_set, + first_query, + query_count, + destination, + destination_offset as u32, + ); + } + + fn mark_acceleration_structures_built<'a>( + &self, + _blas: &mut dyn Iterator, + _tlas: &mut dyn Iterator, + ) { + unimplemented!("Raytracing not implemented for web"); + } + + fn build_acceleration_structures<'a>( + &self, + _blas: &mut dyn Iterator>, + _tlas: &mut dyn Iterator, + ) { + unimplemented!("Raytracing not implemented for web"); + } + + fn transition_resources<'a>( + &mut self, + _buffer_transitions: &mut dyn Iterator< + Item = wgt::BufferTransition<&'a dispatch::DispatchBuffer>, + >, + _texture_transitions: &mut dyn Iterator< + Item = wgt::TextureTransition<&'a dispatch::DispatchTexture>, + >, + ) { + // no-op + } +} +impl Drop for WebCommandEncoder { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::PipelineCacheInterface for WebPipelineCache { + fn get_data(&self) -> Option> { + todo!() + } +} +impl Drop for WebPipelineCache { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::ComputePassInterface for WebComputePassEncoder { + fn set_pipeline(&mut self, pipeline: &dispatch::DispatchComputePipeline) { + let pipeline = &pipeline.as_webgpu().inner; + self.inner.set_pipeline(pipeline); + } + + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&dispatch::DispatchBindGroup>, + offsets: &[crate::DynamicOffset], + ) { + let bind_group = bind_group.map(|bind_group| &bind_group.as_webgpu().inner); + + if offsets.is_empty() { + self.inner.set_bind_group(index, bind_group); + } else { + self.inner + .set_bind_group_with_u32_slice_and_f64_and_dynamic_offsets_data_length( + index, + bind_group, + offsets, + 0f64, + offsets.len() as u32, + ) + .unwrap(); + } + } + + fn set_immediates(&mut self, _offset: u32, _data: &[u8]) { + panic!("IMMEDIATES feature must be enabled to call set_immediates") + } + + fn insert_debug_marker(&mut self, label: &str) { + self.inner.insert_debug_marker(label); + } + + fn push_debug_group(&mut self, group_label: &str) { + self.inner.push_debug_group(group_label); + } + + fn pop_debug_group(&mut self) { + self.inner.pop_debug_group(); + } + + fn write_timestamp(&mut self, _query_set: &dispatch::DispatchQuerySet, _query_index: u32) { + panic!("TIMESTAMP_QUERY_INSIDE_PASSES feature must be enabled to call write_timestamp in a compute pass.") + } + + fn begin_pipeline_statistics_query( + &mut self, + _query_set: &dispatch::DispatchQuerySet, + _query_index: u32, + ) { + // Not available in gecko yet + } + + fn end_pipeline_statistics_query(&mut self) { + // Not available in gecko yet + } + + fn dispatch_workgroups(&mut self, x: u32, y: u32, z: u32) { + self.inner + .dispatch_workgroups_with_workgroup_count_y_and_workgroup_count_z(x, y, z); + } + + fn dispatch_workgroups_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let indirect_buffer = indirect_buffer.as_webgpu(); + + self.inner + .dispatch_workgroups_indirect_with_f64(&indirect_buffer.inner, indirect_offset as f64); + } +} +impl Drop for WebComputePassEncoder { + fn drop(&mut self) { + self.inner.end(); + } +} + +impl dispatch::RenderPassInterface for WebRenderPassEncoder { + fn set_pipeline(&mut self, pipeline: &dispatch::DispatchRenderPipeline) { + let pipeline = &pipeline.as_webgpu().inner; + + self.inner.set_pipeline(pipeline); + } + + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&dispatch::DispatchBindGroup>, + offsets: &[crate::DynamicOffset], + ) { + let bind_group = bind_group.map(|bind_group| &bind_group.as_webgpu().inner); + + if offsets.is_empty() { + self.inner.set_bind_group(index, bind_group); + } else { + self.inner + .set_bind_group_with_u32_slice_and_f64_and_dynamic_offsets_data_length( + index, + bind_group, + offsets, + 0f64, + offsets.len() as u32, + ) + .unwrap(); + } + } + + fn set_index_buffer( + &mut self, + buffer: &dispatch::DispatchBuffer, + index_format: crate::IndexFormat, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_webgpu(); + let index_format = map_index_format(index_format); + + if let Some(size) = size { + self.inner.set_index_buffer_with_f64_and_f64( + &buffer.inner, + index_format, + offset as f64, + size.get() as f64, + ); + } else { + self.inner + .set_index_buffer_with_f64(&buffer.inner, index_format, offset as f64); + } + } + + fn set_vertex_buffer( + &mut self, + slot: u32, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_webgpu(); + + if let Some(size) = size { + self.inner.set_vertex_buffer_with_f64_and_f64( + slot, + Some(&buffer.inner), + offset as f64, + size.get() as f64, + ); + } else { + self.inner + .set_vertex_buffer_with_f64(slot, Some(&buffer.inner), offset as f64); + } + } + + fn set_immediates(&mut self, _offset: u32, _data: &[u8]) { + panic!("IMMEDIATES feature must be enabled to call set_immediates") + } + + fn set_blend_constant(&mut self, color: crate::Color) { + self.inner + .set_blend_constant_with_gpu_color_dict(&map_color(color)) + .unwrap(); + } + + fn set_scissor_rect(&mut self, x: u32, y: u32, width: u32, height: u32) { + self.inner.set_scissor_rect(x, y, width, height); + } + + fn set_viewport( + &mut self, + x: f32, + y: f32, + width: f32, + height: f32, + min_depth: f32, + max_depth: f32, + ) { + self.inner + .set_viewport(x, y, width, height, min_depth, max_depth); + } + + fn set_stencil_reference(&mut self, reference: u32) { + self.inner.set_stencil_reference(reference); + } + + fn draw(&mut self, vertices: Range, instances: Range) { + self.inner + .draw_with_instance_count_and_first_vertex_and_first_instance( + vertices.end - vertices.start, + instances.end - instances.start, + vertices.start, + instances.start, + ); + } + + fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range) { + self.inner + .draw_indexed_with_instance_count_and_first_index_and_base_vertex_and_first_instance( + indices.end - indices.start, + instances.end - instances.start, + indices.start, + base_vertex, + instances.start, + ) + } + + fn draw_mesh_tasks(&mut self, _group_count_x: u32, _group_count_y: u32, _group_count_z: u32) { + panic!("MESH_SHADER feature must be enabled to call draw_mesh_tasks") + } + + fn draw_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let buffer = indirect_buffer.as_webgpu(); + self.inner + .draw_indirect_with_f64(&buffer.inner, indirect_offset as f64); + } + + fn draw_indexed_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let buffer = indirect_buffer.as_webgpu(); + self.inner + .draw_indexed_indirect_with_f64(&buffer.inner, indirect_offset as f64); + } + + fn draw_mesh_tasks_indirect( + &mut self, + _indirect_buffer: &dispatch::DispatchBuffer, + _indirect_offset: crate::BufferAddress, + ) { + panic!("MESH_SHADER feature must be enabled to call draw_mesh_tasks_indirect") + } + + fn multi_draw_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + count: u32, + ) { + let buffer = indirect_buffer.as_webgpu(); + + for i in 0..count { + let offset = indirect_offset + i as crate::BufferAddress * 16; + self.inner + .draw_indirect_with_f64(&buffer.inner, offset as f64); + } + } + + fn multi_draw_indexed_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + count: u32, + ) { + let buffer = indirect_buffer.as_webgpu(); + + for i in 0..count { + let offset = indirect_offset + i as crate::BufferAddress * 20; + self.inner + .draw_indexed_indirect_with_f64(&buffer.inner, offset as f64); + } + } + + fn multi_draw_mesh_tasks_indirect( + &mut self, + _indirect_buffer: &dispatch::DispatchBuffer, + _indirect_offset: crate::BufferAddress, + _count: u32, + ) { + panic!("MESH_SHADER feature must be enabled to call multi_draw_mesh_tasks_indirect") + } + + fn multi_draw_indirect_count( + &mut self, + _indirect_buffer: &dispatch::DispatchBuffer, + _indirect_offset: crate::BufferAddress, + _count_buffer: &dispatch::DispatchBuffer, + _count_buffer_offset: crate::BufferAddress, + _max_count: u32, + ) { + panic!( + "MULTI_DRAW_INDIRECT_COUNT feature must be enabled to call multi_draw_indirect_count" + ) + } + + fn multi_draw_indexed_indirect_count( + &mut self, + _indirect_buffer: &dispatch::DispatchBuffer, + _indirect_offset: crate::BufferAddress, + _count_buffer: &dispatch::DispatchBuffer, + _count_buffer_offset: crate::BufferAddress, + _max_count: u32, + ) { + panic!("MULTI_DRAW_INDIRECT_COUNT feature must be enabled to call multi_draw_indexed_indirect_count") + } + + fn multi_draw_mesh_tasks_indirect_count( + &mut self, + _indirect_buffer: &dispatch::DispatchBuffer, + _indirect_offset: crate::BufferAddress, + _count_buffer: &dispatch::DispatchBuffer, + _count_buffer_offset: crate::BufferAddress, + _max_count: u32, + ) { + panic!("MESH_SHADER feature must be enabled to call multi_draw_mesh_tasks_indirect_count") + } + + fn insert_debug_marker(&mut self, label: &str) { + self.inner.insert_debug_marker(label); + } + + fn push_debug_group(&mut self, group_label: &str) { + self.inner.push_debug_group(group_label); + } + + fn pop_debug_group(&mut self) { + self.inner.pop_debug_group(); + } + + fn write_timestamp(&mut self, _query_set: &dispatch::DispatchQuerySet, _query_index: u32) { + panic!("TIMESTAMP_QUERY_INSIDE_PASSES feature must be enabled to call write_timestamp in a render pass.") + } + + fn begin_occlusion_query(&mut self, query_index: u32) { + self.inner.begin_occlusion_query(query_index); + } + + fn end_occlusion_query(&mut self) { + self.inner.end_occlusion_query(); + } + + fn begin_pipeline_statistics_query( + &mut self, + _query_set: &dispatch::DispatchQuerySet, + _query_index: u32, + ) { + // Removed from WebGPU in https://github.com/gpuweb/gpuweb/pull/2296 + } + + fn end_pipeline_statistics_query(&mut self) { + // Removed from WebGPU https://github.com/gpuweb/gpuweb/pull/2296 + } + + fn execute_bundles( + &mut self, + render_bundles: &mut dyn Iterator, + ) { + let mapped = render_bundles + .map(|bundle| &bundle.as_webgpu().inner) + .collect::(); + self.inner.execute_bundles(&mapped); + } +} +impl Drop for WebRenderPassEncoder { + fn drop(&mut self) { + self.inner.end(); + } +} + +impl dispatch::CommandBufferInterface for WebCommandBuffer {} +impl Drop for WebCommandBuffer { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::RenderBundleEncoderInterface for WebRenderBundleEncoder { + fn set_pipeline(&mut self, pipeline: &dispatch::DispatchRenderPipeline) { + let pipeline = &pipeline.as_webgpu().inner; + self.inner.set_pipeline(pipeline); + } + + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&dispatch::DispatchBindGroup>, + offsets: &[crate::DynamicOffset], + ) { + let bind_group = bind_group.map(|bind_group| &bind_group.as_webgpu().inner); + + if offsets.is_empty() { + self.inner.set_bind_group(index, bind_group); + } else { + self.inner + .set_bind_group_with_u32_slice_and_f64_and_dynamic_offsets_data_length( + index, + bind_group, + offsets, + 0f64, + offsets.len() as u32, + ) + .unwrap(); + } + } + + fn set_index_buffer( + &mut self, + buffer: &dispatch::DispatchBuffer, + index_format: crate::IndexFormat, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_webgpu(); + let index_format = map_index_format(index_format); + + if let Some(size) = size { + self.inner.set_index_buffer_with_f64_and_f64( + &buffer.inner, + index_format, + offset as f64, + size.get() as f64, + ); + } else { + self.inner + .set_index_buffer_with_f64(&buffer.inner, index_format, offset as f64); + } + } + + fn set_vertex_buffer( + &mut self, + slot: u32, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_webgpu(); + + if let Some(size) = size { + self.inner.set_vertex_buffer_with_f64_and_f64( + slot, + Some(&buffer.inner), + offset as f64, + size.get() as f64, + ); + } else { + self.inner + .set_vertex_buffer_with_f64(slot, Some(&buffer.inner), offset as f64); + } + } + + fn set_immediates(&mut self, _offset: u32, _data: &[u8]) { + panic!("IMMEDIATES feature must be enabled to call set_immediates") + } + + fn draw(&mut self, vertices: Range, instances: Range) { + self.inner + .draw_with_instance_count_and_first_vertex_and_first_instance( + vertices.end - vertices.start, + instances.end - instances.start, + vertices.start, + instances.start, + ); + } + + fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range) { + self.inner + .draw_indexed_with_instance_count_and_first_index_and_base_vertex_and_first_instance( + indices.end - indices.start, + instances.end - instances.start, + indices.start, + base_vertex, + instances.start, + ) + } + + fn draw_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let buffer = indirect_buffer.as_webgpu(); + self.inner + .draw_indirect_with_f64(&buffer.inner, indirect_offset as f64); + } + + fn draw_indexed_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let buffer = indirect_buffer.as_webgpu(); + self.inner + .draw_indexed_indirect_with_f64(&buffer.inner, indirect_offset as f64); + } + + fn finish(self, desc: &crate::RenderBundleDescriptor<'_>) -> dispatch::DispatchRenderBundle + where + Self: Sized, + { + let bundle = match desc.label { + Some(label) => { + let mapped_desc = webgpu_sys::GpuRenderBundleDescriptor::new(); + mapped_desc.set_label(label); + self.inner.finish_with_descriptor(&mapped_desc) + } + None => self.inner.finish(), + }; + + WebRenderBundle { + inner: bundle, + ident: crate::cmp::Identifier::create(), + } + .into() + } +} +impl Drop for WebRenderBundleEncoder { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::RenderBundleInterface for WebRenderBundle {} +impl Drop for WebRenderBundle { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::SurfaceInterface for WebSurface { + fn get_capabilities(&self, _adapter: &dispatch::DispatchAdapter) -> wgt::SurfaceCapabilities { + let mut formats = vec![ + wgt::TextureFormat::Rgba8Unorm, + wgt::TextureFormat::Bgra8Unorm, + wgt::TextureFormat::Rgba16Float, + ]; + let mut mapped_formats = formats.iter().map(|format| map_texture_format(*format)); + // Preferred canvas format will only be either "rgba8unorm" or "bgra8unorm". + // https://www.w3.org/TR/webgpu/#dom-gpu-getpreferredcanvasformat + let preferred_format = self + .gpu + .as_ref() + .expect("Caller could not have created an adapter if gpu is undefined.") + .get_preferred_canvas_format(); + if let Some(index) = mapped_formats.position(|format| format == preferred_format) { + formats.swap(0, index); + } + + wgt::SurfaceCapabilities { + // https://gpuweb.github.io/gpuweb/#supported-context-formats + formats, + // Doesn't really have meaning on the web. + present_modes: vec![wgt::PresentMode::Fifo], + alpha_modes: vec![wgt::CompositeAlphaMode::Opaque], + // Statically set to RENDER_ATTACHMENT for now. See https://gpuweb.github.io/gpuweb/#dom-gpucanvasconfiguration-usage + usages: wgt::TextureUsages::RENDER_ATTACHMENT, + } + } + + fn configure(&self, device: &dispatch::DispatchDevice, config: &crate::SurfaceConfiguration) { + let device = device.as_webgpu(); + + match self.canvas { + Canvas::Canvas(ref canvas) => { + canvas.set_width(config.width); + canvas.set_height(config.height); + } + Canvas::Offscreen(ref canvas) => { + canvas.set_width(config.width); + canvas.set_height(config.height); + } + } + + if let wgt::PresentMode::Mailbox | wgt::PresentMode::Immediate = config.present_mode { + panic!("Only FIFO/Auto* is supported on web"); + } + if let wgt::CompositeAlphaMode::PostMultiplied | wgt::CompositeAlphaMode::Inherit = + config.alpha_mode + { + panic!("Only Opaque/Auto or PreMultiplied alpha mode are supported on web"); + } + let alpha_mode = match config.alpha_mode { + wgt::CompositeAlphaMode::PreMultiplied => webgpu_sys::GpuCanvasAlphaMode::Premultiplied, + _ => webgpu_sys::GpuCanvasAlphaMode::Opaque, + }; + let mapped = webgpu_sys::GpuCanvasConfiguration::new( + &device.inner, + map_texture_format(config.format), + ); + mapped.set_usage(config.usage.bits()); + mapped.set_alpha_mode(alpha_mode); + let mapped_view_formats = config + .view_formats + .iter() + .map(|format| JsValue::from(map_texture_format(*format))) + .collect::(); + mapped.set_view_formats(&mapped_view_formats); + self.context.configure(&mapped).unwrap(); + } + + fn get_current_texture( + &self, + ) -> ( + Option, + crate::SurfaceStatus, + dispatch::DispatchSurfaceOutputDetail, + ) { + let surface_texture = self.context.get_current_texture().unwrap(); + + let web_surface_texture = WebTexture { + inner: surface_texture, + ident: crate::cmp::Identifier::create(), + }; + + ( + Some(web_surface_texture.into()), + crate::SurfaceStatus::Good, + WebSurfaceOutputDetail { + ident: crate::cmp::Identifier::create(), + } + .into(), + ) + } +} +impl Drop for WebSurface { + fn drop(&mut self) { + // no-op + } +} + +impl dispatch::SurfaceOutputDetailInterface for WebSurfaceOutputDetail { + fn present(&self) { + // Swapchain is presented automatically on the web. + } + + fn texture_discard(&self) { + // Can't really discard the texture on the web. + } +} +impl Drop for WebSurfaceOutputDetail { + fn drop(&mut self) { + // no-op + } +} + +impl WebBufferMappedRange { + fn get_temporary_mapping(&self) -> &[u8] { + self.temporary_mapping + .get_or_init(|| self.actual_mapping.to_vec()) + } +} +impl dispatch::BufferMappedRangeInterface for WebBufferMappedRange { + fn len(&self) -> usize { + self.get_temporary_mapping().len() + } + + #[inline] + unsafe fn read_slice(&self) -> &[u8] { + self.get_temporary_mapping() + } + + #[inline] + unsafe fn write_slice(&mut self) -> WriteOnly<'_, [u8]> { + self.temporary_mapping_modified = true; + self.get_temporary_mapping(); + let t: &mut Vec = self.temporary_mapping.get_mut().unwrap(); + WriteOnly::from_mut(t) + } + + #[inline] + fn as_uint8array(&self) -> &js_sys::Uint8Array { + &self.actual_mapping + } +} +impl Drop for WebBufferMappedRange { + fn drop(&mut self) { + if !self.temporary_mapping_modified { + // For efficiency, skip the copy if it is not needed. + // This is also how we skip copying back on *read-only* mappings. + return; + } + + // Copy from the temporary mapping back into the array buffer that was + // originally provided by the browser + let temporary_mapping_slice = self.temporary_mapping.get().unwrap().as_slice(); + unsafe { + // Note: no allocations can happen between `view` and `set`, or this + // will break + self.actual_mapping + .set(&js_sys::Uint8Array::view(temporary_mapping_slice), 0); + } + } +} + +impl dispatch::QueueWriteBufferInterface for WebQueueWriteBuffer { + #[inline] + fn len(&self) -> usize { + self.inner.len() + } + + #[inline] + unsafe fn write_slice(&mut self) -> WriteOnly<'_, [u8]> { + WriteOnly::from_mut(&mut *self.inner) + } +} +impl Drop for WebQueueWriteBuffer { + fn drop(&mut self) { + // The api struct calls write_staging_buffer + + // no-op + } +} + +/// Adds the constants map to the given pipeline descriptor if the map is nonempty. +/// Panics if the map cannot be set. +/// +/// This function is necessary because the constants array is not currently +/// exposed by `wasm-bindgen`. See the following issues for details: +/// - [gfx-rs/wgpu#5688](https://github.com/gfx-rs/wgpu/pull/5688) +/// - [rustwasm/wasm-bindgen#3587](https://github.com/rustwasm/wasm-bindgen/issues/3587) +fn insert_constants_map(target: &JsValue, map: &[(&str, f64)]) { + if !map.is_empty() { + js_sys::Reflect::set( + target, + &JsValue::from_str("constants"), + &hashmap_to_jsvalue(map), + ) + .expect("Setting the values in a Javascript pipeline descriptor should never fail"); + } +} + +/// Converts a hashmap to a Javascript object. +fn hashmap_to_jsvalue(map: &[(&str, f64)]) -> JsValue { + let obj = js_sys::Object::new(); + + for &(key, v) in map.iter() { + js_sys::Reflect::set(&obj, &JsValue::from_str(key), &JsValue::from_f64(v)) + .expect("Setting the values in a Javascript map should never fail"); + } + + JsValue::from(obj) +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/defined_non_null_js_value.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/defined_non_null_js_value.rs new file mode 100644 index 00000000..c2b61bfd --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/defined_non_null_js_value.rs @@ -0,0 +1,46 @@ +use core::ops::{Deref, DerefMut}; + +use wasm_bindgen::JsValue; + +/// Derefs to a [`JsValue`] that's known not to be `undefined` or `null`. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DefinedNonNullJsValue(T); + +impl DefinedNonNullJsValue +where + T: AsRef, +{ + pub fn new(value: T) -> Option { + if value.as_ref().is_undefined() || value.as_ref().is_null() { + None + } else { + Some(Self(value)) + } + } +} + +impl Deref for DefinedNonNullJsValue { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for DefinedNonNullJsValue { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl AsRef for DefinedNonNullJsValue { + fn as_ref(&self) -> &T { + &self.0 + } +} + +impl AsMut for DefinedNonNullJsValue { + fn as_mut(&mut self) -> &mut T { + &mut self.0 + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/ext_bindings.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/ext_bindings.rs new file mode 100644 index 00000000..218caf75 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/ext_bindings.rs @@ -0,0 +1,45 @@ +//! Extension bindings for WebGPU. +//! +//! These contain ideomatic Rust extension traits for various parts of the WebGPU +//! bindings that are missing, need to be improved, or otherwise need to be different +//! from the generated web_sys bindings. + +use crate::backend::webgpu::webgpu_sys; +use wasm_bindgen::prelude::*; + +/// Extension trait for [`web_sys::Navigator`] and [`web_sys::WorkerNavigator`] to +/// access the `gpu` property. +pub trait NavigatorGpu { + /// Get the `gpu` property. + /// + /// This is intentionally a free function, to prevent overload conflicts with + /// the method if it is enabled in web-sys itself. + fn gpu(navigator: &Self) -> webgpu_sys::Gpu; +} + +// --- Bindings for `Navigator` --- +#[wasm_bindgen] +extern "C" { + /// Create a fake class which we tell wasm-bindgen has access to the `gpu` property. + #[wasm_bindgen] + type NavigatorWithGpu; + + #[wasm_bindgen(method, getter)] + fn gpu(ext: &NavigatorWithGpu) -> webgpu_sys::Gpu; +} + +impl NavigatorGpu for web_sys::Navigator { + fn gpu(navigator: &Self) -> webgpu_sys::Gpu { + // Must be an unchecked ref as this class does not exist at runtime. + let extension: &NavigatorWithGpu = navigator.unchecked_ref(); + extension.gpu() + } +} + +impl NavigatorGpu for web_sys::WorkerNavigator { + fn gpu(navigator: &Self) -> webgpu_sys::Gpu { + // Must be an unchecked ref as this class does not exist at runtime. + let extension: &NavigatorWithGpu = navigator.unchecked_ref(); + extension.gpu() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_Gpu.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_Gpu.rs new file mode 100644 index 00000000..3a70b0c2 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_Gpu.rs @@ -0,0 +1,87 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPU , typescript_type = "GPU")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `Gpu` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPU)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `Gpu`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type Gpu; + + # [wasm_bindgen (structural , method , getter , js_class = "GPU" , js_name = wgslLanguageFeatures)] + #[doc = "Getter for the `wgslLanguageFeatures` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPU/wgslLanguageFeatures)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `Gpu`, `WgslLanguageFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn wgsl_language_features(this: &Gpu) -> WgslLanguageFeatures; + + # [wasm_bindgen (method , structural , js_class = "GPU" , js_name = getPreferredCanvasFormat)] + #[doc = "The `getPreferredCanvasFormat()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPU/getPreferredCanvasFormat)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `Gpu`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_preferred_canvas_format(this: &Gpu) -> GpuTextureFormat; + + # [wasm_bindgen (method , structural , js_class = "GPU" , js_name = requestAdapter)] + #[doc = "The `requestAdapter()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPU/requestAdapter)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `Gpu`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn request_adapter(this: &Gpu) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPU" , js_name = requestAdapter)] + #[doc = "The `requestAdapter()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPU/requestAdapter)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `Gpu`, `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn request_adapter_with_options( + this: &Gpu, + options: &GpuRequestAdapterOptions, + ) -> ::js_sys::Promise; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAdapter.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAdapter.rs new file mode 100644 index 00000000..c1bc6d46 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAdapter.rs @@ -0,0 +1,109 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUAdapter , typescript_type = "GPUAdapter")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuAdapter` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapter)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapter`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuAdapter; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUAdapter" , js_name = features)] + #[doc = "Getter for the `features` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapter/features)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapter`, `GpuSupportedFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn features(this: &GpuAdapter) -> GpuSupportedFeatures; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUAdapter" , js_name = limits)] + #[doc = "Getter for the `limits` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapter/limits)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapter`, `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn limits(this: &GpuAdapter) -> GpuSupportedLimits; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUAdapter" , js_name = info)] + #[doc = "Getter for the `info` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapter/info)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapter`, `GpuAdapterInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn info(this: &GpuAdapter) -> GpuAdapterInfo; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUAdapter" , js_name = isFallbackAdapter)] + #[doc = "Getter for the `isFallbackAdapter` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapter/isFallbackAdapter)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapter`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn is_fallback_adapter(this: &GpuAdapter) -> bool; + + # [wasm_bindgen (method , structural , js_class = "GPUAdapter" , js_name = requestDevice)] + #[doc = "The `requestDevice()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapter/requestDevice)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapter`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn request_device(this: &GpuAdapter) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUAdapter" , js_name = requestDevice)] + #[doc = "The `requestDevice()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapter/requestDevice)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapter`, `GpuDeviceDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn request_device_with_descriptor( + this: &GpuAdapter, + descriptor: &GpuDeviceDescriptor, + ) -> ::js_sys::Promise; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAdapterInfo.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAdapterInfo.rs new file mode 100644 index 00000000..c99000fd --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAdapterInfo.rs @@ -0,0 +1,84 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUAdapterInfo , typescript_type = "GPUAdapterInfo")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuAdapterInfo` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapterInfo)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapterInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuAdapterInfo; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUAdapterInfo" , js_name = vendor)] + #[doc = "Getter for the `vendor` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapterInfo/vendor)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapterInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn vendor(this: &GpuAdapterInfo) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUAdapterInfo" , js_name = architecture)] + #[doc = "Getter for the `architecture` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapterInfo/architecture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapterInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn architecture(this: &GpuAdapterInfo) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUAdapterInfo" , js_name = device)] + #[doc = "Getter for the `device` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapterInfo/device)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapterInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn device(this: &GpuAdapterInfo) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUAdapterInfo" , js_name = description)] + #[doc = "Getter for the `description` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUAdapterInfo/description)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapterInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn description(this: &GpuAdapterInfo) -> ::alloc::string::String; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAddressMode.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAddressMode.rs new file mode 100644 index 00000000..11e70cff --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAddressMode.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuAddressMode` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuAddressMode`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuAddressMode { + ClampToEdge = "clamp-to-edge", + Repeat = "repeat", + MirrorRepeat = "mirror-repeat", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAutoLayoutMode.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAutoLayoutMode.rs new file mode 100644 index 00000000..62d94465 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuAutoLayoutMode.rs @@ -0,0 +1,36 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuAutoLayoutMode` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuAutoLayoutMode`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuAutoLayoutMode { + Auto = "auto", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroup.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroup.rs new file mode 100644 index 00000000..c056487a --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroup.rs @@ -0,0 +1,62 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBindGroup , typescript_type = "GPUBindGroup")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBindGroup` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBindGroup; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUBindGroup" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBindGroup/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuBindGroup) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUBindGroup" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBindGroup/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuBindGroup, value: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupDescriptor.rs new file mode 100644 index 00000000..3c58c1c2 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupDescriptor.rs @@ -0,0 +1,126 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBindGroupDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBindGroupDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBindGroupDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuBindGroupDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuBindGroupDescriptor, val: &str); + + #[doc = "Get the `entries` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "entries")] + pub fn get_entries(this: &GpuBindGroupDescriptor) -> ::js_sys::Array; + + #[doc = "Change the `entries` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "entries")] + pub fn set_entries(this: &GpuBindGroupDescriptor, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `layout` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupDescriptor`, `GpuBindGroupLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "layout")] + pub fn get_layout(this: &GpuBindGroupDescriptor) -> GpuBindGroupLayout; + + #[doc = "Change the `layout` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupDescriptor`, `GpuBindGroupLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "layout")] + pub fn set_layout(this: &GpuBindGroupDescriptor, val: &GpuBindGroupLayout); +} + +impl GpuBindGroupDescriptor { + #[doc = "Construct a new `GpuBindGroupDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupDescriptor`, `GpuBindGroupLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(entries: &::wasm_bindgen::JsValue, layout: &GpuBindGroupLayout) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_entries(entries); + ret.set_layout(layout); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_entries()` instead."] + pub fn entries(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_entries(val); + self + } + + #[deprecated = "Use `set_layout()` instead."] + pub fn layout(&mut self, val: &GpuBindGroupLayout) -> &mut Self { + self.set_layout(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupEntry.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupEntry.rs new file mode 100644 index 00000000..266fd1cb --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupEntry.rs @@ -0,0 +1,102 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBindGroupEntry)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBindGroupEntry` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBindGroupEntry; + + #[doc = "Get the `binding` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "binding")] + pub fn get_binding(this: &GpuBindGroupEntry) -> u32; + + #[doc = "Change the `binding` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "binding")] + pub fn set_binding(this: &GpuBindGroupEntry, val: u32); + + #[doc = "Get the `resource` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "resource")] + pub fn get_resource(this: &GpuBindGroupEntry) -> ::wasm_bindgen::JsValue; + + #[doc = "Change the `resource` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "resource")] + pub fn set_resource(this: &GpuBindGroupEntry, val: &::wasm_bindgen::JsValue); +} + +impl GpuBindGroupEntry { + #[doc = "Construct a new `GpuBindGroupEntry`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(binding: u32, resource: &::wasm_bindgen::JsValue) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_binding(binding); + ret.set_resource(resource); + ret + } + + #[deprecated = "Use `set_binding()` instead."] + pub fn binding(&mut self, val: u32) -> &mut Self { + self.set_binding(val); + self + } + + #[deprecated = "Use `set_resource()` instead."] + pub fn resource(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_resource(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayout.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayout.rs new file mode 100644 index 00000000..7c500ea0 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayout.rs @@ -0,0 +1,62 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBindGroupLayout , typescript_type = "GPUBindGroupLayout")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBindGroupLayout` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBindGroupLayout)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBindGroupLayout; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUBindGroupLayout" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBindGroupLayout/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuBindGroupLayout) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUBindGroupLayout" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBindGroupLayout/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuBindGroupLayout, value: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayoutDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayoutDescriptor.rs new file mode 100644 index 00000000..5e11b9f8 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayoutDescriptor.rs @@ -0,0 +1,101 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBindGroupLayoutDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBindGroupLayoutDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBindGroupLayoutDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuBindGroupLayoutDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuBindGroupLayoutDescriptor, val: &str); + + #[doc = "Get the `entries` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "entries")] + pub fn get_entries(this: &GpuBindGroupLayoutDescriptor) -> ::js_sys::Array; + + #[doc = "Change the `entries` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "entries")] + pub fn set_entries(this: &GpuBindGroupLayoutDescriptor, val: &::wasm_bindgen::JsValue); +} + +impl GpuBindGroupLayoutDescriptor { + #[doc = "Construct a new `GpuBindGroupLayoutDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(entries: &::wasm_bindgen::JsValue) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_entries(entries); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_entries()` instead."] + pub fn entries(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_entries(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayoutEntry.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayoutEntry.rs new file mode 100644 index 00000000..d641712e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBindGroupLayoutEntry.rs @@ -0,0 +1,232 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBindGroupLayoutEntry)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBindGroupLayoutEntry` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBindGroupLayoutEntry; + + #[doc = "Get the `binding` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "binding")] + pub fn get_binding(this: &GpuBindGroupLayoutEntry) -> u32; + + #[doc = "Change the `binding` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "binding")] + pub fn set_binding(this: &GpuBindGroupLayoutEntry, val: u32); + + #[doc = "Get the `buffer` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuBufferBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "buffer")] + pub fn get_buffer(this: &GpuBindGroupLayoutEntry) -> Option; + + #[doc = "Change the `buffer` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuBufferBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "buffer")] + pub fn set_buffer(this: &GpuBindGroupLayoutEntry, val: &GpuBufferBindingLayout); + + #[doc = "Get the `externalTexture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuExternalTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "externalTexture")] + pub fn get_external_texture( + this: &GpuBindGroupLayoutEntry, + ) -> Option; + + #[doc = "Change the `externalTexture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuExternalTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "externalTexture")] + pub fn set_external_texture( + this: &GpuBindGroupLayoutEntry, + val: &GpuExternalTextureBindingLayout, + ); + + #[doc = "Get the `sampler` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuSamplerBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "sampler")] + pub fn get_sampler(this: &GpuBindGroupLayoutEntry) -> Option; + + #[doc = "Change the `sampler` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuSamplerBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "sampler")] + pub fn set_sampler(this: &GpuBindGroupLayoutEntry, val: &GpuSamplerBindingLayout); + + #[doc = "Get the `storageTexture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuStorageTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "storageTexture")] + pub fn get_storage_texture( + this: &GpuBindGroupLayoutEntry, + ) -> Option; + + #[doc = "Change the `storageTexture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuStorageTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "storageTexture")] + pub fn set_storage_texture( + this: &GpuBindGroupLayoutEntry, + val: &GpuStorageTextureBindingLayout, + ); + + #[doc = "Get the `texture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "texture")] + pub fn get_texture(this: &GpuBindGroupLayoutEntry) -> Option; + + #[doc = "Change the `texture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`, `GpuTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "texture")] + pub fn set_texture(this: &GpuBindGroupLayoutEntry, val: &GpuTextureBindingLayout); + + #[doc = "Get the `visibility` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "visibility")] + pub fn get_visibility(this: &GpuBindGroupLayoutEntry) -> u32; + + #[doc = "Change the `visibility` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "visibility")] + pub fn set_visibility(this: &GpuBindGroupLayoutEntry, val: u32); +} + +impl GpuBindGroupLayoutEntry { + #[doc = "Construct a new `GpuBindGroupLayoutEntry`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayoutEntry`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(binding: u32, visibility: u32) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_binding(binding); + ret.set_visibility(visibility); + ret + } + + #[deprecated = "Use `set_binding()` instead."] + pub fn binding(&mut self, val: u32) -> &mut Self { + self.set_binding(val); + self + } + + #[deprecated = "Use `set_buffer()` instead."] + pub fn buffer(&mut self, val: &GpuBufferBindingLayout) -> &mut Self { + self.set_buffer(val); + self + } + + #[deprecated = "Use `set_external_texture()` instead."] + pub fn external_texture(&mut self, val: &GpuExternalTextureBindingLayout) -> &mut Self { + self.set_external_texture(val); + self + } + + #[deprecated = "Use `set_sampler()` instead."] + pub fn sampler(&mut self, val: &GpuSamplerBindingLayout) -> &mut Self { + self.set_sampler(val); + self + } + + #[deprecated = "Use `set_storage_texture()` instead."] + pub fn storage_texture(&mut self, val: &GpuStorageTextureBindingLayout) -> &mut Self { + self.set_storage_texture(val); + self + } + + #[deprecated = "Use `set_texture()` instead."] + pub fn texture(&mut self, val: &GpuTextureBindingLayout) -> &mut Self { + self.set_texture(val); + self + } + + #[deprecated = "Use `set_visibility()` instead."] + pub fn visibility(&mut self, val: u32) -> &mut Self { + self.set_visibility(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendComponent.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendComponent.rs new file mode 100644 index 00000000..0f11495e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendComponent.rs @@ -0,0 +1,130 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBlendComponent)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBlendComponent` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBlendComponent; + + #[doc = "Get the `dstFactor` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendFactor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "dstFactor")] + pub fn get_dst_factor(this: &GpuBlendComponent) -> Option; + + #[doc = "Change the `dstFactor` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendFactor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "dstFactor")] + pub fn set_dst_factor(this: &GpuBlendComponent, val: GpuBlendFactor); + + #[doc = "Get the `operation` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendOperation`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "operation")] + pub fn get_operation(this: &GpuBlendComponent) -> Option; + + #[doc = "Change the `operation` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendOperation`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "operation")] + pub fn set_operation(this: &GpuBlendComponent, val: GpuBlendOperation); + + #[doc = "Get the `srcFactor` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendFactor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "srcFactor")] + pub fn get_src_factor(this: &GpuBlendComponent) -> Option; + + #[doc = "Change the `srcFactor` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendFactor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "srcFactor")] + pub fn set_src_factor(this: &GpuBlendComponent, val: GpuBlendFactor); +} + +impl GpuBlendComponent { + #[doc = "Construct a new `GpuBlendComponent`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_dst_factor()` instead."] + pub fn dst_factor(&mut self, val: GpuBlendFactor) -> &mut Self { + self.set_dst_factor(val); + self + } + + #[deprecated = "Use `set_operation()` instead."] + pub fn operation(&mut self, val: GpuBlendOperation) -> &mut Self { + self.set_operation(val); + self + } + + #[deprecated = "Use `set_src_factor()` instead."] + pub fn src_factor(&mut self, val: GpuBlendFactor) -> &mut Self { + self.set_src_factor(val); + self + } +} + +impl Default for GpuBlendComponent { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendFactor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendFactor.rs new file mode 100644 index 00000000..0b1f7c39 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendFactor.rs @@ -0,0 +1,52 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuBlendFactor` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuBlendFactor`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuBlendFactor { + Zero = "zero", + One = "one", + Src = "src", + OneMinusSrc = "one-minus-src", + SrcAlpha = "src-alpha", + OneMinusSrcAlpha = "one-minus-src-alpha", + Dst = "dst", + OneMinusDst = "one-minus-dst", + DstAlpha = "dst-alpha", + OneMinusDstAlpha = "one-minus-dst-alpha", + SrcAlphaSaturated = "src-alpha-saturated", + Constant = "constant", + OneMinusConstant = "one-minus-constant", + Src1 = "src1", + OneMinusSrc1 = "one-minus-src1", + Src1Alpha = "src1-alpha", + OneMinusSrc1Alpha = "one-minus-src1-alpha", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendOperation.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendOperation.rs new file mode 100644 index 00000000..13da1a1b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendOperation.rs @@ -0,0 +1,40 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuBlendOperation` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuBlendOperation`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuBlendOperation { + Add = "add", + Subtract = "subtract", + ReverseSubtract = "reverse-subtract", + Min = "min", + Max = "max", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendState.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendState.rs new file mode 100644 index 00000000..8c54586f --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBlendState.rs @@ -0,0 +1,102 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBlendState)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBlendState` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBlendState; + + #[doc = "Get the `alpha` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "alpha")] + pub fn get_alpha(this: &GpuBlendState) -> GpuBlendComponent; + + #[doc = "Change the `alpha` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "alpha")] + pub fn set_alpha(this: &GpuBlendState, val: &GpuBlendComponent); + + #[doc = "Get the `color` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "color")] + pub fn get_color(this: &GpuBlendState) -> GpuBlendComponent; + + #[doc = "Change the `color` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "color")] + pub fn set_color(this: &GpuBlendState, val: &GpuBlendComponent); +} + +impl GpuBlendState { + #[doc = "Construct a new `GpuBlendState`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendComponent`, `GpuBlendState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(alpha: &GpuBlendComponent, color: &GpuBlendComponent) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_alpha(alpha); + ret.set_color(color); + ret + } + + #[deprecated = "Use `set_alpha()` instead."] + pub fn alpha(&mut self, val: &GpuBlendComponent) -> &mut Self { + self.set_alpha(val); + self + } + + #[deprecated = "Use `set_color()` instead."] + pub fn color(&mut self, val: &GpuBlendComponent) -> &mut Self { + self.set_color(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBuffer.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBuffer.rs new file mode 100644 index 00000000..05f60aa5 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBuffer.rs @@ -0,0 +1,313 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBuffer , typescript_type = "GPUBuffer")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBuffer` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBuffer; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUBuffer" , js_name = size)] + #[doc = "Getter for the `size` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/size)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn size(this: &GpuBuffer) -> f64; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUBuffer" , js_name = usage)] + #[doc = "Getter for the `usage` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/usage)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn usage(this: &GpuBuffer) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUBuffer" , js_name = mapState)] + #[doc = "Getter for the `mapState` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/mapState)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuBufferMapState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn map_state(this: &GpuBuffer) -> GpuBufferMapState; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUBuffer" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuBuffer) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUBuffer" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuBuffer, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPUBuffer" , js_name = destroy)] + #[doc = "The `destroy()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/destroy)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn destroy(this: &GpuBuffer); + + # [wasm_bindgen (catch , method , structural , js_class = "GPUBuffer" , js_name = getMappedRange)] + #[doc = "The `getMappedRange()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/getMappedRange)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_mapped_range(this: &GpuBuffer) -> Result<::js_sys::ArrayBuffer, JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUBuffer" , js_name = getMappedRange)] + #[doc = "The `getMappedRange()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/getMappedRange)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_mapped_range_with_u32( + this: &GpuBuffer, + offset: u32, + ) -> Result<::js_sys::ArrayBuffer, JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUBuffer" , js_name = getMappedRange)] + #[doc = "The `getMappedRange()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/getMappedRange)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_mapped_range_with_f64( + this: &GpuBuffer, + offset: f64, + ) -> Result<::js_sys::ArrayBuffer, JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUBuffer" , js_name = getMappedRange)] + #[doc = "The `getMappedRange()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/getMappedRange)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_mapped_range_with_u32_and_u32( + this: &GpuBuffer, + offset: u32, + size: u32, + ) -> Result<::js_sys::ArrayBuffer, JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUBuffer" , js_name = getMappedRange)] + #[doc = "The `getMappedRange()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/getMappedRange)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_mapped_range_with_f64_and_u32( + this: &GpuBuffer, + offset: f64, + size: u32, + ) -> Result<::js_sys::ArrayBuffer, JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUBuffer" , js_name = getMappedRange)] + #[doc = "The `getMappedRange()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/getMappedRange)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_mapped_range_with_u32_and_f64( + this: &GpuBuffer, + offset: u32, + size: f64, + ) -> Result<::js_sys::ArrayBuffer, JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUBuffer" , js_name = getMappedRange)] + #[doc = "The `getMappedRange()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/getMappedRange)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_mapped_range_with_f64_and_f64( + this: &GpuBuffer, + offset: f64, + size: f64, + ) -> Result<::js_sys::ArrayBuffer, JsValue>; + + # [wasm_bindgen (method , structural , js_class = "GPUBuffer" , js_name = mapAsync)] + #[doc = "The `mapAsync()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/mapAsync)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn map_async(this: &GpuBuffer, mode: u32) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUBuffer" , js_name = mapAsync)] + #[doc = "The `mapAsync()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/mapAsync)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn map_async_with_u32(this: &GpuBuffer, mode: u32, offset: u32) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUBuffer" , js_name = mapAsync)] + #[doc = "The `mapAsync()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/mapAsync)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn map_async_with_f64(this: &GpuBuffer, mode: u32, offset: f64) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUBuffer" , js_name = mapAsync)] + #[doc = "The `mapAsync()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/mapAsync)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn map_async_with_u32_and_u32( + this: &GpuBuffer, + mode: u32, + offset: u32, + size: u32, + ) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUBuffer" , js_name = mapAsync)] + #[doc = "The `mapAsync()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/mapAsync)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn map_async_with_f64_and_u32( + this: &GpuBuffer, + mode: u32, + offset: f64, + size: u32, + ) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUBuffer" , js_name = mapAsync)] + #[doc = "The `mapAsync()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/mapAsync)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn map_async_with_u32_and_f64( + this: &GpuBuffer, + mode: u32, + offset: u32, + size: f64, + ) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUBuffer" , js_name = mapAsync)] + #[doc = "The `mapAsync()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/mapAsync)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn map_async_with_f64_and_f64( + this: &GpuBuffer, + mode: u32, + offset: f64, + size: f64, + ) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUBuffer" , js_name = unmap)] + #[doc = "The `unmap()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUBuffer/unmap)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn unmap(this: &GpuBuffer); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBinding.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBinding.rs new file mode 100644 index 00000000..5174557c --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBinding.rs @@ -0,0 +1,125 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBufferBinding)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBufferBinding` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBinding`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBufferBinding; + + #[doc = "Get the `buffer` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuBufferBinding`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "buffer")] + pub fn get_buffer(this: &GpuBufferBinding) -> GpuBuffer; + + #[doc = "Change the `buffer` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuBufferBinding`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "buffer")] + pub fn set_buffer(this: &GpuBufferBinding, val: &GpuBuffer); + + #[doc = "Get the `offset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBinding`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "offset")] + pub fn get_offset(this: &GpuBufferBinding) -> Option; + + #[doc = "Change the `offset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBinding`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "offset")] + pub fn set_offset(this: &GpuBufferBinding, val: f64); + + #[doc = "Get the `size` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBinding`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "size")] + pub fn get_size(this: &GpuBufferBinding) -> Option; + + #[doc = "Change the `size` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBinding`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "size")] + pub fn set_size(this: &GpuBufferBinding, val: f64); +} + +impl GpuBufferBinding { + #[doc = "Construct a new `GpuBufferBinding`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuBufferBinding`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(buffer: &GpuBuffer) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_buffer(buffer); + ret + } + + #[deprecated = "Use `set_buffer()` instead."] + pub fn buffer(&mut self, val: &GpuBuffer) -> &mut Self { + self.set_buffer(val); + self + } + + #[deprecated = "Use `set_offset()` instead."] + pub fn offset(&mut self, val: f64) -> &mut Self { + self.set_offset(val); + self + } + + #[deprecated = "Use `set_size()` instead."] + pub fn size(&mut self, val: f64) -> &mut Self { + self.set_size(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBindingLayout.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBindingLayout.rs new file mode 100644 index 00000000..311d42bc --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBindingLayout.rs @@ -0,0 +1,130 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBufferBindingLayout)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBufferBindingLayout` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBufferBindingLayout; + + #[doc = "Get the `hasDynamicOffset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "hasDynamicOffset")] + pub fn get_has_dynamic_offset(this: &GpuBufferBindingLayout) -> Option; + + #[doc = "Change the `hasDynamicOffset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "hasDynamicOffset")] + pub fn set_has_dynamic_offset(this: &GpuBufferBindingLayout, val: bool); + + #[doc = "Get the `minBindingSize` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "minBindingSize")] + pub fn get_min_binding_size(this: &GpuBufferBindingLayout) -> Option; + + #[doc = "Change the `minBindingSize` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "minBindingSize")] + pub fn set_min_binding_size(this: &GpuBufferBindingLayout, val: f64); + + #[doc = "Get the `type` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBindingLayout`, `GpuBufferBindingType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "type")] + pub fn get_type(this: &GpuBufferBindingLayout) -> Option; + + #[doc = "Change the `type` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBindingLayout`, `GpuBufferBindingType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "type")] + pub fn set_type(this: &GpuBufferBindingLayout, val: GpuBufferBindingType); +} + +impl GpuBufferBindingLayout { + #[doc = "Construct a new `GpuBufferBindingLayout`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_has_dynamic_offset()` instead."] + pub fn has_dynamic_offset(&mut self, val: bool) -> &mut Self { + self.set_has_dynamic_offset(val); + self + } + + #[deprecated = "Use `set_min_binding_size()` instead."] + pub fn min_binding_size(&mut self, val: f64) -> &mut Self { + self.set_min_binding_size(val); + self + } + + #[deprecated = "Use `set_type()` instead."] + pub fn type_(&mut self, val: GpuBufferBindingType) -> &mut Self { + self.set_type(val); + self + } +} + +impl Default for GpuBufferBindingLayout { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBindingType.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBindingType.rs new file mode 100644 index 00000000..7b1a6478 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferBindingType.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuBufferBindingType` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuBufferBindingType`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuBufferBindingType { + Uniform = "uniform", + Storage = "storage", + ReadOnlyStorage = "read-only-storage", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferDescriptor.rs new file mode 100644 index 00000000..0303470c --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferDescriptor.rs @@ -0,0 +1,150 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUBufferDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuBufferDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuBufferDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuBufferDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuBufferDescriptor, val: &str); + + #[doc = "Get the `mappedAtCreation` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "mappedAtCreation")] + pub fn get_mapped_at_creation(this: &GpuBufferDescriptor) -> Option; + + #[doc = "Change the `mappedAtCreation` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "mappedAtCreation")] + pub fn set_mapped_at_creation(this: &GpuBufferDescriptor, val: bool); + + #[doc = "Get the `size` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "size")] + pub fn get_size(this: &GpuBufferDescriptor) -> f64; + + #[doc = "Change the `size` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "size")] + pub fn set_size(this: &GpuBufferDescriptor, val: f64); + + #[doc = "Get the `usage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "usage")] + pub fn get_usage(this: &GpuBufferDescriptor) -> u32; + + #[doc = "Change the `usage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "usage")] + pub fn set_usage(this: &GpuBufferDescriptor, val: u32); +} + +impl GpuBufferDescriptor { + #[doc = "Construct a new `GpuBufferDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(size: f64, usage: u32) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_size(size); + ret.set_usage(usage); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_mapped_at_creation()` instead."] + pub fn mapped_at_creation(&mut self, val: bool) -> &mut Self { + self.set_mapped_at_creation(val); + self + } + + #[deprecated = "Use `set_size()` instead."] + pub fn size(&mut self, val: f64) -> &mut Self { + self.set_size(val); + self + } + + #[deprecated = "Use `set_usage()` instead."] + pub fn usage(&mut self, val: u32) -> &mut Self { + self.set_usage(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferMapState.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferMapState.rs new file mode 100644 index 00000000..c44b6ec4 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuBufferMapState.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuBufferMapState` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuBufferMapState`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuBufferMapState { + Unmapped = "unmapped", + Pending = "pending", + Mapped = "mapped", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasAlphaMode.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasAlphaMode.rs new file mode 100644 index 00000000..0ec4ac45 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasAlphaMode.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuCanvasAlphaMode` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuCanvasAlphaMode`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuCanvasAlphaMode { + Opaque = "opaque", + Premultiplied = "premultiplied", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasConfiguration.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasConfiguration.rs new file mode 100644 index 00000000..469376af --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasConfiguration.rs @@ -0,0 +1,198 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCanvasConfiguration)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCanvasConfiguration` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCanvasConfiguration; + + #[doc = "Get the `alphaMode` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasAlphaMode`, `GpuCanvasConfiguration`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "alphaMode")] + pub fn get_alpha_mode(this: &GpuCanvasConfiguration) -> Option; + + #[doc = "Change the `alphaMode` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasAlphaMode`, `GpuCanvasConfiguration`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "alphaMode")] + pub fn set_alpha_mode(this: &GpuCanvasConfiguration, val: GpuCanvasAlphaMode); + + #[doc = "Get the `device` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "device")] + pub fn get_device(this: &GpuCanvasConfiguration) -> GpuDevice; + + #[doc = "Change the `device` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "device")] + pub fn set_device(this: &GpuCanvasConfiguration, val: &GpuDevice); + + #[doc = "Get the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "format")] + pub fn get_format(this: &GpuCanvasConfiguration) -> GpuTextureFormat; + + #[doc = "Change the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "format")] + pub fn set_format(this: &GpuCanvasConfiguration, val: GpuTextureFormat); + + #[doc = "Get the `toneMapping` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`, `GpuCanvasToneMapping`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "toneMapping")] + pub fn get_tone_mapping(this: &GpuCanvasConfiguration) -> Option; + + #[doc = "Change the `toneMapping` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`, `GpuCanvasToneMapping`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "toneMapping")] + pub fn set_tone_mapping(this: &GpuCanvasConfiguration, val: &GpuCanvasToneMapping); + + #[doc = "Get the `usage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "usage")] + pub fn get_usage(this: &GpuCanvasConfiguration) -> Option; + + #[doc = "Change the `usage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "usage")] + pub fn set_usage(this: &GpuCanvasConfiguration, val: u32); + + #[doc = "Get the `viewFormats` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "viewFormats")] + pub fn get_view_formats(this: &GpuCanvasConfiguration) -> Option<::js_sys::Array>; + + #[doc = "Change the `viewFormats` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "viewFormats")] + pub fn set_view_formats(this: &GpuCanvasConfiguration, val: &::wasm_bindgen::JsValue); +} + +impl GpuCanvasConfiguration { + #[doc = "Construct a new `GpuCanvasConfiguration`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`, `GpuDevice`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(device: &GpuDevice, format: GpuTextureFormat) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_device(device); + ret.set_format(format); + ret + } + + #[deprecated = "Use `set_alpha_mode()` instead."] + pub fn alpha_mode(&mut self, val: GpuCanvasAlphaMode) -> &mut Self { + self.set_alpha_mode(val); + self + } + + #[deprecated = "Use `set_device()` instead."] + pub fn device(&mut self, val: &GpuDevice) -> &mut Self { + self.set_device(val); + self + } + + #[deprecated = "Use `set_format()` instead."] + pub fn format(&mut self, val: GpuTextureFormat) -> &mut Self { + self.set_format(val); + self + } + + #[deprecated = "Use `set_tone_mapping()` instead."] + pub fn tone_mapping(&mut self, val: &GpuCanvasToneMapping) -> &mut Self { + self.set_tone_mapping(val); + self + } + + #[deprecated = "Use `set_usage()` instead."] + pub fn usage(&mut self, val: u32) -> &mut Self { + self.set_usage(val); + self + } + + #[deprecated = "Use `set_view_formats()` instead."] + pub fn view_formats(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_view_formats(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasContext.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasContext.rs new file mode 100644 index 00000000..5504b3bd --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasContext.rs @@ -0,0 +1,98 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCanvasContext , typescript_type = "GPUCanvasContext")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCanvasContext` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCanvasContext)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasContext`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCanvasContext; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCanvasContext" , js_name = canvas)] + #[doc = "Getter for the `canvas` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCanvasContext/canvas)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasContext`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn canvas(this: &GpuCanvasContext) -> ::js_sys::Object; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCanvasContext" , js_name = configure)] + #[doc = "The `configure()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCanvasContext/configure)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`, `GpuCanvasContext`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn configure( + this: &GpuCanvasContext, + configuration: &GpuCanvasConfiguration, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (method , structural , js_class = "GPUCanvasContext" , js_name = getConfiguration)] + #[doc = "The `getConfiguration()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCanvasContext/getConfiguration)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasConfiguration`, `GpuCanvasContext`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_configuration(this: &GpuCanvasContext) -> Option; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCanvasContext" , js_name = getCurrentTexture)] + #[doc = "The `getCurrentTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCanvasContext/getCurrentTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasContext`, `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_current_texture(this: &GpuCanvasContext) -> Result; + + # [wasm_bindgen (method , structural , js_class = "GPUCanvasContext" , js_name = unconfigure)] + #[doc = "The `unconfigure()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCanvasContext/unconfigure)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasContext`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn unconfigure(this: &GpuCanvasContext); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasToneMapping.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasToneMapping.rs new file mode 100644 index 00000000..8b528f73 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasToneMapping.rs @@ -0,0 +1,82 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCanvasToneMapping)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCanvasToneMapping` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasToneMapping`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCanvasToneMapping; + + #[doc = "Get the `mode` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasToneMapping`, `GpuCanvasToneMappingMode`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "mode")] + pub fn get_mode(this: &GpuCanvasToneMapping) -> Option; + + #[doc = "Change the `mode` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasToneMapping`, `GpuCanvasToneMappingMode`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "mode")] + pub fn set_mode(this: &GpuCanvasToneMapping, val: GpuCanvasToneMappingMode); +} + +impl GpuCanvasToneMapping { + #[doc = "Construct a new `GpuCanvasToneMapping`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCanvasToneMapping`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_mode()` instead."] + pub fn mode(&mut self, val: GpuCanvasToneMappingMode) -> &mut Self { + self.set_mode(val); + self + } +} + +impl Default for GpuCanvasToneMapping { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasToneMappingMode.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasToneMappingMode.rs new file mode 100644 index 00000000..e601a631 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCanvasToneMappingMode.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuCanvasToneMappingMode` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuCanvasToneMappingMode`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuCanvasToneMappingMode { + Standard = "standard", + Extended = "extended", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuColorDict.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuColorDict.rs new file mode 100644 index 00000000..eb02d961 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuColorDict.rs @@ -0,0 +1,152 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUColorDict)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuColorDict` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuColorDict; + + #[doc = "Get the `a` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "a")] + pub fn get_a(this: &GpuColorDict) -> f64; + + #[doc = "Change the `a` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "a")] + pub fn set_a(this: &GpuColorDict, val: f64); + + #[doc = "Get the `b` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "b")] + pub fn get_b(this: &GpuColorDict) -> f64; + + #[doc = "Change the `b` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "b")] + pub fn set_b(this: &GpuColorDict, val: f64); + + #[doc = "Get the `g` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "g")] + pub fn get_g(this: &GpuColorDict) -> f64; + + #[doc = "Change the `g` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "g")] + pub fn set_g(this: &GpuColorDict, val: f64); + + #[doc = "Get the `r` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "r")] + pub fn get_r(this: &GpuColorDict) -> f64; + + #[doc = "Change the `r` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "r")] + pub fn set_r(this: &GpuColorDict, val: f64); +} + +impl GpuColorDict { + #[doc = "Construct a new `GpuColorDict`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(a: f64, b: f64, g: f64, r: f64) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_a(a); + ret.set_b(b); + ret.set_g(g); + ret.set_r(r); + ret + } + + #[deprecated = "Use `set_a()` instead."] + pub fn a(&mut self, val: f64) -> &mut Self { + self.set_a(val); + self + } + + #[deprecated = "Use `set_b()` instead."] + pub fn b(&mut self, val: f64) -> &mut Self { + self.set_b(val); + self + } + + #[deprecated = "Use `set_g()` instead."] + pub fn g(&mut self, val: f64) -> &mut Self { + self.set_g(val); + self + } + + #[deprecated = "Use `set_r()` instead."] + pub fn r(&mut self, val: f64) -> &mut Self { + self.set_r(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuColorTargetState.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuColorTargetState.rs new file mode 100644 index 00000000..4f92ba30 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuColorTargetState.rs @@ -0,0 +1,125 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUColorTargetState)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuColorTargetState` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorTargetState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuColorTargetState; + + #[doc = "Get the `blend` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendState`, `GpuColorTargetState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "blend")] + pub fn get_blend(this: &GpuColorTargetState) -> Option; + + #[doc = "Change the `blend` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBlendState`, `GpuColorTargetState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "blend")] + pub fn set_blend(this: &GpuColorTargetState, val: &GpuBlendState); + + #[doc = "Get the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorTargetState`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "format")] + pub fn get_format(this: &GpuColorTargetState) -> GpuTextureFormat; + + #[doc = "Change the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorTargetState`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "format")] + pub fn set_format(this: &GpuColorTargetState, val: GpuTextureFormat); + + #[doc = "Get the `writeMask` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorTargetState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "writeMask")] + pub fn get_write_mask(this: &GpuColorTargetState) -> Option; + + #[doc = "Change the `writeMask` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorTargetState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "writeMask")] + pub fn set_write_mask(this: &GpuColorTargetState, val: u32); +} + +impl GpuColorTargetState { + #[doc = "Construct a new `GpuColorTargetState`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorTargetState`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(format: GpuTextureFormat) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_format(format); + ret + } + + #[deprecated = "Use `set_blend()` instead."] + pub fn blend(&mut self, val: &GpuBlendState) -> &mut Self { + self.set_blend(val); + self + } + + #[deprecated = "Use `set_format()` instead."] + pub fn format(&mut self, val: GpuTextureFormat) -> &mut Self { + self.set_format(val); + self + } + + #[deprecated = "Use `set_write_mask()` instead."] + pub fn write_mask(&mut self, val: u32) -> &mut Self { + self.set_write_mask(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandBuffer.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandBuffer.rs new file mode 100644 index 00000000..44e47a23 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandBuffer.rs @@ -0,0 +1,62 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCommandBuffer , typescript_type = "GPUCommandBuffer")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCommandBuffer` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCommandBuffer; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCommandBuffer" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandBuffer/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuCommandBuffer) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUCommandBuffer" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandBuffer/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandBuffer`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuCommandBuffer, value: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandBufferDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandBufferDescriptor.rs new file mode 100644 index 00000000..c571b408 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandBufferDescriptor.rs @@ -0,0 +1,82 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCommandBufferDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCommandBufferDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCommandBufferDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuCommandBufferDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuCommandBufferDescriptor, val: &str); +} + +impl GpuCommandBufferDescriptor { + #[doc = "Construct a new `GpuCommandBufferDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandBufferDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } +} + +impl Default for GpuCommandBufferDescriptor { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandEncoder.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandEncoder.rs new file mode 100644 index 00000000..2116f8a1 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandEncoder.rs @@ -0,0 +1,600 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCommandEncoder , typescript_type = "GPUCommandEncoder")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCommandEncoder` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCommandEncoder; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCommandEncoder" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuCommandEncoder) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUCommandEncoder" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuCommandEncoder, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = beginComputePass)] + #[doc = "The `beginComputePass()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/beginComputePass)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn begin_compute_pass(this: &GpuCommandEncoder) -> GpuComputePassEncoder; + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = beginComputePass)] + #[doc = "The `beginComputePass()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/beginComputePass)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuComputePassDescriptor`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn begin_compute_pass_with_descriptor( + this: &GpuCommandEncoder, + descriptor: &GpuComputePassDescriptor, + ) -> GpuComputePassEncoder; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = beginRenderPass)] + #[doc = "The `beginRenderPass()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/beginRenderPass)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuRenderPassDescriptor`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn begin_render_pass( + this: &GpuCommandEncoder, + descriptor: &GpuRenderPassDescriptor, + ) -> Result; + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = clearBuffer)] + #[doc = "The `clearBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/clearBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn clear_buffer(this: &GpuCommandEncoder, buffer: &GpuBuffer); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = clearBuffer)] + #[doc = "The `clearBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/clearBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn clear_buffer_with_u32(this: &GpuCommandEncoder, buffer: &GpuBuffer, offset: u32); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = clearBuffer)] + #[doc = "The `clearBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/clearBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn clear_buffer_with_f64(this: &GpuCommandEncoder, buffer: &GpuBuffer, offset: f64); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = clearBuffer)] + #[doc = "The `clearBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/clearBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn clear_buffer_with_u32_and_u32( + this: &GpuCommandEncoder, + buffer: &GpuBuffer, + offset: u32, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = clearBuffer)] + #[doc = "The `clearBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/clearBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn clear_buffer_with_f64_and_u32( + this: &GpuCommandEncoder, + buffer: &GpuBuffer, + offset: f64, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = clearBuffer)] + #[doc = "The `clearBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/clearBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn clear_buffer_with_u32_and_f64( + this: &GpuCommandEncoder, + buffer: &GpuBuffer, + offset: u32, + size: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = clearBuffer)] + #[doc = "The `clearBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/clearBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn clear_buffer_with_f64_and_f64( + this: &GpuCommandEncoder, + buffer: &GpuBuffer, + offset: f64, + size: f64, + ); + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_u32_and_u32( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: u32, + destination: &GpuBuffer, + destination_offset: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_f64_and_u32( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: f64, + destination: &GpuBuffer, + destination_offset: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_u32_and_f64( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: u32, + destination: &GpuBuffer, + destination_offset: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_f64_and_f64( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: f64, + destination: &GpuBuffer, + destination_offset: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_u32_and_u32_and_u32( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: u32, + destination: &GpuBuffer, + destination_offset: u32, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_f64_and_u32_and_u32( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: f64, + destination: &GpuBuffer, + destination_offset: u32, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_u32_and_f64_and_u32( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: u32, + destination: &GpuBuffer, + destination_offset: f64, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_f64_and_f64_and_u32( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: f64, + destination: &GpuBuffer, + destination_offset: f64, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_u32_and_u32_and_f64( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: u32, + destination: &GpuBuffer, + destination_offset: u32, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_f64_and_u32_and_f64( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: f64, + destination: &GpuBuffer, + destination_offset: u32, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_u32_and_f64_and_f64( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: u32, + destination: &GpuBuffer, + destination_offset: f64, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToBuffer)] + #[doc = "The `copyBufferToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_buffer_with_f64_and_f64_and_f64( + this: &GpuCommandEncoder, + source: &GpuBuffer, + source_offset: f64, + destination: &GpuBuffer, + destination_offset: f64, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToTexture)] + #[doc = "The `copyBufferToTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuTexelCopyBufferInfo`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_texture_with_u32_sequence( + this: &GpuCommandEncoder, + source: &GpuTexelCopyBufferInfo, + destination: &GpuTexelCopyTextureInfo, + copy_size: &::wasm_bindgen::JsValue, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyBufferToTexture)] + #[doc = "The `copyBufferToTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyBufferToTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuExtent3dDict`, `GpuTexelCopyBufferInfo`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_buffer_to_texture_with_gpu_extent_3d_dict( + this: &GpuCommandEncoder, + source: &GpuTexelCopyBufferInfo, + destination: &GpuTexelCopyTextureInfo, + copy_size: &GpuExtent3dDict, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyTextureToBuffer)] + #[doc = "The `copyTextureToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyTextureToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuTexelCopyBufferInfo`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_texture_to_buffer_with_u32_sequence( + this: &GpuCommandEncoder, + source: &GpuTexelCopyTextureInfo, + destination: &GpuTexelCopyBufferInfo, + copy_size: &::wasm_bindgen::JsValue, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyTextureToBuffer)] + #[doc = "The `copyTextureToBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyTextureToBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuExtent3dDict`, `GpuTexelCopyBufferInfo`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_texture_to_buffer_with_gpu_extent_3d_dict( + this: &GpuCommandEncoder, + source: &GpuTexelCopyTextureInfo, + destination: &GpuTexelCopyBufferInfo, + copy_size: &GpuExtent3dDict, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyTextureToTexture)] + #[doc = "The `copyTextureToTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyTextureToTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_texture_to_texture_with_u32_sequence( + this: &GpuCommandEncoder, + source: &GpuTexelCopyTextureInfo, + destination: &GpuTexelCopyTextureInfo, + copy_size: &::wasm_bindgen::JsValue, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUCommandEncoder" , js_name = copyTextureToTexture)] + #[doc = "The `copyTextureToTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/copyTextureToTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuExtent3dDict`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_texture_to_texture_with_gpu_extent_3d_dict( + this: &GpuCommandEncoder, + source: &GpuTexelCopyTextureInfo, + destination: &GpuTexelCopyTextureInfo, + copy_size: &GpuExtent3dDict, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = finish)] + #[doc = "The `finish()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/finish)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandBuffer`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn finish(this: &GpuCommandEncoder) -> GpuCommandBuffer; + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = finish)] + #[doc = "The `finish()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/finish)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandBuffer`, `GpuCommandBufferDescriptor`, `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn finish_with_descriptor( + this: &GpuCommandEncoder, + descriptor: &GpuCommandBufferDescriptor, + ) -> GpuCommandBuffer; + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = resolveQuerySet)] + #[doc = "The `resolveQuerySet()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/resolveQuerySet)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`, `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn resolve_query_set_with_u32( + this: &GpuCommandEncoder, + query_set: &GpuQuerySet, + first_query: u32, + query_count: u32, + destination: &GpuBuffer, + destination_offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = resolveQuerySet)] + #[doc = "The `resolveQuerySet()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/resolveQuerySet)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuCommandEncoder`, `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn resolve_query_set_with_f64( + this: &GpuCommandEncoder, + query_set: &GpuQuerySet, + first_query: u32, + query_count: u32, + destination: &GpuBuffer, + destination_offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = insertDebugMarker)] + #[doc = "The `insertDebugMarker()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/insertDebugMarker)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn insert_debug_marker(this: &GpuCommandEncoder, marker_label: &str); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = popDebugGroup)] + #[doc = "The `popDebugGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/popDebugGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn pop_debug_group(this: &GpuCommandEncoder); + + # [wasm_bindgen (method , structural , js_class = "GPUCommandEncoder" , js_name = pushDebugGroup)] + #[doc = "The `pushDebugGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCommandEncoder/pushDebugGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn push_debug_group(this: &GpuCommandEncoder, group_label: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandEncoderDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandEncoderDescriptor.rs new file mode 100644 index 00000000..64d08c43 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCommandEncoderDescriptor.rs @@ -0,0 +1,82 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCommandEncoderDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCommandEncoderDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCommandEncoderDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuCommandEncoderDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuCommandEncoderDescriptor, val: &str); +} + +impl GpuCommandEncoderDescriptor { + #[doc = "Construct a new `GpuCommandEncoderDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } +} + +impl Default for GpuCommandEncoderDescriptor { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompareFunction.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompareFunction.rs new file mode 100644 index 00000000..bd7008fe --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompareFunction.rs @@ -0,0 +1,43 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuCompareFunction` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuCompareFunction`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuCompareFunction { + Never = "never", + Less = "less", + Equal = "equal", + LessEqual = "less-equal", + Greater = "greater", + NotEqual = "not-equal", + GreaterEqual = "greater-equal", + Always = "always", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationInfo.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationInfo.rs new file mode 100644 index 00000000..029797d1 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationInfo.rs @@ -0,0 +1,51 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCompilationInfo , typescript_type = "GPUCompilationInfo")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCompilationInfo` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCompilationInfo)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompilationInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCompilationInfo; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCompilationInfo" , js_name = messages)] + #[doc = "Getter for the `messages` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCompilationInfo/messages)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompilationInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn messages(this: &GpuCompilationInfo) -> ::js_sys::Array; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationMessage.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationMessage.rs new file mode 100644 index 00000000..927af739 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationMessage.rs @@ -0,0 +1,106 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCompilationMessage , typescript_type = "GPUCompilationMessage")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCompilationMessage` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCompilationMessage)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompilationMessage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCompilationMessage; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCompilationMessage" , js_name = message)] + #[doc = "Getter for the `message` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCompilationMessage/message)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompilationMessage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn message(this: &GpuCompilationMessage) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCompilationMessage" , js_name = type)] + #[doc = "Getter for the `type` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCompilationMessage/type)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompilationMessage`, `GpuCompilationMessageType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn type_(this: &GpuCompilationMessage) -> GpuCompilationMessageType; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCompilationMessage" , js_name = lineNum)] + #[doc = "Getter for the `lineNum` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCompilationMessage/lineNum)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompilationMessage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn line_num(this: &GpuCompilationMessage) -> f64; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCompilationMessage" , js_name = linePos)] + #[doc = "Getter for the `linePos` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCompilationMessage/linePos)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompilationMessage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn line_pos(this: &GpuCompilationMessage) -> f64; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCompilationMessage" , js_name = offset)] + #[doc = "Getter for the `offset` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCompilationMessage/offset)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompilationMessage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn offset(this: &GpuCompilationMessage) -> f64; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUCompilationMessage" , js_name = length)] + #[doc = "Getter for the `length` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUCompilationMessage/length)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompilationMessage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn length(this: &GpuCompilationMessage) -> f64; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationMessageType.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationMessageType.rs new file mode 100644 index 00000000..6b98eb78 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCompilationMessageType.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuCompilationMessageType` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuCompilationMessageType`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuCompilationMessageType { + Error = "error", + Warning = "warning", + Info = "info", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassDescriptor.rs new file mode 100644 index 00000000..593a06f4 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassDescriptor.rs @@ -0,0 +1,111 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUComputePassDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuComputePassDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuComputePassDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuComputePassDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuComputePassDescriptor, val: &str); + + #[doc = "Get the `timestampWrites` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassDescriptor`, `GpuComputePassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "timestampWrites")] + pub fn get_timestamp_writes( + this: &GpuComputePassDescriptor, + ) -> Option; + + #[doc = "Change the `timestampWrites` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassDescriptor`, `GpuComputePassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "timestampWrites")] + pub fn set_timestamp_writes( + this: &GpuComputePassDescriptor, + val: &GpuComputePassTimestampWrites, + ); +} + +impl GpuComputePassDescriptor { + #[doc = "Construct a new `GpuComputePassDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_timestamp_writes()` instead."] + pub fn timestamp_writes(&mut self, val: &GpuComputePassTimestampWrites) -> &mut Self { + self.set_timestamp_writes(val); + self + } +} + +impl Default for GpuComputePassDescriptor { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassEncoder.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassEncoder.rs new file mode 100644 index 00000000..1e57894e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassEncoder.rs @@ -0,0 +1,292 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUComputePassEncoder , typescript_type = "GPUComputePassEncoder")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuComputePassEncoder` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuComputePassEncoder; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUComputePassEncoder" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuComputePassEncoder) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUComputePassEncoder" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuComputePassEncoder, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = dispatchWorkgroups)] + #[doc = "The `dispatchWorkgroups()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/dispatchWorkgroups)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn dispatch_workgroups(this: &GpuComputePassEncoder, workgroup_count_x: u32); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = dispatchWorkgroups)] + #[doc = "The `dispatchWorkgroups()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/dispatchWorkgroups)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn dispatch_workgroups_with_workgroup_count_y( + this: &GpuComputePassEncoder, + workgroup_count_x: u32, + workgroup_count_y: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = dispatchWorkgroups)] + #[doc = "The `dispatchWorkgroups()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/dispatchWorkgroups)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn dispatch_workgroups_with_workgroup_count_y_and_workgroup_count_z( + this: &GpuComputePassEncoder, + workgroup_count_x: u32, + workgroup_count_y: u32, + workgroup_count_z: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = dispatchWorkgroupsIndirect)] + #[doc = "The `dispatchWorkgroupsIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/dispatchWorkgroupsIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn dispatch_workgroups_indirect_with_u32( + this: &GpuComputePassEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = dispatchWorkgroupsIndirect)] + #[doc = "The `dispatchWorkgroupsIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/dispatchWorkgroupsIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn dispatch_workgroups_indirect_with_f64( + this: &GpuComputePassEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = end)] + #[doc = "The `end()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/end)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn end(this: &GpuComputePassEncoder); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = setPipeline)] + #[doc = "The `setPipeline()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/setPipeline)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`, `GpuComputePipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_pipeline(this: &GpuComputePassEncoder, pipeline: &GpuComputePipeline); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group( + this: &GpuComputePassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + ); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_sequence( + this: &GpuComputePassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets: &::wasm_bindgen::JsValue, + ); + + # [wasm_bindgen (catch , method , structural , js_class = "GPUComputePassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_slice_and_u32_and_dynamic_offsets_data_length( + this: &GpuComputePassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &[u32], + dynamic_offsets_data_start: u32, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUComputePassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_array_and_u32_and_dynamic_offsets_data_length( + this: &GpuComputePassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &::js_sys::Uint32Array, + dynamic_offsets_data_start: u32, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUComputePassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_slice_and_f64_and_dynamic_offsets_data_length( + this: &GpuComputePassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &[u32], + dynamic_offsets_data_start: f64, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUComputePassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_array_and_f64_and_dynamic_offsets_data_length( + this: &GpuComputePassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &::js_sys::Uint32Array, + dynamic_offsets_data_start: f64, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = insertDebugMarker)] + #[doc = "The `insertDebugMarker()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/insertDebugMarker)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn insert_debug_marker(this: &GpuComputePassEncoder, marker_label: &str); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = popDebugGroup)] + #[doc = "The `popDebugGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/popDebugGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn pop_debug_group(this: &GpuComputePassEncoder); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePassEncoder" , js_name = pushDebugGroup)] + #[doc = "The `pushDebugGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePassEncoder/pushDebugGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn push_debug_group(this: &GpuComputePassEncoder, group_label: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassTimestampWrites.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassTimestampWrites.rs new file mode 100644 index 00000000..7d91fd2a --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePassTimestampWrites.rs @@ -0,0 +1,125 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUComputePassTimestampWrites)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuComputePassTimestampWrites` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuComputePassTimestampWrites; + + #[doc = "Get the `beginningOfPassWriteIndex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "beginningOfPassWriteIndex")] + pub fn get_beginning_of_pass_write_index(this: &GpuComputePassTimestampWrites) -> Option; + + #[doc = "Change the `beginningOfPassWriteIndex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "beginningOfPassWriteIndex")] + pub fn set_beginning_of_pass_write_index(this: &GpuComputePassTimestampWrites, val: u32); + + #[doc = "Get the `endOfPassWriteIndex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "endOfPassWriteIndex")] + pub fn get_end_of_pass_write_index(this: &GpuComputePassTimestampWrites) -> Option; + + #[doc = "Change the `endOfPassWriteIndex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "endOfPassWriteIndex")] + pub fn set_end_of_pass_write_index(this: &GpuComputePassTimestampWrites, val: u32); + + #[doc = "Get the `querySet` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassTimestampWrites`, `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "querySet")] + pub fn get_query_set(this: &GpuComputePassTimestampWrites) -> GpuQuerySet; + + #[doc = "Change the `querySet` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassTimestampWrites`, `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "querySet")] + pub fn set_query_set(this: &GpuComputePassTimestampWrites, val: &GpuQuerySet); +} + +impl GpuComputePassTimestampWrites { + #[doc = "Construct a new `GpuComputePassTimestampWrites`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePassTimestampWrites`, `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(query_set: &GpuQuerySet) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_query_set(query_set); + ret + } + + #[deprecated = "Use `set_beginning_of_pass_write_index()` instead."] + pub fn beginning_of_pass_write_index(&mut self, val: u32) -> &mut Self { + self.set_beginning_of_pass_write_index(val); + self + } + + #[deprecated = "Use `set_end_of_pass_write_index()` instead."] + pub fn end_of_pass_write_index(&mut self, val: u32) -> &mut Self { + self.set_end_of_pass_write_index(val); + self + } + + #[deprecated = "Use `set_query_set()` instead."] + pub fn query_set(&mut self, val: &GpuQuerySet) -> &mut Self { + self.set_query_set(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePipeline.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePipeline.rs new file mode 100644 index 00000000..c022652a --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePipeline.rs @@ -0,0 +1,73 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUComputePipeline , typescript_type = "GPUComputePipeline")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuComputePipeline` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePipeline)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuComputePipeline; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUComputePipeline" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePipeline/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuComputePipeline) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUComputePipeline" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePipeline/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuComputePipeline, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPUComputePipeline" , js_name = getBindGroupLayout)] + #[doc = "The `getBindGroupLayout()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUComputePipeline/getBindGroupLayout)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayout`, `GpuComputePipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_bind_group_layout(this: &GpuComputePipeline, index: u32) -> GpuBindGroupLayout; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePipelineDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePipelineDescriptor.rs new file mode 100644 index 00000000..2ca6458d --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuComputePipelineDescriptor.rs @@ -0,0 +1,126 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUComputePipelineDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuComputePipelineDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuComputePipelineDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuComputePipelineDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuComputePipelineDescriptor, val: &str); + + #[doc = "Get the `layout` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "layout")] + pub fn get_layout(this: &GpuComputePipelineDescriptor) -> ::wasm_bindgen::JsValue; + + #[doc = "Change the `layout` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "layout")] + pub fn set_layout(this: &GpuComputePipelineDescriptor, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `compute` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipelineDescriptor`, `GpuProgrammableStage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "compute")] + pub fn get_compute(this: &GpuComputePipelineDescriptor) -> GpuProgrammableStage; + + #[doc = "Change the `compute` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipelineDescriptor`, `GpuProgrammableStage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "compute")] + pub fn set_compute(this: &GpuComputePipelineDescriptor, val: &GpuProgrammableStage); +} + +impl GpuComputePipelineDescriptor { + #[doc = "Construct a new `GpuComputePipelineDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipelineDescriptor`, `GpuProgrammableStage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(layout: &::wasm_bindgen::JsValue, compute: &GpuProgrammableStage) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_layout(layout); + ret.set_compute(compute); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_layout()` instead."] + pub fn layout(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_layout(val); + self + } + + #[deprecated = "Use `set_compute()` instead."] + pub fn compute(&mut self, val: &GpuProgrammableStage) -> &mut Self { + self.set_compute(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCopyExternalImageDestInfo.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCopyExternalImageDestInfo.rs new file mode 100644 index 00000000..4b9eaa3f --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCopyExternalImageDestInfo.rs @@ -0,0 +1,173 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCopyExternalImageDestInfo)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCopyExternalImageDestInfo` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCopyExternalImageDestInfo; + + #[doc = "Get the `aspect` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`, `GpuTextureAspect`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "aspect")] + pub fn get_aspect(this: &GpuCopyExternalImageDestInfo) -> Option; + + #[doc = "Change the `aspect` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`, `GpuTextureAspect`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "aspect")] + pub fn set_aspect(this: &GpuCopyExternalImageDestInfo, val: GpuTextureAspect); + + #[doc = "Get the `mipLevel` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "mipLevel")] + pub fn get_mip_level(this: &GpuCopyExternalImageDestInfo) -> Option; + + #[doc = "Change the `mipLevel` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "mipLevel")] + pub fn set_mip_level(this: &GpuCopyExternalImageDestInfo, val: u32); + + #[doc = "Get the `origin` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "origin")] + pub fn get_origin(this: &GpuCopyExternalImageDestInfo) -> ::wasm_bindgen::JsValue; + + #[doc = "Change the `origin` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "origin")] + pub fn set_origin(this: &GpuCopyExternalImageDestInfo, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `texture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`, `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "texture")] + pub fn get_texture(this: &GpuCopyExternalImageDestInfo) -> GpuTexture; + + #[doc = "Change the `texture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`, `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "texture")] + pub fn set_texture(this: &GpuCopyExternalImageDestInfo, val: &GpuTexture); + + #[doc = "Get the `premultipliedAlpha` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "premultipliedAlpha")] + pub fn get_premultiplied_alpha(this: &GpuCopyExternalImageDestInfo) -> Option; + + #[doc = "Change the `premultipliedAlpha` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "premultipliedAlpha")] + pub fn set_premultiplied_alpha(this: &GpuCopyExternalImageDestInfo, val: bool); +} + +impl GpuCopyExternalImageDestInfo { + #[doc = "Construct a new `GpuCopyExternalImageDestInfo`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`, `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(texture: &GpuTexture) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_texture(texture); + ret + } + + #[deprecated = "Use `set_aspect()` instead."] + pub fn aspect(&mut self, val: GpuTextureAspect) -> &mut Self { + self.set_aspect(val); + self + } + + #[deprecated = "Use `set_mip_level()` instead."] + pub fn mip_level(&mut self, val: u32) -> &mut Self { + self.set_mip_level(val); + self + } + + #[deprecated = "Use `set_origin()` instead."] + pub fn origin(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_origin(val); + self + } + + #[deprecated = "Use `set_texture()` instead."] + pub fn texture(&mut self, val: &GpuTexture) -> &mut Self { + self.set_texture(val); + self + } + + #[deprecated = "Use `set_premultiplied_alpha()` instead."] + pub fn premultiplied_alpha(&mut self, val: bool) -> &mut Self { + self.set_premultiplied_alpha(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCopyExternalImageSourceInfo.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCopyExternalImageSourceInfo.rs new file mode 100644 index 00000000..bd787899 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCopyExternalImageSourceInfo.rs @@ -0,0 +1,125 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUCopyExternalImageSourceInfo)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuCopyExternalImageSourceInfo` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageSourceInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuCopyExternalImageSourceInfo; + + #[doc = "Get the `flipY` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageSourceInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "flipY")] + pub fn get_flip_y(this: &GpuCopyExternalImageSourceInfo) -> Option; + + #[doc = "Change the `flipY` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageSourceInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "flipY")] + pub fn set_flip_y(this: &GpuCopyExternalImageSourceInfo, val: bool); + + #[doc = "Get the `origin` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageSourceInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "origin")] + pub fn get_origin(this: &GpuCopyExternalImageSourceInfo) -> ::wasm_bindgen::JsValue; + + #[doc = "Change the `origin` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageSourceInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "origin")] + pub fn set_origin(this: &GpuCopyExternalImageSourceInfo, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `source` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageSourceInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "source")] + pub fn get_source(this: &GpuCopyExternalImageSourceInfo) -> ::js_sys::Object; + + #[doc = "Change the `source` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageSourceInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "source")] + pub fn set_source(this: &GpuCopyExternalImageSourceInfo, val: &::js_sys::Object); +} + +impl GpuCopyExternalImageSourceInfo { + #[doc = "Construct a new `GpuCopyExternalImageSourceInfo`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageSourceInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(source: &::js_sys::Object) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_source(source); + ret + } + + #[deprecated = "Use `set_flip_y()` instead."] + pub fn flip_y(&mut self, val: bool) -> &mut Self { + self.set_flip_y(val); + self + } + + #[deprecated = "Use `set_origin()` instead."] + pub fn origin(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_origin(val); + self + } + + #[deprecated = "Use `set_source()` instead."] + pub fn source(&mut self, val: &::js_sys::Object) -> &mut Self { + self.set_source(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCullMode.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCullMode.rs new file mode 100644 index 00000000..02e0a666 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuCullMode.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuCullMode` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuCullMode`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuCullMode { + None = "none", + Front = "front", + Back = "back", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDepthStencilState.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDepthStencilState.rs new file mode 100644 index 00000000..58de872f --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDepthStencilState.rs @@ -0,0 +1,293 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUDepthStencilState)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuDepthStencilState` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuDepthStencilState; + + #[doc = "Get the `depthBias` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthBias")] + pub fn get_depth_bias(this: &GpuDepthStencilState) -> Option; + + #[doc = "Change the `depthBias` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthBias")] + pub fn set_depth_bias(this: &GpuDepthStencilState, val: i32); + + #[doc = "Get the `depthBiasClamp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthBiasClamp")] + pub fn get_depth_bias_clamp(this: &GpuDepthStencilState) -> Option; + + #[doc = "Change the `depthBiasClamp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthBiasClamp")] + pub fn set_depth_bias_clamp(this: &GpuDepthStencilState, val: f32); + + #[doc = "Get the `depthBiasSlopeScale` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthBiasSlopeScale")] + pub fn get_depth_bias_slope_scale(this: &GpuDepthStencilState) -> Option; + + #[doc = "Change the `depthBiasSlopeScale` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthBiasSlopeScale")] + pub fn set_depth_bias_slope_scale(this: &GpuDepthStencilState, val: f32); + + #[doc = "Get the `depthCompare` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompareFunction`, `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthCompare")] + pub fn get_depth_compare(this: &GpuDepthStencilState) -> Option; + + #[doc = "Change the `depthCompare` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompareFunction`, `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthCompare")] + pub fn set_depth_compare(this: &GpuDepthStencilState, val: GpuCompareFunction); + + #[doc = "Get the `depthWriteEnabled` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthWriteEnabled")] + pub fn get_depth_write_enabled(this: &GpuDepthStencilState) -> Option; + + #[doc = "Change the `depthWriteEnabled` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthWriteEnabled")] + pub fn set_depth_write_enabled(this: &GpuDepthStencilState, val: bool); + + #[doc = "Get the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "format")] + pub fn get_format(this: &GpuDepthStencilState) -> GpuTextureFormat; + + #[doc = "Change the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "format")] + pub fn set_format(this: &GpuDepthStencilState, val: GpuTextureFormat); + + #[doc = "Get the `stencilBack` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`, `GpuStencilFaceState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stencilBack")] + pub fn get_stencil_back(this: &GpuDepthStencilState) -> Option; + + #[doc = "Change the `stencilBack` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`, `GpuStencilFaceState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stencilBack")] + pub fn set_stencil_back(this: &GpuDepthStencilState, val: &GpuStencilFaceState); + + #[doc = "Get the `stencilFront` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`, `GpuStencilFaceState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stencilFront")] + pub fn get_stencil_front(this: &GpuDepthStencilState) -> Option; + + #[doc = "Change the `stencilFront` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`, `GpuStencilFaceState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stencilFront")] + pub fn set_stencil_front(this: &GpuDepthStencilState, val: &GpuStencilFaceState); + + #[doc = "Get the `stencilReadMask` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stencilReadMask")] + pub fn get_stencil_read_mask(this: &GpuDepthStencilState) -> Option; + + #[doc = "Change the `stencilReadMask` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stencilReadMask")] + pub fn set_stencil_read_mask(this: &GpuDepthStencilState, val: u32); + + #[doc = "Get the `stencilWriteMask` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stencilWriteMask")] + pub fn get_stencil_write_mask(this: &GpuDepthStencilState) -> Option; + + #[doc = "Change the `stencilWriteMask` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stencilWriteMask")] + pub fn set_stencil_write_mask(this: &GpuDepthStencilState, val: u32); +} + +impl GpuDepthStencilState { + #[doc = "Construct a new `GpuDepthStencilState`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(format: GpuTextureFormat) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_format(format); + ret + } + + #[deprecated = "Use `set_depth_bias()` instead."] + pub fn depth_bias(&mut self, val: i32) -> &mut Self { + self.set_depth_bias(val); + self + } + + #[deprecated = "Use `set_depth_bias_clamp()` instead."] + pub fn depth_bias_clamp(&mut self, val: f32) -> &mut Self { + self.set_depth_bias_clamp(val); + self + } + + #[deprecated = "Use `set_depth_bias_slope_scale()` instead."] + pub fn depth_bias_slope_scale(&mut self, val: f32) -> &mut Self { + self.set_depth_bias_slope_scale(val); + self + } + + #[deprecated = "Use `set_depth_compare()` instead."] + pub fn depth_compare(&mut self, val: GpuCompareFunction) -> &mut Self { + self.set_depth_compare(val); + self + } + + #[deprecated = "Use `set_depth_write_enabled()` instead."] + pub fn depth_write_enabled(&mut self, val: bool) -> &mut Self { + self.set_depth_write_enabled(val); + self + } + + #[deprecated = "Use `set_format()` instead."] + pub fn format(&mut self, val: GpuTextureFormat) -> &mut Self { + self.set_format(val); + self + } + + #[deprecated = "Use `set_stencil_back()` instead."] + pub fn stencil_back(&mut self, val: &GpuStencilFaceState) -> &mut Self { + self.set_stencil_back(val); + self + } + + #[deprecated = "Use `set_stencil_front()` instead."] + pub fn stencil_front(&mut self, val: &GpuStencilFaceState) -> &mut Self { + self.set_stencil_front(val); + self + } + + #[deprecated = "Use `set_stencil_read_mask()` instead."] + pub fn stencil_read_mask(&mut self, val: u32) -> &mut Self { + self.set_stencil_read_mask(val); + self + } + + #[deprecated = "Use `set_stencil_write_mask()` instead."] + pub fn stencil_write_mask(&mut self, val: u32) -> &mut Self { + self.set_stencil_write_mask(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDevice.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDevice.rs new file mode 100644 index 00000000..966c0bd5 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDevice.rs @@ -0,0 +1,402 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = EventTarget , extends = :: js_sys :: Object , js_name = GPUDevice , typescript_type = "GPUDevice")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuDevice` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuDevice; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUDevice" , js_name = features)] + #[doc = "Getter for the `features` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/features)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuSupportedFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn features(this: &GpuDevice) -> GpuSupportedFeatures; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUDevice" , js_name = limits)] + #[doc = "Getter for the `limits` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/limits)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn limits(this: &GpuDevice) -> GpuSupportedLimits; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUDevice" , js_name = adapterInfo)] + #[doc = "Getter for the `adapterInfo` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/adapterInfo)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAdapterInfo`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn adapter_info(this: &GpuDevice) -> GpuAdapterInfo; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUDevice" , js_name = queue)] + #[doc = "Getter for the `queue` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/queue)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn queue(this: &GpuDevice) -> GpuQueue; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUDevice" , js_name = lost)] + #[doc = "Getter for the `lost` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/lost)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn lost(this: &GpuDevice) -> ::js_sys::Promise; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUDevice" , js_name = onuncapturederror)] + #[doc = "Getter for the `onuncapturederror` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/onuncapturederror)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn onuncapturederror(this: &GpuDevice) -> Option<::js_sys::Function>; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUDevice" , js_name = onuncapturederror)] + #[doc = "Setter for the `onuncapturederror` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/onuncapturederror)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_onuncapturederror(this: &GpuDevice, value: Option<&::js_sys::Function>); + + # [wasm_bindgen (structural , method , getter , js_class = "GPUDevice" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuDevice) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUDevice" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuDevice, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createBindGroup)] + #[doc = "The `createBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuBindGroupDescriptor`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_bind_group(this: &GpuDevice, descriptor: &GpuBindGroupDescriptor) + -> GpuBindGroup; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUDevice" , js_name = createBindGroupLayout)] + #[doc = "The `createBindGroupLayout()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createBindGroupLayout)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayout`, `GpuBindGroupLayoutDescriptor`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_bind_group_layout( + this: &GpuDevice, + descriptor: &GpuBindGroupLayoutDescriptor, + ) -> Result; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUDevice" , js_name = createBuffer)] + #[doc = "The `createBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuBufferDescriptor`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_buffer( + this: &GpuDevice, + descriptor: &GpuBufferDescriptor, + ) -> Result; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createCommandEncoder)] + #[doc = "The `createCommandEncoder()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createCommandEncoder)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_command_encoder(this: &GpuDevice) -> GpuCommandEncoder; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createCommandEncoder)] + #[doc = "The `createCommandEncoder()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createCommandEncoder)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCommandEncoder`, `GpuCommandEncoderDescriptor`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_command_encoder_with_descriptor( + this: &GpuDevice, + descriptor: &GpuCommandEncoderDescriptor, + ) -> GpuCommandEncoder; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createComputePipeline)] + #[doc = "The `createComputePipeline()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createComputePipeline)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipeline`, `GpuComputePipelineDescriptor`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_compute_pipeline( + this: &GpuDevice, + descriptor: &GpuComputePipelineDescriptor, + ) -> GpuComputePipeline; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createComputePipelineAsync)] + #[doc = "The `createComputePipelineAsync()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createComputePipelineAsync)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuComputePipelineDescriptor`, `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_compute_pipeline_async( + this: &GpuDevice, + descriptor: &GpuComputePipelineDescriptor, + ) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createPipelineLayout)] + #[doc = "The `createPipelineLayout()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createPipelineLayout)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuPipelineLayout`, `GpuPipelineLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_pipeline_layout( + this: &GpuDevice, + descriptor: &GpuPipelineLayoutDescriptor, + ) -> GpuPipelineLayout; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUDevice" , js_name = createQuerySet)] + #[doc = "The `createQuerySet()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createQuerySet)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuQuerySet`, `GpuQuerySetDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_query_set( + this: &GpuDevice, + descriptor: &GpuQuerySetDescriptor, + ) -> Result; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUDevice" , js_name = createRenderBundleEncoder)] + #[doc = "The `createRenderBundleEncoder()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createRenderBundleEncoder)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuRenderBundleEncoder`, `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_render_bundle_encoder( + this: &GpuDevice, + descriptor: &GpuRenderBundleEncoderDescriptor, + ) -> Result; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUDevice" , js_name = createRenderPipeline)] + #[doc = "The `createRenderPipeline()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createRenderPipeline)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuRenderPipeline`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_render_pipeline( + this: &GpuDevice, + descriptor: &GpuRenderPipelineDescriptor, + ) -> Result; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createRenderPipelineAsync)] + #[doc = "The `createRenderPipelineAsync()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createRenderPipelineAsync)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_render_pipeline_async( + this: &GpuDevice, + descriptor: &GpuRenderPipelineDescriptor, + ) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createSampler)] + #[doc = "The `createSampler()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createSampler)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuSampler`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_sampler(this: &GpuDevice) -> GpuSampler; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createSampler)] + #[doc = "The `createSampler()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createSampler)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuSampler`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_sampler_with_descriptor( + this: &GpuDevice, + descriptor: &GpuSamplerDescriptor, + ) -> GpuSampler; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = createShaderModule)] + #[doc = "The `createShaderModule()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createShaderModule)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuShaderModule`, `GpuShaderModuleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_shader_module( + this: &GpuDevice, + descriptor: &GpuShaderModuleDescriptor, + ) -> GpuShaderModule; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUDevice" , js_name = createTexture)] + #[doc = "The `createTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/createTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuTexture`, `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_texture( + this: &GpuDevice, + descriptor: &GpuTextureDescriptor, + ) -> Result; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = destroy)] + #[doc = "The `destroy()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/destroy)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn destroy(this: &GpuDevice); + + # [wasm_bindgen (catch , method , structural , js_class = "GPUDevice" , js_name = importExternalTexture)] + #[doc = "The `importExternalTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/importExternalTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuExternalTexture`, `GpuExternalTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn import_external_texture( + this: &GpuDevice, + descriptor: &GpuExternalTextureDescriptor, + ) -> Result; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = popErrorScope)] + #[doc = "The `popErrorScope()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/popErrorScope)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn pop_error_scope(this: &GpuDevice) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUDevice" , js_name = pushErrorScope)] + #[doc = "The `pushErrorScope()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDevice/pushErrorScope)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDevice`, `GpuErrorFilter`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn push_error_scope(this: &GpuDevice, filter: GpuErrorFilter); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceDescriptor.rs new file mode 100644 index 00000000..9a1bb4f2 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceDescriptor.rs @@ -0,0 +1,154 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUDeviceDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuDeviceDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuDeviceDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuDeviceDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuDeviceDescriptor, val: &str); + + #[doc = "Get the `defaultQueue` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`, `GpuQueueDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "defaultQueue")] + pub fn get_default_queue(this: &GpuDeviceDescriptor) -> Option; + + #[doc = "Change the `defaultQueue` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`, `GpuQueueDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "defaultQueue")] + pub fn set_default_queue(this: &GpuDeviceDescriptor, val: &GpuQueueDescriptor); + + #[doc = "Get the `requiredFeatures` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "requiredFeatures")] + pub fn get_required_features(this: &GpuDeviceDescriptor) -> Option<::js_sys::Array>; + + #[doc = "Change the `requiredFeatures` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "requiredFeatures")] + pub fn set_required_features(this: &GpuDeviceDescriptor, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `requiredLimits` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "requiredLimits")] + pub fn get_required_limits(this: &GpuDeviceDescriptor) -> Option<::js_sys::Object>; + + #[doc = "Change the `requiredLimits` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "requiredLimits")] + pub fn set_required_limits(this: &GpuDeviceDescriptor, val: &::js_sys::Object); +} + +impl GpuDeviceDescriptor { + #[doc = "Construct a new `GpuDeviceDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_default_queue()` instead."] + pub fn default_queue(&mut self, val: &GpuQueueDescriptor) -> &mut Self { + self.set_default_queue(val); + self + } + + #[deprecated = "Use `set_required_features()` instead."] + pub fn required_features(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_required_features(val); + self + } + + #[deprecated = "Use `set_required_limits()` instead."] + pub fn required_limits(&mut self, val: &::js_sys::Object) -> &mut Self { + self.set_required_limits(val); + self + } +} + +impl Default for GpuDeviceDescriptor { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceLostInfo.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceLostInfo.rs new file mode 100644 index 00000000..898c45c6 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceLostInfo.rs @@ -0,0 +1,62 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUDeviceLostInfo , typescript_type = "GPUDeviceLostInfo")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuDeviceLostInfo` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDeviceLostInfo)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceLostInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuDeviceLostInfo; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUDeviceLostInfo" , js_name = reason)] + #[doc = "Getter for the `reason` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDeviceLostInfo/reason)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceLostInfo`, `GpuDeviceLostReason`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn reason(this: &GpuDeviceLostInfo) -> GpuDeviceLostReason; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUDeviceLostInfo" , js_name = message)] + #[doc = "Getter for the `message` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUDeviceLostInfo/message)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDeviceLostInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn message(this: &GpuDeviceLostInfo) -> ::alloc::string::String; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceLostReason.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceLostReason.rs new file mode 100644 index 00000000..31ed7b13 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuDeviceLostReason.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuDeviceLostReason` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuDeviceLostReason`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuDeviceLostReason { + Unknown = "unknown", + Destroyed = "destroyed", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuError.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuError.rs new file mode 100644 index 00000000..6ce5b5fe --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuError.rs @@ -0,0 +1,51 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUError , typescript_type = "GPUError")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuError` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUError)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuError`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuError; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUError" , js_name = message)] + #[doc = "Getter for the `message` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUError/message)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuError`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn message(this: &GpuError) -> ::alloc::string::String; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuErrorFilter.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuErrorFilter.rs new file mode 100644 index 00000000..26a63aef --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuErrorFilter.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuErrorFilter` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuErrorFilter`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuErrorFilter { + Validation = "validation", + OutOfMemory = "out-of-memory", + Internal = "internal", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExtent3dDict.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExtent3dDict.rs new file mode 100644 index 00000000..0f81c383 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExtent3dDict.rs @@ -0,0 +1,125 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUExtent3DDict)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuExtent3dDict` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuExtent3dDict; + + #[doc = "Get the `depthOrArrayLayers` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthOrArrayLayers")] + pub fn get_depth_or_array_layers(this: &GpuExtent3dDict) -> Option; + + #[doc = "Change the `depthOrArrayLayers` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthOrArrayLayers")] + pub fn set_depth_or_array_layers(this: &GpuExtent3dDict, val: u32); + + #[doc = "Get the `height` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "height")] + pub fn get_height(this: &GpuExtent3dDict) -> Option; + + #[doc = "Change the `height` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "height")] + pub fn set_height(this: &GpuExtent3dDict, val: u32); + + #[doc = "Get the `width` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "width")] + pub fn get_width(this: &GpuExtent3dDict) -> u32; + + #[doc = "Change the `width` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "width")] + pub fn set_width(this: &GpuExtent3dDict, val: u32); +} + +impl GpuExtent3dDict { + #[doc = "Construct a new `GpuExtent3dDict`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(width: u32) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_width(width); + ret + } + + #[deprecated = "Use `set_depth_or_array_layers()` instead."] + pub fn depth_or_array_layers(&mut self, val: u32) -> &mut Self { + self.set_depth_or_array_layers(val); + self + } + + #[deprecated = "Use `set_height()` instead."] + pub fn height(&mut self, val: u32) -> &mut Self { + self.set_height(val); + self + } + + #[deprecated = "Use `set_width()` instead."] + pub fn width(&mut self, val: u32) -> &mut Self { + self.set_width(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTexture.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTexture.rs new file mode 100644 index 00000000..ea9a557b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTexture.rs @@ -0,0 +1,62 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUExternalTexture , typescript_type = "GPUExternalTexture")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuExternalTexture` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUExternalTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuExternalTexture; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUExternalTexture" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUExternalTexture/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuExternalTexture) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUExternalTexture" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUExternalTexture/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuExternalTexture, value: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTextureBindingLayout.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTextureBindingLayout.rs new file mode 100644 index 00000000..c773de83 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTextureBindingLayout.rs @@ -0,0 +1,58 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUExternalTextureBindingLayout)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuExternalTextureBindingLayout` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuExternalTextureBindingLayout; +} + +impl GpuExternalTextureBindingLayout { + #[doc = "Construct a new `GpuExternalTextureBindingLayout`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } +} + +impl Default for GpuExternalTextureBindingLayout { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTextureDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTextureDescriptor.rs new file mode 100644 index 00000000..b1c6cdc8 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuExternalTextureDescriptor.rs @@ -0,0 +1,101 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUExternalTextureDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuExternalTextureDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuExternalTextureDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuExternalTextureDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuExternalTextureDescriptor, val: &str); + + #[doc = "Get the `source` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "source")] + pub fn get_source(this: &GpuExternalTextureDescriptor) -> ::js_sys::Object; + + #[doc = "Change the `source` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "source")] + pub fn set_source(this: &GpuExternalTextureDescriptor, val: &::js_sys::Object); +} + +impl GpuExternalTextureDescriptor { + #[doc = "Construct a new `GpuExternalTextureDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExternalTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(source: &::js_sys::Object) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_source(source); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_source()` instead."] + pub fn source(&mut self, val: &::js_sys::Object) -> &mut Self { + self.set_source(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFeatureName.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFeatureName.rs new file mode 100644 index 00000000..cf3a5e85 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFeatureName.rs @@ -0,0 +1,51 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuFeatureName` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuFeatureName`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuFeatureName { + DepthClipControl = "depth-clip-control", + Depth32floatStencil8 = "depth32float-stencil8", + TextureCompressionBc = "texture-compression-bc", + TextureCompressionBcSliced3d = "texture-compression-bc-sliced-3d", + TextureCompressionEtc2 = "texture-compression-etc2", + TextureCompressionAstc = "texture-compression-astc", + TextureCompressionAstcSliced3d = "texture-compression-astc-sliced-3d", + TimestampQuery = "timestamp-query", + IndirectFirstInstance = "indirect-first-instance", + ShaderF16 = "shader-f16", + Rg11b10ufloatRenderable = "rg11b10ufloat-renderable", + Bgra8unormStorage = "bgra8unorm-storage", + Float32Filterable = "float32-filterable", + Float32Blendable = "float32-blendable", + ClipDistances = "clip-distances", + DualSourceBlending = "dual-source-blending", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFilterMode.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFilterMode.rs new file mode 100644 index 00000000..46fb0b64 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFilterMode.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuFilterMode` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuFilterMode`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuFilterMode { + Nearest = "nearest", + Linear = "linear", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFragmentState.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFragmentState.rs new file mode 100644 index 00000000..ba16aca0 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFragmentState.rs @@ -0,0 +1,150 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUFragmentState)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuFragmentState` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuFragmentState; + + #[doc = "Get the `constants` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "constants")] + pub fn get_constants(this: &GpuFragmentState) -> Option<::js_sys::Object>; + + #[doc = "Change the `constants` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "constants")] + pub fn set_constants(this: &GpuFragmentState, val: &::js_sys::Object); + + #[doc = "Get the `entryPoint` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "entryPoint")] + pub fn get_entry_point(this: &GpuFragmentState) -> Option<::alloc::string::String>; + + #[doc = "Change the `entryPoint` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "entryPoint")] + pub fn set_entry_point(this: &GpuFragmentState, val: &str); + + #[doc = "Get the `module` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`, `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "module")] + pub fn get_module(this: &GpuFragmentState) -> GpuShaderModule; + + #[doc = "Change the `module` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`, `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "module")] + pub fn set_module(this: &GpuFragmentState, val: &GpuShaderModule); + + #[doc = "Get the `targets` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "targets")] + pub fn get_targets(this: &GpuFragmentState) -> ::js_sys::Array; + + #[doc = "Change the `targets` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "targets")] + pub fn set_targets(this: &GpuFragmentState, val: &::wasm_bindgen::JsValue); +} + +impl GpuFragmentState { + #[doc = "Construct a new `GpuFragmentState`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`, `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(module: &GpuShaderModule, targets: &::wasm_bindgen::JsValue) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_module(module); + ret.set_targets(targets); + ret + } + + #[deprecated = "Use `set_constants()` instead."] + pub fn constants(&mut self, val: &::js_sys::Object) -> &mut Self { + self.set_constants(val); + self + } + + #[deprecated = "Use `set_entry_point()` instead."] + pub fn entry_point(&mut self, val: &str) -> &mut Self { + self.set_entry_point(val); + self + } + + #[deprecated = "Use `set_module()` instead."] + pub fn module(&mut self, val: &GpuShaderModule) -> &mut Self { + self.set_module(val); + self + } + + #[deprecated = "Use `set_targets()` instead."] + pub fn targets(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_targets(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFrontFace.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFrontFace.rs new file mode 100644 index 00000000..eb9eda7b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuFrontFace.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuFrontFace` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuFrontFace`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuFrontFace { + Ccw = "ccw", + Cw = "cw", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuIndexFormat.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuIndexFormat.rs new file mode 100644 index 00000000..7931e961 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuIndexFormat.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuIndexFormat` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuIndexFormat`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuIndexFormat { + Uint16 = "uint16", + Uint32 = "uint32", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuLoadOp.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuLoadOp.rs new file mode 100644 index 00000000..6717dc76 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuLoadOp.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuLoadOp` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuLoadOp`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuLoadOp { + Load = "load", + Clear = "clear", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuMipmapFilterMode.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuMipmapFilterMode.rs new file mode 100644 index 00000000..9d962297 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuMipmapFilterMode.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuMipmapFilterMode` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuMipmapFilterMode`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuMipmapFilterMode { + Nearest = "nearest", + Linear = "linear", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuMultisampleState.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuMultisampleState.rs new file mode 100644 index 00000000..28cd36d2 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuMultisampleState.rs @@ -0,0 +1,130 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUMultisampleState)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuMultisampleState` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuMultisampleState; + + #[doc = "Get the `alphaToCoverageEnabled` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "alphaToCoverageEnabled")] + pub fn get_alpha_to_coverage_enabled(this: &GpuMultisampleState) -> Option; + + #[doc = "Change the `alphaToCoverageEnabled` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "alphaToCoverageEnabled")] + pub fn set_alpha_to_coverage_enabled(this: &GpuMultisampleState, val: bool); + + #[doc = "Get the `count` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "count")] + pub fn get_count(this: &GpuMultisampleState) -> Option; + + #[doc = "Change the `count` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "count")] + pub fn set_count(this: &GpuMultisampleState, val: u32); + + #[doc = "Get the `mask` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "mask")] + pub fn get_mask(this: &GpuMultisampleState) -> Option; + + #[doc = "Change the `mask` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "mask")] + pub fn set_mask(this: &GpuMultisampleState, val: u32); +} + +impl GpuMultisampleState { + #[doc = "Construct a new `GpuMultisampleState`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_alpha_to_coverage_enabled()` instead."] + pub fn alpha_to_coverage_enabled(&mut self, val: bool) -> &mut Self { + self.set_alpha_to_coverage_enabled(val); + self + } + + #[deprecated = "Use `set_count()` instead."] + pub fn count(&mut self, val: u32) -> &mut Self { + self.set_count(val); + self + } + + #[deprecated = "Use `set_mask()` instead."] + pub fn mask(&mut self, val: u32) -> &mut Self { + self.set_mask(val); + self + } +} + +impl Default for GpuMultisampleState { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuObjectDescriptorBase.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuObjectDescriptorBase.rs new file mode 100644 index 00000000..f46278f7 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuObjectDescriptorBase.rs @@ -0,0 +1,82 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUObjectDescriptorBase)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuObjectDescriptorBase` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuObjectDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuObjectDescriptorBase; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuObjectDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuObjectDescriptorBase) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuObjectDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuObjectDescriptorBase, val: &str); +} + +impl GpuObjectDescriptorBase { + #[doc = "Construct a new `GpuObjectDescriptorBase`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuObjectDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } +} + +impl Default for GpuObjectDescriptorBase { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOrigin2dDict.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOrigin2dDict.rs new file mode 100644 index 00000000..c8b52100 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOrigin2dDict.rs @@ -0,0 +1,106 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUOrigin2DDict)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuOrigin2dDict` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin2dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuOrigin2dDict; + + #[doc = "Get the `x` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin2dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "x")] + pub fn get_x(this: &GpuOrigin2dDict) -> Option; + + #[doc = "Change the `x` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin2dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "x")] + pub fn set_x(this: &GpuOrigin2dDict, val: u32); + + #[doc = "Get the `y` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin2dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "y")] + pub fn get_y(this: &GpuOrigin2dDict) -> Option; + + #[doc = "Change the `y` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin2dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "y")] + pub fn set_y(this: &GpuOrigin2dDict, val: u32); +} + +impl GpuOrigin2dDict { + #[doc = "Construct a new `GpuOrigin2dDict`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin2dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_x()` instead."] + pub fn x(&mut self, val: u32) -> &mut Self { + self.set_x(val); + self + } + + #[deprecated = "Use `set_y()` instead."] + pub fn y(&mut self, val: u32) -> &mut Self { + self.set_y(val); + self + } +} + +impl Default for GpuOrigin2dDict { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOrigin3dDict.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOrigin3dDict.rs new file mode 100644 index 00000000..a6c77558 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOrigin3dDict.rs @@ -0,0 +1,130 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUOrigin3DDict)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuOrigin3dDict` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuOrigin3dDict; + + #[doc = "Get the `x` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "x")] + pub fn get_x(this: &GpuOrigin3dDict) -> Option; + + #[doc = "Change the `x` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "x")] + pub fn set_x(this: &GpuOrigin3dDict, val: u32); + + #[doc = "Get the `y` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "y")] + pub fn get_y(this: &GpuOrigin3dDict) -> Option; + + #[doc = "Change the `y` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "y")] + pub fn set_y(this: &GpuOrigin3dDict, val: u32); + + #[doc = "Get the `z` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "z")] + pub fn get_z(this: &GpuOrigin3dDict) -> Option; + + #[doc = "Change the `z` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "z")] + pub fn set_z(this: &GpuOrigin3dDict, val: u32); +} + +impl GpuOrigin3dDict { + #[doc = "Construct a new `GpuOrigin3dDict`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOrigin3dDict`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_x()` instead."] + pub fn x(&mut self, val: u32) -> &mut Self { + self.set_x(val); + self + } + + #[deprecated = "Use `set_y()` instead."] + pub fn y(&mut self, val: u32) -> &mut Self { + self.set_y(val); + self + } + + #[deprecated = "Use `set_z()` instead."] + pub fn z(&mut self, val: u32) -> &mut Self { + self.set_z(val); + self + } +} + +impl Default for GpuOrigin3dDict { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOutOfMemoryError.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOutOfMemoryError.rs new file mode 100644 index 00000000..21f527de --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuOutOfMemoryError.rs @@ -0,0 +1,51 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = GpuError , extends = :: js_sys :: Object , js_name = GPUOutOfMemoryError , typescript_type = "GPUOutOfMemoryError")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuOutOfMemoryError` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUOutOfMemoryError)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOutOfMemoryError`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuOutOfMemoryError; + + #[wasm_bindgen(catch, constructor, js_class = "GPUOutOfMemoryError")] + #[doc = "The `new GpuOutOfMemoryError(..)` constructor, creating a new instance of `GpuOutOfMemoryError`."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUOutOfMemoryError/GPUOutOfMemoryError)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuOutOfMemoryError`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(message: &str) -> Result; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineDescriptorBase.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineDescriptorBase.rs new file mode 100644 index 00000000..e0067ddc --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineDescriptorBase.rs @@ -0,0 +1,101 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUPipelineDescriptorBase)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuPipelineDescriptorBase` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuPipelineDescriptorBase; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuPipelineDescriptorBase) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuPipelineDescriptorBase, val: &str); + + #[doc = "Get the `layout` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "layout")] + pub fn get_layout(this: &GpuPipelineDescriptorBase) -> ::wasm_bindgen::JsValue; + + #[doc = "Change the `layout` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "layout")] + pub fn set_layout(this: &GpuPipelineDescriptorBase, val: &::wasm_bindgen::JsValue); +} + +impl GpuPipelineDescriptorBase { + #[doc = "Construct a new `GpuPipelineDescriptorBase`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineDescriptorBase`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(layout: &::wasm_bindgen::JsValue) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_layout(layout); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_layout()` instead."] + pub fn layout(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_layout(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineLayout.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineLayout.rs new file mode 100644 index 00000000..da21f62e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineLayout.rs @@ -0,0 +1,62 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUPipelineLayout , typescript_type = "GPUPipelineLayout")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuPipelineLayout` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUPipelineLayout)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuPipelineLayout; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUPipelineLayout" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUPipelineLayout/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuPipelineLayout) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUPipelineLayout" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUPipelineLayout/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuPipelineLayout, value: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineLayoutDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineLayoutDescriptor.rs new file mode 100644 index 00000000..7fc19056 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPipelineLayoutDescriptor.rs @@ -0,0 +1,104 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUPipelineLayoutDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuPipelineLayoutDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuPipelineLayoutDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuPipelineLayoutDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuPipelineLayoutDescriptor, val: &str); + + #[doc = "Get the `bindGroupLayouts` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "bindGroupLayouts")] + pub fn get_bind_group_layouts(this: &GpuPipelineLayoutDescriptor) -> ::js_sys::Array; + + #[doc = "Change the `bindGroupLayouts` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "bindGroupLayouts")] + pub fn set_bind_group_layouts( + this: &GpuPipelineLayoutDescriptor, + val: &::wasm_bindgen::JsValue, + ); +} + +impl GpuPipelineLayoutDescriptor { + #[doc = "Construct a new `GpuPipelineLayoutDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPipelineLayoutDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(bind_group_layouts: &::wasm_bindgen::JsValue) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_bind_group_layouts(bind_group_layouts); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_bind_group_layouts()` instead."] + pub fn bind_group_layouts(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_bind_group_layouts(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPowerPreference.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPowerPreference.rs new file mode 100644 index 00000000..d674f0ef --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPowerPreference.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuPowerPreference` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuPowerPreference`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuPowerPreference { + LowPower = "low-power", + HighPerformance = "high-performance", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPrimitiveState.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPrimitiveState.rs new file mode 100644 index 00000000..cf9cdb3e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPrimitiveState.rs @@ -0,0 +1,178 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUPrimitiveState)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuPrimitiveState` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuPrimitiveState; + + #[doc = "Get the `cullMode` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCullMode`, `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "cullMode")] + pub fn get_cull_mode(this: &GpuPrimitiveState) -> Option; + + #[doc = "Change the `cullMode` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCullMode`, `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "cullMode")] + pub fn set_cull_mode(this: &GpuPrimitiveState, val: GpuCullMode); + + #[doc = "Get the `frontFace` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFrontFace`, `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "frontFace")] + pub fn get_front_face(this: &GpuPrimitiveState) -> Option; + + #[doc = "Change the `frontFace` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFrontFace`, `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "frontFace")] + pub fn set_front_face(this: &GpuPrimitiveState, val: GpuFrontFace); + + #[doc = "Get the `stripIndexFormat` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuIndexFormat`, `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stripIndexFormat")] + pub fn get_strip_index_format(this: &GpuPrimitiveState) -> Option; + + #[doc = "Change the `stripIndexFormat` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuIndexFormat`, `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stripIndexFormat")] + pub fn set_strip_index_format(this: &GpuPrimitiveState, val: GpuIndexFormat); + + #[doc = "Get the `topology` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPrimitiveState`, `GpuPrimitiveTopology`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "topology")] + pub fn get_topology(this: &GpuPrimitiveState) -> Option; + + #[doc = "Change the `topology` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPrimitiveState`, `GpuPrimitiveTopology`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "topology")] + pub fn set_topology(this: &GpuPrimitiveState, val: GpuPrimitiveTopology); + + #[doc = "Get the `unclippedDepth` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "unclippedDepth")] + pub fn get_unclipped_depth(this: &GpuPrimitiveState) -> Option; + + #[doc = "Change the `unclippedDepth` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "unclippedDepth")] + pub fn set_unclipped_depth(this: &GpuPrimitiveState, val: bool); +} + +impl GpuPrimitiveState { + #[doc = "Construct a new `GpuPrimitiveState`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPrimitiveState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_cull_mode()` instead."] + pub fn cull_mode(&mut self, val: GpuCullMode) -> &mut Self { + self.set_cull_mode(val); + self + } + + #[deprecated = "Use `set_front_face()` instead."] + pub fn front_face(&mut self, val: GpuFrontFace) -> &mut Self { + self.set_front_face(val); + self + } + + #[deprecated = "Use `set_strip_index_format()` instead."] + pub fn strip_index_format(&mut self, val: GpuIndexFormat) -> &mut Self { + self.set_strip_index_format(val); + self + } + + #[deprecated = "Use `set_topology()` instead."] + pub fn topology(&mut self, val: GpuPrimitiveTopology) -> &mut Self { + self.set_topology(val); + self + } + + #[deprecated = "Use `set_unclipped_depth()` instead."] + pub fn unclipped_depth(&mut self, val: bool) -> &mut Self { + self.set_unclipped_depth(val); + self + } +} + +impl Default for GpuPrimitiveState { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPrimitiveTopology.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPrimitiveTopology.rs new file mode 100644 index 00000000..6ec5b281 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuPrimitiveTopology.rs @@ -0,0 +1,40 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuPrimitiveTopology` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuPrimitiveTopology`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuPrimitiveTopology { + PointList = "point-list", + LineList = "line-list", + LineStrip = "line-strip", + TriangleList = "triangle-list", + TriangleStrip = "triangle-strip", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuProgrammableStage.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuProgrammableStage.rs new file mode 100644 index 00000000..03110080 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuProgrammableStage.rs @@ -0,0 +1,125 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUProgrammableStage)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuProgrammableStage` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuProgrammableStage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuProgrammableStage; + + #[doc = "Get the `constants` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuProgrammableStage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "constants")] + pub fn get_constants(this: &GpuProgrammableStage) -> Option<::js_sys::Object>; + + #[doc = "Change the `constants` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuProgrammableStage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "constants")] + pub fn set_constants(this: &GpuProgrammableStage, val: &::js_sys::Object); + + #[doc = "Get the `entryPoint` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuProgrammableStage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "entryPoint")] + pub fn get_entry_point(this: &GpuProgrammableStage) -> Option<::alloc::string::String>; + + #[doc = "Change the `entryPoint` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuProgrammableStage`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "entryPoint")] + pub fn set_entry_point(this: &GpuProgrammableStage, val: &str); + + #[doc = "Get the `module` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuProgrammableStage`, `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "module")] + pub fn get_module(this: &GpuProgrammableStage) -> GpuShaderModule; + + #[doc = "Change the `module` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuProgrammableStage`, `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "module")] + pub fn set_module(this: &GpuProgrammableStage, val: &GpuShaderModule); +} + +impl GpuProgrammableStage { + #[doc = "Construct a new `GpuProgrammableStage`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuProgrammableStage`, `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(module: &GpuShaderModule) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_module(module); + ret + } + + #[deprecated = "Use `set_constants()` instead."] + pub fn constants(&mut self, val: &::js_sys::Object) -> &mut Self { + self.set_constants(val); + self + } + + #[deprecated = "Use `set_entry_point()` instead."] + pub fn entry_point(&mut self, val: &str) -> &mut Self { + self.set_entry_point(val); + self + } + + #[deprecated = "Use `set_module()` instead."] + pub fn module(&mut self, val: &GpuShaderModule) -> &mut Self { + self.set_module(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQuerySet.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQuerySet.rs new file mode 100644 index 00000000..ddaf35ac --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQuerySet.rs @@ -0,0 +1,95 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUQuerySet , typescript_type = "GPUQuerySet")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuQuerySet` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQuerySet)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuQuerySet; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUQuerySet" , js_name = type)] + #[doc = "Getter for the `type` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQuerySet/type)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`, `GpuQueryType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn type_(this: &GpuQuerySet) -> GpuQueryType; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUQuerySet" , js_name = count)] + #[doc = "Getter for the `count` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQuerySet/count)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn count(this: &GpuQuerySet) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUQuerySet" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQuerySet/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuQuerySet) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUQuerySet" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQuerySet/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuQuerySet, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPUQuerySet" , js_name = destroy)] + #[doc = "The `destroy()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQuerySet/destroy)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn destroy(this: &GpuQuerySet); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQuerySetDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQuerySetDescriptor.rs new file mode 100644 index 00000000..def89e14 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQuerySetDescriptor.rs @@ -0,0 +1,126 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUQuerySetDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuQuerySetDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySetDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuQuerySetDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySetDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuQuerySetDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySetDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuQuerySetDescriptor, val: &str); + + #[doc = "Get the `count` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySetDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "count")] + pub fn get_count(this: &GpuQuerySetDescriptor) -> u32; + + #[doc = "Change the `count` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySetDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "count")] + pub fn set_count(this: &GpuQuerySetDescriptor, val: u32); + + #[doc = "Get the `type` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySetDescriptor`, `GpuQueryType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "type")] + pub fn get_type(this: &GpuQuerySetDescriptor) -> GpuQueryType; + + #[doc = "Change the `type` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySetDescriptor`, `GpuQueryType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "type")] + pub fn set_type(this: &GpuQuerySetDescriptor, val: GpuQueryType); +} + +impl GpuQuerySetDescriptor { + #[doc = "Construct a new `GpuQuerySetDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySetDescriptor`, `GpuQueryType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(count: u32, type_: GpuQueryType) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_count(count); + ret.set_type(type_); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_count()` instead."] + pub fn count(&mut self, val: u32) -> &mut Self { + self.set_count(val); + self + } + + #[deprecated = "Use `set_type()` instead."] + pub fn type_(&mut self, val: GpuQueryType) -> &mut Self { + self.set_type(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueryType.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueryType.rs new file mode 100644 index 00000000..58d693ab --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueryType.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuQueryType` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuQueryType`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuQueryType { + Occlusion = "occlusion", + Timestamp = "timestamp", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueue.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueue.rs new file mode 100644 index 00000000..4b65d7ad --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueue.rs @@ -0,0 +1,950 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUQueue , typescript_type = "GPUQueue")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuQueue` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuQueue; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUQueue" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuQueue) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUQueue" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuQueue, value: &str); + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = copyExternalImageToTexture)] + #[doc = "The `copyExternalImageToTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/copyExternalImageToTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`, `GpuCopyExternalImageSourceInfo`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_external_image_to_texture_with_u32_sequence( + this: &GpuQueue, + source: &GpuCopyExternalImageSourceInfo, + destination: &GpuCopyExternalImageDestInfo, + copy_size: &::wasm_bindgen::JsValue, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = copyExternalImageToTexture)] + #[doc = "The `copyExternalImageToTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/copyExternalImageToTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCopyExternalImageDestInfo`, `GpuCopyExternalImageSourceInfo`, `GpuExtent3dDict`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn copy_external_image_to_texture_with_gpu_extent_3d_dict( + this: &GpuQueue, + source: &GpuCopyExternalImageSourceInfo, + destination: &GpuCopyExternalImageDestInfo, + copy_size: &GpuExtent3dDict, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (method , structural , js_class = "GPUQueue" , js_name = onSubmittedWorkDone)] + #[doc = "The `onSubmittedWorkDone()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/onSubmittedWorkDone)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn on_submitted_work_done(this: &GpuQueue) -> ::js_sys::Promise; + + # [wasm_bindgen (method , structural , js_class = "GPUQueue" , js_name = submit)] + #[doc = "The `submit()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/submit)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn submit(this: &GpuQueue, command_buffers: &::wasm_bindgen::JsValue); + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_buffer_source( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Object, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_buffer_source( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Object, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_slice( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &[u8], + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_slice( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &[u8], + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_array( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Uint8Array, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_array( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Uint8Array, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_buffer_source_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Object, + data_offset: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_buffer_source_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Object, + data_offset: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_slice_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &[u8], + data_offset: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_slice_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &[u8], + data_offset: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_array_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Uint8Array, + data_offset: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_array_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Uint8Array, + data_offset: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_buffer_source_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Object, + data_offset: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_buffer_source_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Object, + data_offset: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_slice_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &[u8], + data_offset: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_slice_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &[u8], + data_offset: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_array_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Uint8Array, + data_offset: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_array_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Uint8Array, + data_offset: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_buffer_source_and_u32_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Object, + data_offset: u32, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_buffer_source_and_u32_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Object, + data_offset: u32, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_slice_and_u32_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &[u8], + data_offset: u32, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_slice_and_u32_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &[u8], + data_offset: u32, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_array_and_u32_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Uint8Array, + data_offset: u32, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_array_and_u32_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Uint8Array, + data_offset: u32, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_buffer_source_and_f64_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Object, + data_offset: f64, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_buffer_source_and_f64_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Object, + data_offset: f64, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_slice_and_f64_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &[u8], + data_offset: f64, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_slice_and_f64_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &[u8], + data_offset: f64, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_array_and_f64_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Uint8Array, + data_offset: f64, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_array_and_f64_and_u32( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Uint8Array, + data_offset: f64, + size: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_buffer_source_and_u32_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Object, + data_offset: u32, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_buffer_source_and_u32_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Object, + data_offset: u32, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_slice_and_u32_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &[u8], + data_offset: u32, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_slice_and_u32_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &[u8], + data_offset: u32, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_array_and_u32_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Uint8Array, + data_offset: u32, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_array_and_u32_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Uint8Array, + data_offset: u32, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_buffer_source_and_f64_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Object, + data_offset: f64, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_buffer_source_and_f64_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Object, + data_offset: f64, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_slice_and_f64_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &[u8], + data_offset: f64, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_slice_and_f64_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &[u8], + data_offset: f64, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_u32_and_u8_array_and_f64_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: u32, + data: &::js_sys::Uint8Array, + data_offset: f64, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeBuffer)] + #[doc = "The `writeBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuQueue`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_buffer_with_f64_and_u8_array_and_f64_and_f64( + this: &GpuQueue, + buffer: &GpuBuffer, + buffer_offset: f64, + data: &::js_sys::Uint8Array, + data_offset: f64, + size: f64, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeTexture)] + #[doc = "The `writeTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueue`, `GpuTexelCopyBufferLayout`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_texture_with_buffer_source_and_u32_sequence( + this: &GpuQueue, + destination: &GpuTexelCopyTextureInfo, + data: &::js_sys::Object, + data_layout: &GpuTexelCopyBufferLayout, + size: &::wasm_bindgen::JsValue, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeTexture)] + #[doc = "The `writeTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueue`, `GpuTexelCopyBufferLayout`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_texture_with_u8_slice_and_u32_sequence( + this: &GpuQueue, + destination: &GpuTexelCopyTextureInfo, + data: &[u8], + data_layout: &GpuTexelCopyBufferLayout, + size: &::wasm_bindgen::JsValue, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeTexture)] + #[doc = "The `writeTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueue`, `GpuTexelCopyBufferLayout`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_texture_with_u8_array_and_u32_sequence( + this: &GpuQueue, + destination: &GpuTexelCopyTextureInfo, + data: &::js_sys::Uint8Array, + data_layout: &GpuTexelCopyBufferLayout, + size: &::wasm_bindgen::JsValue, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeTexture)] + #[doc = "The `writeTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`, `GpuQueue`, `GpuTexelCopyBufferLayout`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_texture_with_buffer_source_and_gpu_extent_3d_dict( + this: &GpuQueue, + destination: &GpuTexelCopyTextureInfo, + data: &::js_sys::Object, + data_layout: &GpuTexelCopyBufferLayout, + size: &GpuExtent3dDict, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeTexture)] + #[doc = "The `writeTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`, `GpuQueue`, `GpuTexelCopyBufferLayout`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_texture_with_u8_slice_and_gpu_extent_3d_dict( + this: &GpuQueue, + destination: &GpuTexelCopyTextureInfo, + data: &[u8], + data_layout: &GpuTexelCopyBufferLayout, + size: &GpuExtent3dDict, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUQueue" , js_name = writeTexture)] + #[doc = "The `writeTexture()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUQueue/writeTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuExtent3dDict`, `GpuQueue`, `GpuTexelCopyBufferLayout`, `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn write_texture_with_u8_array_and_gpu_extent_3d_dict( + this: &GpuQueue, + destination: &GpuTexelCopyTextureInfo, + data: &::js_sys::Uint8Array, + data_layout: &GpuTexelCopyBufferLayout, + size: &GpuExtent3dDict, + ) -> Result<(), JsValue>; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueueDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueueDescriptor.rs new file mode 100644 index 00000000..822a6f06 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuQueueDescriptor.rs @@ -0,0 +1,82 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUQueueDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuQueueDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueueDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuQueueDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueueDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuQueueDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueueDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuQueueDescriptor, val: &str); +} + +impl GpuQueueDescriptor { + #[doc = "Construct a new `GpuQueueDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQueueDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } +} + +impl Default for GpuQueueDescriptor { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundle.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundle.rs new file mode 100644 index 00000000..36f85f3b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundle.rs @@ -0,0 +1,62 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderBundle , typescript_type = "GPURenderBundle")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderBundle` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundle)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundle`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderBundle; + + # [wasm_bindgen (structural , method , getter , js_class = "GPURenderBundle" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundle/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundle`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuRenderBundle) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPURenderBundle" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundle/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundle`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuRenderBundle, value: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleDescriptor.rs new file mode 100644 index 00000000..4f2c6c28 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleDescriptor.rs @@ -0,0 +1,82 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderBundleDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderBundleDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderBundleDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuRenderBundleDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuRenderBundleDescriptor, val: &str); +} + +impl GpuRenderBundleDescriptor { + #[doc = "Construct a new `GpuRenderBundleDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } +} + +impl Default for GpuRenderBundleDescriptor { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleEncoder.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleEncoder.rs new file mode 100644 index 00000000..36519cd3 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleEncoder.rs @@ -0,0 +1,656 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderBundleEncoder , typescript_type = "GPURenderBundleEncoder")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderBundleEncoder` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderBundleEncoder; + + # [wasm_bindgen (structural , method , getter , js_class = "GPURenderBundleEncoder" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuRenderBundleEncoder) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPURenderBundleEncoder" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuRenderBundleEncoder, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = finish)] + #[doc = "The `finish()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/finish)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundle`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn finish(this: &GpuRenderBundleEncoder) -> GpuRenderBundle; + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = finish)] + #[doc = "The `finish()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/finish)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundle`, `GpuRenderBundleDescriptor`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn finish_with_descriptor( + this: &GpuRenderBundleEncoder, + descriptor: &GpuRenderBundleDescriptor, + ) -> GpuRenderBundle; + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group( + this: &GpuRenderBundleEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_sequence( + this: &GpuRenderBundleEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets: &::wasm_bindgen::JsValue, + ); + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderBundleEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_slice_and_u32_and_dynamic_offsets_data_length( + this: &GpuRenderBundleEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &[u32], + dynamic_offsets_data_start: u32, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderBundleEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_array_and_u32_and_dynamic_offsets_data_length( + this: &GpuRenderBundleEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &::js_sys::Uint32Array, + dynamic_offsets_data_start: u32, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderBundleEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_slice_and_f64_and_dynamic_offsets_data_length( + this: &GpuRenderBundleEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &[u32], + dynamic_offsets_data_start: f64, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderBundleEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_array_and_f64_and_dynamic_offsets_data_length( + this: &GpuRenderBundleEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &::js_sys::Uint32Array, + dynamic_offsets_data_start: f64, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = insertDebugMarker)] + #[doc = "The `insertDebugMarker()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/insertDebugMarker)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn insert_debug_marker(this: &GpuRenderBundleEncoder, marker_label: &str); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = popDebugGroup)] + #[doc = "The `popDebugGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/popDebugGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn pop_debug_group(this: &GpuRenderBundleEncoder); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = pushDebugGroup)] + #[doc = "The `pushDebugGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/pushDebugGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn push_debug_group(this: &GpuRenderBundleEncoder, group_label: &str); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = draw)] + #[doc = "The `draw()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/draw)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw(this: &GpuRenderBundleEncoder, vertex_count: u32); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = draw)] + #[doc = "The `draw()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/draw)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_with_instance_count( + this: &GpuRenderBundleEncoder, + vertex_count: u32, + instance_count: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = draw)] + #[doc = "The `draw()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/draw)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_with_instance_count_and_first_vertex( + this: &GpuRenderBundleEncoder, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = draw)] + #[doc = "The `draw()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/draw)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_with_instance_count_and_first_vertex_and_first_instance( + this: &GpuRenderBundleEncoder, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed(this: &GpuRenderBundleEncoder, index_count: u32); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_with_instance_count( + this: &GpuRenderBundleEncoder, + index_count: u32, + instance_count: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_with_instance_count_and_first_index( + this: &GpuRenderBundleEncoder, + index_count: u32, + instance_count: u32, + first_index: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_with_instance_count_and_first_index_and_base_vertex( + this: &GpuRenderBundleEncoder, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_with_instance_count_and_first_index_and_base_vertex_and_first_instance( + this: &GpuRenderBundleEncoder, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = drawIndexedIndirect)] + #[doc = "The `drawIndexedIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/drawIndexedIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_indirect_with_u32( + this: &GpuRenderBundleEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = drawIndexedIndirect)] + #[doc = "The `drawIndexedIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/drawIndexedIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_indirect_with_f64( + this: &GpuRenderBundleEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = drawIndirect)] + #[doc = "The `drawIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/drawIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indirect_with_u32( + this: &GpuRenderBundleEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = drawIndirect)] + #[doc = "The `drawIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/drawIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indirect_with_f64( + this: &GpuRenderBundleEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer( + this: &GpuRenderBundleEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_u32( + this: &GpuRenderBundleEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_f64( + this: &GpuRenderBundleEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_u32_and_u32( + this: &GpuRenderBundleEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: u32, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_f64_and_u32( + this: &GpuRenderBundleEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: f64, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_u32_and_f64( + this: &GpuRenderBundleEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: u32, + size: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_f64_and_f64( + this: &GpuRenderBundleEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: f64, + size: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setPipeline)] + #[doc = "The `setPipeline()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setPipeline)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoder`, `GpuRenderPipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_pipeline(this: &GpuRenderBundleEncoder, pipeline: &GpuRenderPipeline); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer(this: &GpuRenderBundleEncoder, slot: u32, buffer: Option<&GpuBuffer>); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_u32( + this: &GpuRenderBundleEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_f64( + this: &GpuRenderBundleEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_u32_and_u32( + this: &GpuRenderBundleEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: u32, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_f64_and_u32( + this: &GpuRenderBundleEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: f64, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_u32_and_f64( + this: &GpuRenderBundleEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: u32, + size: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderBundleEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderBundleEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderBundleEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_f64_and_f64( + this: &GpuRenderBundleEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: f64, + size: f64, + ); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleEncoderDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleEncoderDescriptor.rs new file mode 100644 index 00000000..e4a7939a --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderBundleEncoderDescriptor.rs @@ -0,0 +1,202 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderBundleEncoderDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderBundleEncoderDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderBundleEncoderDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuRenderBundleEncoderDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuRenderBundleEncoderDescriptor, val: &str); + + #[doc = "Get the `colorFormats` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "colorFormats")] + pub fn get_color_formats(this: &GpuRenderBundleEncoderDescriptor) -> ::js_sys::Array; + + #[doc = "Change the `colorFormats` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "colorFormats")] + pub fn set_color_formats( + this: &GpuRenderBundleEncoderDescriptor, + val: &::wasm_bindgen::JsValue, + ); + + #[doc = "Get the `depthStencilFormat` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthStencilFormat")] + pub fn get_depth_stencil_format( + this: &GpuRenderBundleEncoderDescriptor, + ) -> Option; + + #[doc = "Change the `depthStencilFormat` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthStencilFormat")] + pub fn set_depth_stencil_format(this: &GpuRenderBundleEncoderDescriptor, val: GpuTextureFormat); + + #[doc = "Get the `sampleCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "sampleCount")] + pub fn get_sample_count(this: &GpuRenderBundleEncoderDescriptor) -> Option; + + #[doc = "Change the `sampleCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "sampleCount")] + pub fn set_sample_count(this: &GpuRenderBundleEncoderDescriptor, val: u32); + + #[doc = "Get the `depthReadOnly` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthReadOnly")] + pub fn get_depth_read_only(this: &GpuRenderBundleEncoderDescriptor) -> Option; + + #[doc = "Change the `depthReadOnly` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthReadOnly")] + pub fn set_depth_read_only(this: &GpuRenderBundleEncoderDescriptor, val: bool); + + #[doc = "Get the `stencilReadOnly` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stencilReadOnly")] + pub fn get_stencil_read_only(this: &GpuRenderBundleEncoderDescriptor) -> Option; + + #[doc = "Change the `stencilReadOnly` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stencilReadOnly")] + pub fn set_stencil_read_only(this: &GpuRenderBundleEncoderDescriptor, val: bool); +} + +impl GpuRenderBundleEncoderDescriptor { + #[doc = "Construct a new `GpuRenderBundleEncoderDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderBundleEncoderDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(color_formats: &::wasm_bindgen::JsValue) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_color_formats(color_formats); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_color_formats()` instead."] + pub fn color_formats(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_color_formats(val); + self + } + + #[deprecated = "Use `set_depth_stencil_format()` instead."] + pub fn depth_stencil_format(&mut self, val: GpuTextureFormat) -> &mut Self { + self.set_depth_stencil_format(val); + self + } + + #[deprecated = "Use `set_sample_count()` instead."] + pub fn sample_count(&mut self, val: u32) -> &mut Self { + self.set_sample_count(val); + self + } + + #[deprecated = "Use `set_depth_read_only()` instead."] + pub fn depth_read_only(&mut self, val: bool) -> &mut Self { + self.set_depth_read_only(val); + self + } + + #[deprecated = "Use `set_stencil_read_only()` instead."] + pub fn stencil_read_only(&mut self, val: bool) -> &mut Self { + self.set_stencil_read_only(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassColorAttachment.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassColorAttachment.rs new file mode 100644 index 00000000..7bba182b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassColorAttachment.rs @@ -0,0 +1,199 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderPassColorAttachment)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderPassColorAttachment` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderPassColorAttachment; + + #[doc = "Get the `clearValue` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "clearValue")] + pub fn get_clear_value(this: &GpuRenderPassColorAttachment) -> ::wasm_bindgen::JsValue; + + #[doc = "Change the `clearValue` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "clearValue")] + pub fn set_clear_value(this: &GpuRenderPassColorAttachment, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `depthSlice` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthSlice")] + pub fn get_depth_slice(this: &GpuRenderPassColorAttachment) -> Option; + + #[doc = "Change the `depthSlice` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthSlice")] + pub fn set_depth_slice(this: &GpuRenderPassColorAttachment, val: u32); + + #[doc = "Get the `loadOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuLoadOp`, `GpuRenderPassColorAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "loadOp")] + pub fn get_load_op(this: &GpuRenderPassColorAttachment) -> GpuLoadOp; + + #[doc = "Change the `loadOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuLoadOp`, `GpuRenderPassColorAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "loadOp")] + pub fn set_load_op(this: &GpuRenderPassColorAttachment, val: GpuLoadOp); + + #[doc = "Get the `resolveTarget` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`, `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "resolveTarget")] + pub fn get_resolve_target(this: &GpuRenderPassColorAttachment) -> Option; + + #[doc = "Change the `resolveTarget` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`, `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "resolveTarget")] + pub fn set_resolve_target(this: &GpuRenderPassColorAttachment, val: &GpuTextureView); + + #[doc = "Get the `storeOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`, `GpuStoreOp`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "storeOp")] + pub fn get_store_op(this: &GpuRenderPassColorAttachment) -> GpuStoreOp; + + #[doc = "Change the `storeOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`, `GpuStoreOp`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "storeOp")] + pub fn set_store_op(this: &GpuRenderPassColorAttachment, val: GpuStoreOp); + + #[doc = "Get the `view` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`, `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "view")] + pub fn get_view(this: &GpuRenderPassColorAttachment) -> GpuTextureView; + + #[doc = "Change the `view` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassColorAttachment`, `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "view")] + pub fn set_view(this: &GpuRenderPassColorAttachment, val: &GpuTextureView); +} + +impl GpuRenderPassColorAttachment { + #[doc = "Construct a new `GpuRenderPassColorAttachment`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuLoadOp`, `GpuRenderPassColorAttachment`, `GpuStoreOp`, `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(load_op: GpuLoadOp, store_op: GpuStoreOp, view: &GpuTextureView) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_load_op(load_op); + ret.set_store_op(store_op); + ret.set_view(view); + ret + } + + #[deprecated = "Use `set_clear_value()` instead."] + pub fn clear_value(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_clear_value(val); + self + } + + #[deprecated = "Use `set_depth_slice()` instead."] + pub fn depth_slice(&mut self, val: u32) -> &mut Self { + self.set_depth_slice(val); + self + } + + #[deprecated = "Use `set_load_op()` instead."] + pub fn load_op(&mut self, val: GpuLoadOp) -> &mut Self { + self.set_load_op(val); + self + } + + #[deprecated = "Use `set_resolve_target()` instead."] + pub fn resolve_target(&mut self, val: &GpuTextureView) -> &mut Self { + self.set_resolve_target(val); + self + } + + #[deprecated = "Use `set_store_op()` instead."] + pub fn store_op(&mut self, val: GpuStoreOp) -> &mut Self { + self.set_store_op(val); + self + } + + #[deprecated = "Use `set_view()` instead."] + pub fn view(&mut self, val: &GpuTextureView) -> &mut Self { + self.set_view(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassDepthStencilAttachment.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassDepthStencilAttachment.rs new file mode 100644 index 00000000..41135ecf --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassDepthStencilAttachment.rs @@ -0,0 +1,269 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderPassDepthStencilAttachment)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderPassDepthStencilAttachment` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderPassDepthStencilAttachment; + + #[doc = "Get the `depthClearValue` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthClearValue")] + pub fn get_depth_clear_value(this: &GpuRenderPassDepthStencilAttachment) -> Option; + + #[doc = "Change the `depthClearValue` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthClearValue")] + pub fn set_depth_clear_value(this: &GpuRenderPassDepthStencilAttachment, val: f32); + + #[doc = "Get the `depthLoadOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuLoadOp`, `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthLoadOp")] + pub fn get_depth_load_op(this: &GpuRenderPassDepthStencilAttachment) -> Option; + + #[doc = "Change the `depthLoadOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuLoadOp`, `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthLoadOp")] + pub fn set_depth_load_op(this: &GpuRenderPassDepthStencilAttachment, val: GpuLoadOp); + + #[doc = "Get the `depthReadOnly` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthReadOnly")] + pub fn get_depth_read_only(this: &GpuRenderPassDepthStencilAttachment) -> Option; + + #[doc = "Change the `depthReadOnly` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthReadOnly")] + pub fn set_depth_read_only(this: &GpuRenderPassDepthStencilAttachment, val: bool); + + #[doc = "Get the `depthStoreOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`, `GpuStoreOp`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthStoreOp")] + pub fn get_depth_store_op(this: &GpuRenderPassDepthStencilAttachment) -> Option; + + #[doc = "Change the `depthStoreOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`, `GpuStoreOp`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthStoreOp")] + pub fn set_depth_store_op(this: &GpuRenderPassDepthStencilAttachment, val: GpuStoreOp); + + #[doc = "Get the `stencilClearValue` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stencilClearValue")] + pub fn get_stencil_clear_value(this: &GpuRenderPassDepthStencilAttachment) -> Option; + + #[doc = "Change the `stencilClearValue` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stencilClearValue")] + pub fn set_stencil_clear_value(this: &GpuRenderPassDepthStencilAttachment, val: u32); + + #[doc = "Get the `stencilLoadOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuLoadOp`, `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stencilLoadOp")] + pub fn get_stencil_load_op(this: &GpuRenderPassDepthStencilAttachment) -> Option; + + #[doc = "Change the `stencilLoadOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuLoadOp`, `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stencilLoadOp")] + pub fn set_stencil_load_op(this: &GpuRenderPassDepthStencilAttachment, val: GpuLoadOp); + + #[doc = "Get the `stencilReadOnly` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stencilReadOnly")] + pub fn get_stencil_read_only(this: &GpuRenderPassDepthStencilAttachment) -> Option; + + #[doc = "Change the `stencilReadOnly` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stencilReadOnly")] + pub fn set_stencil_read_only(this: &GpuRenderPassDepthStencilAttachment, val: bool); + + #[doc = "Get the `stencilStoreOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`, `GpuStoreOp`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stencilStoreOp")] + pub fn get_stencil_store_op(this: &GpuRenderPassDepthStencilAttachment) -> Option; + + #[doc = "Change the `stencilStoreOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`, `GpuStoreOp`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stencilStoreOp")] + pub fn set_stencil_store_op(this: &GpuRenderPassDepthStencilAttachment, val: GpuStoreOp); + + #[doc = "Get the `view` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`, `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "view")] + pub fn get_view(this: &GpuRenderPassDepthStencilAttachment) -> GpuTextureView; + + #[doc = "Change the `view` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`, `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "view")] + pub fn set_view(this: &GpuRenderPassDepthStencilAttachment, val: &GpuTextureView); +} + +impl GpuRenderPassDepthStencilAttachment { + #[doc = "Construct a new `GpuRenderPassDepthStencilAttachment`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`, `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(view: &GpuTextureView) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_view(view); + ret + } + + #[deprecated = "Use `set_depth_clear_value()` instead."] + pub fn depth_clear_value(&mut self, val: f32) -> &mut Self { + self.set_depth_clear_value(val); + self + } + + #[deprecated = "Use `set_depth_load_op()` instead."] + pub fn depth_load_op(&mut self, val: GpuLoadOp) -> &mut Self { + self.set_depth_load_op(val); + self + } + + #[deprecated = "Use `set_depth_read_only()` instead."] + pub fn depth_read_only(&mut self, val: bool) -> &mut Self { + self.set_depth_read_only(val); + self + } + + #[deprecated = "Use `set_depth_store_op()` instead."] + pub fn depth_store_op(&mut self, val: GpuStoreOp) -> &mut Self { + self.set_depth_store_op(val); + self + } + + #[deprecated = "Use `set_stencil_clear_value()` instead."] + pub fn stencil_clear_value(&mut self, val: u32) -> &mut Self { + self.set_stencil_clear_value(val); + self + } + + #[deprecated = "Use `set_stencil_load_op()` instead."] + pub fn stencil_load_op(&mut self, val: GpuLoadOp) -> &mut Self { + self.set_stencil_load_op(val); + self + } + + #[deprecated = "Use `set_stencil_read_only()` instead."] + pub fn stencil_read_only(&mut self, val: bool) -> &mut Self { + self.set_stencil_read_only(val); + self + } + + #[deprecated = "Use `set_stencil_store_op()` instead."] + pub fn stencil_store_op(&mut self, val: GpuStoreOp) -> &mut Self { + self.set_stencil_store_op(val); + self + } + + #[deprecated = "Use `set_view()` instead."] + pub fn view(&mut self, val: &GpuTextureView) -> &mut Self { + self.set_view(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassDescriptor.rs new file mode 100644 index 00000000..97cefbe3 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassDescriptor.rs @@ -0,0 +1,207 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderPassDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderPassDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderPassDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuRenderPassDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuRenderPassDescriptor, val: &str); + + #[doc = "Get the `colorAttachments` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "colorAttachments")] + pub fn get_color_attachments(this: &GpuRenderPassDescriptor) -> ::js_sys::Array; + + #[doc = "Change the `colorAttachments` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "colorAttachments")] + pub fn set_color_attachments(this: &GpuRenderPassDescriptor, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `depthStencilAttachment` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`, `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthStencilAttachment")] + pub fn get_depth_stencil_attachment( + this: &GpuRenderPassDescriptor, + ) -> Option; + + #[doc = "Change the `depthStencilAttachment` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDepthStencilAttachment`, `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthStencilAttachment")] + pub fn set_depth_stencil_attachment( + this: &GpuRenderPassDescriptor, + val: &GpuRenderPassDepthStencilAttachment, + ); + + #[doc = "Get the `maxDrawCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "maxDrawCount")] + pub fn get_max_draw_count(this: &GpuRenderPassDescriptor) -> Option; + + #[doc = "Change the `maxDrawCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "maxDrawCount")] + pub fn set_max_draw_count(this: &GpuRenderPassDescriptor, val: f64); + + #[doc = "Get the `occlusionQuerySet` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`, `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "occlusionQuerySet")] + pub fn get_occlusion_query_set(this: &GpuRenderPassDescriptor) -> Option; + + #[doc = "Change the `occlusionQuerySet` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`, `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "occlusionQuerySet")] + pub fn set_occlusion_query_set(this: &GpuRenderPassDescriptor, val: &GpuQuerySet); + + #[doc = "Get the `timestampWrites` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`, `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "timestampWrites")] + pub fn get_timestamp_writes( + this: &GpuRenderPassDescriptor, + ) -> Option; + + #[doc = "Change the `timestampWrites` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`, `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "timestampWrites")] + pub fn set_timestamp_writes(this: &GpuRenderPassDescriptor, val: &GpuRenderPassTimestampWrites); +} + +impl GpuRenderPassDescriptor { + #[doc = "Construct a new `GpuRenderPassDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(color_attachments: &::wasm_bindgen::JsValue) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_color_attachments(color_attachments); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_color_attachments()` instead."] + pub fn color_attachments(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_color_attachments(val); + self + } + + #[deprecated = "Use `set_depth_stencil_attachment()` instead."] + pub fn depth_stencil_attachment( + &mut self, + val: &GpuRenderPassDepthStencilAttachment, + ) -> &mut Self { + self.set_depth_stencil_attachment(val); + self + } + + #[deprecated = "Use `set_max_draw_count()` instead."] + pub fn max_draw_count(&mut self, val: f64) -> &mut Self { + self.set_max_draw_count(val); + self + } + + #[deprecated = "Use `set_occlusion_query_set()` instead."] + pub fn occlusion_query_set(&mut self, val: &GpuQuerySet) -> &mut Self { + self.set_occlusion_query_set(val); + self + } + + #[deprecated = "Use `set_timestamp_writes()` instead."] + pub fn timestamp_writes(&mut self, val: &GpuRenderPassTimestampWrites) -> &mut Self { + self.set_timestamp_writes(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassEncoder.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassEncoder.rs new file mode 100644 index 00000000..7de607c6 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassEncoder.rs @@ -0,0 +1,744 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderPassEncoder , typescript_type = "GPURenderPassEncoder")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderPassEncoder` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderPassEncoder; + + # [wasm_bindgen (structural , method , getter , js_class = "GPURenderPassEncoder" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuRenderPassEncoder) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPURenderPassEncoder" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuRenderPassEncoder, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = beginOcclusionQuery)] + #[doc = "The `beginOcclusionQuery()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/beginOcclusionQuery)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn begin_occlusion_query(this: &GpuRenderPassEncoder, query_index: u32); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = end)] + #[doc = "The `end()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/end)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn end(this: &GpuRenderPassEncoder); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = endOcclusionQuery)] + #[doc = "The `endOcclusionQuery()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/endOcclusionQuery)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn end_occlusion_query(this: &GpuRenderPassEncoder); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = executeBundles)] + #[doc = "The `executeBundles()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/executeBundles)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn execute_bundles(this: &GpuRenderPassEncoder, bundles: &::wasm_bindgen::JsValue); + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderPassEncoder" , js_name = setBlendConstant)] + #[doc = "The `setBlendConstant()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setBlendConstant)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_blend_constant_with_f64_sequence( + this: &GpuRenderPassEncoder, + color: &::wasm_bindgen::JsValue, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderPassEncoder" , js_name = setBlendConstant)] + #[doc = "The `setBlendConstant()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setBlendConstant)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuColorDict`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_blend_constant_with_gpu_color_dict( + this: &GpuRenderPassEncoder, + color: &GpuColorDict, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setScissorRect)] + #[doc = "The `setScissorRect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setScissorRect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_scissor_rect(this: &GpuRenderPassEncoder, x: u32, y: u32, width: u32, height: u32); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setStencilReference)] + #[doc = "The `setStencilReference()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setStencilReference)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_stencil_reference(this: &GpuRenderPassEncoder, reference: u32); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setViewport)] + #[doc = "The `setViewport()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setViewport)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_viewport( + this: &GpuRenderPassEncoder, + x: f32, + y: f32, + width: f32, + height: f32, + min_depth: f32, + max_depth: f32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group( + this: &GpuRenderPassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_sequence( + this: &GpuRenderPassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets: &::wasm_bindgen::JsValue, + ); + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderPassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_slice_and_u32_and_dynamic_offsets_data_length( + this: &GpuRenderPassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &[u32], + dynamic_offsets_data_start: u32, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderPassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_array_and_u32_and_dynamic_offsets_data_length( + this: &GpuRenderPassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &::js_sys::Uint32Array, + dynamic_offsets_data_start: u32, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderPassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_slice_and_f64_and_dynamic_offsets_data_length( + this: &GpuRenderPassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &[u32], + dynamic_offsets_data_start: f64, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (catch , method , structural , js_class = "GPURenderPassEncoder" , js_name = setBindGroup)] + #[doc = "The `setBindGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroup`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_bind_group_with_u32_array_and_f64_and_dynamic_offsets_data_length( + this: &GpuRenderPassEncoder, + index: u32, + bind_group: Option<&GpuBindGroup>, + dynamic_offsets_data: &::js_sys::Uint32Array, + dynamic_offsets_data_start: f64, + dynamic_offsets_data_length: u32, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = insertDebugMarker)] + #[doc = "The `insertDebugMarker()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/insertDebugMarker)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn insert_debug_marker(this: &GpuRenderPassEncoder, marker_label: &str); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = popDebugGroup)] + #[doc = "The `popDebugGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/popDebugGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn pop_debug_group(this: &GpuRenderPassEncoder); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = pushDebugGroup)] + #[doc = "The `pushDebugGroup()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/pushDebugGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn push_debug_group(this: &GpuRenderPassEncoder, group_label: &str); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = draw)] + #[doc = "The `draw()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/draw)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw(this: &GpuRenderPassEncoder, vertex_count: u32); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = draw)] + #[doc = "The `draw()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/draw)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_with_instance_count( + this: &GpuRenderPassEncoder, + vertex_count: u32, + instance_count: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = draw)] + #[doc = "The `draw()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/draw)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_with_instance_count_and_first_vertex( + this: &GpuRenderPassEncoder, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = draw)] + #[doc = "The `draw()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/draw)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_with_instance_count_and_first_vertex_and_first_instance( + this: &GpuRenderPassEncoder, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed(this: &GpuRenderPassEncoder, index_count: u32); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_with_instance_count( + this: &GpuRenderPassEncoder, + index_count: u32, + instance_count: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_with_instance_count_and_first_index( + this: &GpuRenderPassEncoder, + index_count: u32, + instance_count: u32, + first_index: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_with_instance_count_and_first_index_and_base_vertex( + this: &GpuRenderPassEncoder, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = drawIndexed)] + #[doc = "The `drawIndexed()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/drawIndexed)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_with_instance_count_and_first_index_and_base_vertex_and_first_instance( + this: &GpuRenderPassEncoder, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = drawIndexedIndirect)] + #[doc = "The `drawIndexedIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/drawIndexedIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_indirect_with_u32( + this: &GpuRenderPassEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = drawIndexedIndirect)] + #[doc = "The `drawIndexedIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/drawIndexedIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indexed_indirect_with_f64( + this: &GpuRenderPassEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = drawIndirect)] + #[doc = "The `drawIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/drawIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indirect_with_u32( + this: &GpuRenderPassEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = drawIndirect)] + #[doc = "The `drawIndirect()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/drawIndirect)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn draw_indirect_with_f64( + this: &GpuRenderPassEncoder, + indirect_buffer: &GpuBuffer, + indirect_offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer( + this: &GpuRenderPassEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_u32( + this: &GpuRenderPassEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_f64( + this: &GpuRenderPassEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_u32_and_u32( + this: &GpuRenderPassEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: u32, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_f64_and_u32( + this: &GpuRenderPassEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: f64, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_u32_and_f64( + this: &GpuRenderPassEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: u32, + size: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setIndexBuffer)] + #[doc = "The `setIndexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setIndexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuIndexFormat`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_index_buffer_with_f64_and_f64( + this: &GpuRenderPassEncoder, + buffer: &GpuBuffer, + index_format: GpuIndexFormat, + offset: f64, + size: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setPipeline)] + #[doc = "The `setPipeline()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setPipeline)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassEncoder`, `GpuRenderPipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_pipeline(this: &GpuRenderPassEncoder, pipeline: &GpuRenderPipeline); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer(this: &GpuRenderPassEncoder, slot: u32, buffer: Option<&GpuBuffer>); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_u32( + this: &GpuRenderPassEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_f64( + this: &GpuRenderPassEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_u32_and_u32( + this: &GpuRenderPassEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: u32, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_f64_and_u32( + this: &GpuRenderPassEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: f64, + size: u32, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_u32_and_f64( + this: &GpuRenderPassEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: u32, + size: f64, + ); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPassEncoder" , js_name = setVertexBuffer)] + #[doc = "The `setVertexBuffer()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPassEncoder/setVertexBuffer)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuRenderPassEncoder`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_vertex_buffer_with_f64_and_f64( + this: &GpuRenderPassEncoder, + slot: u32, + buffer: Option<&GpuBuffer>, + offset: f64, + size: f64, + ); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassTimestampWrites.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassTimestampWrites.rs new file mode 100644 index 00000000..3d6b4d1e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPassTimestampWrites.rs @@ -0,0 +1,125 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderPassTimestampWrites)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderPassTimestampWrites` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderPassTimestampWrites; + + #[doc = "Get the `beginningOfPassWriteIndex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "beginningOfPassWriteIndex")] + pub fn get_beginning_of_pass_write_index(this: &GpuRenderPassTimestampWrites) -> Option; + + #[doc = "Change the `beginningOfPassWriteIndex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "beginningOfPassWriteIndex")] + pub fn set_beginning_of_pass_write_index(this: &GpuRenderPassTimestampWrites, val: u32); + + #[doc = "Get the `endOfPassWriteIndex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "endOfPassWriteIndex")] + pub fn get_end_of_pass_write_index(this: &GpuRenderPassTimestampWrites) -> Option; + + #[doc = "Change the `endOfPassWriteIndex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "endOfPassWriteIndex")] + pub fn set_end_of_pass_write_index(this: &GpuRenderPassTimestampWrites, val: u32); + + #[doc = "Get the `querySet` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`, `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "querySet")] + pub fn get_query_set(this: &GpuRenderPassTimestampWrites) -> GpuQuerySet; + + #[doc = "Change the `querySet` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`, `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "querySet")] + pub fn set_query_set(this: &GpuRenderPassTimestampWrites, val: &GpuQuerySet); +} + +impl GpuRenderPassTimestampWrites { + #[doc = "Construct a new `GpuRenderPassTimestampWrites`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuQuerySet`, `GpuRenderPassTimestampWrites`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(query_set: &GpuQuerySet) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_query_set(query_set); + ret + } + + #[deprecated = "Use `set_beginning_of_pass_write_index()` instead."] + pub fn beginning_of_pass_write_index(&mut self, val: u32) -> &mut Self { + self.set_beginning_of_pass_write_index(val); + self + } + + #[deprecated = "Use `set_end_of_pass_write_index()` instead."] + pub fn end_of_pass_write_index(&mut self, val: u32) -> &mut Self { + self.set_end_of_pass_write_index(val); + self + } + + #[deprecated = "Use `set_query_set()` instead."] + pub fn query_set(&mut self, val: &GpuQuerySet) -> &mut Self { + self.set_query_set(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPipeline.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPipeline.rs new file mode 100644 index 00000000..795bc24d --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPipeline.rs @@ -0,0 +1,73 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderPipeline , typescript_type = "GPURenderPipeline")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderPipeline` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPipeline)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderPipeline; + + # [wasm_bindgen (structural , method , getter , js_class = "GPURenderPipeline" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPipeline/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuRenderPipeline) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPURenderPipeline" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPipeline/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuRenderPipeline, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPURenderPipeline" , js_name = getBindGroupLayout)] + #[doc = "The `getBindGroupLayout()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPURenderPipeline/getBindGroupLayout)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBindGroupLayout`, `GpuRenderPipeline`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_bind_group_layout(this: &GpuRenderPipeline, index: u32) -> GpuBindGroupLayout; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPipelineDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPipelineDescriptor.rs new file mode 100644 index 00000000..a0f2875b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRenderPipelineDescriptor.rs @@ -0,0 +1,222 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURenderPipelineDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRenderPipelineDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRenderPipelineDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuRenderPipelineDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuRenderPipelineDescriptor, val: &str); + + #[doc = "Get the `layout` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "layout")] + pub fn get_layout(this: &GpuRenderPipelineDescriptor) -> ::wasm_bindgen::JsValue; + + #[doc = "Change the `layout` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "layout")] + pub fn set_layout(this: &GpuRenderPipelineDescriptor, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `depthStencil` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthStencil")] + pub fn get_depth_stencil(this: &GpuRenderPipelineDescriptor) -> Option; + + #[doc = "Change the `depthStencil` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuDepthStencilState`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthStencil")] + pub fn set_depth_stencil(this: &GpuRenderPipelineDescriptor, val: &GpuDepthStencilState); + + #[doc = "Get the `fragment` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "fragment")] + pub fn get_fragment(this: &GpuRenderPipelineDescriptor) -> Option; + + #[doc = "Change the `fragment` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFragmentState`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "fragment")] + pub fn set_fragment(this: &GpuRenderPipelineDescriptor, val: &GpuFragmentState); + + #[doc = "Get the `multisample` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "multisample")] + pub fn get_multisample(this: &GpuRenderPipelineDescriptor) -> Option; + + #[doc = "Change the `multisample` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMultisampleState`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "multisample")] + pub fn set_multisample(this: &GpuRenderPipelineDescriptor, val: &GpuMultisampleState); + + #[doc = "Get the `primitive` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPrimitiveState`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "primitive")] + pub fn get_primitive(this: &GpuRenderPipelineDescriptor) -> Option; + + #[doc = "Change the `primitive` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPrimitiveState`, `GpuRenderPipelineDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "primitive")] + pub fn set_primitive(this: &GpuRenderPipelineDescriptor, val: &GpuPrimitiveState); + + #[doc = "Get the `vertex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipelineDescriptor`, `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "vertex")] + pub fn get_vertex(this: &GpuRenderPipelineDescriptor) -> GpuVertexState; + + #[doc = "Change the `vertex` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipelineDescriptor`, `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "vertex")] + pub fn set_vertex(this: &GpuRenderPipelineDescriptor, val: &GpuVertexState); +} + +impl GpuRenderPipelineDescriptor { + #[doc = "Construct a new `GpuRenderPipelineDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRenderPipelineDescriptor`, `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(layout: &::wasm_bindgen::JsValue, vertex: &GpuVertexState) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_layout(layout); + ret.set_vertex(vertex); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_layout()` instead."] + pub fn layout(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_layout(val); + self + } + + #[deprecated = "Use `set_depth_stencil()` instead."] + pub fn depth_stencil(&mut self, val: &GpuDepthStencilState) -> &mut Self { + self.set_depth_stencil(val); + self + } + + #[deprecated = "Use `set_fragment()` instead."] + pub fn fragment(&mut self, val: &GpuFragmentState) -> &mut Self { + self.set_fragment(val); + self + } + + #[deprecated = "Use `set_multisample()` instead."] + pub fn multisample(&mut self, val: &GpuMultisampleState) -> &mut Self { + self.set_multisample(val); + self + } + + #[deprecated = "Use `set_primitive()` instead."] + pub fn primitive(&mut self, val: &GpuPrimitiveState) -> &mut Self { + self.set_primitive(val); + self + } + + #[deprecated = "Use `set_vertex()` instead."] + pub fn vertex(&mut self, val: &GpuVertexState) -> &mut Self { + self.set_vertex(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRequestAdapterOptions.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRequestAdapterOptions.rs new file mode 100644 index 00000000..3543ad8e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuRequestAdapterOptions.rs @@ -0,0 +1,154 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPURequestAdapterOptions)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuRequestAdapterOptions` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuRequestAdapterOptions; + + #[doc = "Get the `featureLevel` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "featureLevel")] + pub fn get_feature_level(this: &GpuRequestAdapterOptions) -> Option<::alloc::string::String>; + + #[doc = "Change the `featureLevel` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "featureLevel")] + pub fn set_feature_level(this: &GpuRequestAdapterOptions, val: &str); + + #[doc = "Get the `forceFallbackAdapter` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "forceFallbackAdapter")] + pub fn get_force_fallback_adapter(this: &GpuRequestAdapterOptions) -> Option; + + #[doc = "Change the `forceFallbackAdapter` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "forceFallbackAdapter")] + pub fn set_force_fallback_adapter(this: &GpuRequestAdapterOptions, val: bool); + + #[doc = "Get the `powerPreference` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPowerPreference`, `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "powerPreference")] + pub fn get_power_preference(this: &GpuRequestAdapterOptions) -> Option; + + #[doc = "Change the `powerPreference` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuPowerPreference`, `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "powerPreference")] + pub fn set_power_preference(this: &GpuRequestAdapterOptions, val: GpuPowerPreference); + + #[doc = "Get the `xrCompatible` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "xrCompatible")] + pub fn get_xr_compatible(this: &GpuRequestAdapterOptions) -> Option; + + #[doc = "Change the `xrCompatible` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "xrCompatible")] + pub fn set_xr_compatible(this: &GpuRequestAdapterOptions, val: bool); +} + +impl GpuRequestAdapterOptions { + #[doc = "Construct a new `GpuRequestAdapterOptions`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuRequestAdapterOptions`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_feature_level()` instead."] + pub fn feature_level(&mut self, val: &str) -> &mut Self { + self.set_feature_level(val); + self + } + + #[deprecated = "Use `set_force_fallback_adapter()` instead."] + pub fn force_fallback_adapter(&mut self, val: bool) -> &mut Self { + self.set_force_fallback_adapter(val); + self + } + + #[deprecated = "Use `set_power_preference()` instead."] + pub fn power_preference(&mut self, val: GpuPowerPreference) -> &mut Self { + self.set_power_preference(val); + self + } + + #[deprecated = "Use `set_xr_compatible()` instead."] + pub fn xr_compatible(&mut self, val: bool) -> &mut Self { + self.set_xr_compatible(val); + self + } +} + +impl Default for GpuRequestAdapterOptions { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSampler.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSampler.rs new file mode 100644 index 00000000..5f88d524 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSampler.rs @@ -0,0 +1,62 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUSampler , typescript_type = "GPUSampler")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuSampler` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSampler)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSampler`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuSampler; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSampler" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSampler/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSampler`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuSampler) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUSampler" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSampler/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSampler`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuSampler, value: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerBindingLayout.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerBindingLayout.rs new file mode 100644 index 00000000..b14f2901 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerBindingLayout.rs @@ -0,0 +1,82 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUSamplerBindingLayout)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuSamplerBindingLayout` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuSamplerBindingLayout; + + #[doc = "Get the `type` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerBindingLayout`, `GpuSamplerBindingType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "type")] + pub fn get_type(this: &GpuSamplerBindingLayout) -> Option; + + #[doc = "Change the `type` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerBindingLayout`, `GpuSamplerBindingType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "type")] + pub fn set_type(this: &GpuSamplerBindingLayout, val: GpuSamplerBindingType); +} + +impl GpuSamplerBindingLayout { + #[doc = "Construct a new `GpuSamplerBindingLayout`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_type()` instead."] + pub fn type_(&mut self, val: GpuSamplerBindingType) -> &mut Self { + self.set_type(val); + self + } +} + +impl Default for GpuSamplerBindingLayout { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerBindingType.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerBindingType.rs new file mode 100644 index 00000000..6cebb038 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerBindingType.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuSamplerBindingType` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuSamplerBindingType`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuSamplerBindingType { + Filtering = "filtering", + NonFiltering = "non-filtering", + Comparison = "comparison", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerDescriptor.rs new file mode 100644 index 00000000..3cb3237d --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSamplerDescriptor.rs @@ -0,0 +1,322 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUSamplerDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuSamplerDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuSamplerDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuSamplerDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuSamplerDescriptor, val: &str); + + #[doc = "Get the `addressModeU` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAddressMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "addressModeU")] + pub fn get_address_mode_u(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `addressModeU` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAddressMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "addressModeU")] + pub fn set_address_mode_u(this: &GpuSamplerDescriptor, val: GpuAddressMode); + + #[doc = "Get the `addressModeV` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAddressMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "addressModeV")] + pub fn get_address_mode_v(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `addressModeV` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAddressMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "addressModeV")] + pub fn set_address_mode_v(this: &GpuSamplerDescriptor, val: GpuAddressMode); + + #[doc = "Get the `addressModeW` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAddressMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "addressModeW")] + pub fn get_address_mode_w(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `addressModeW` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuAddressMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "addressModeW")] + pub fn set_address_mode_w(this: &GpuSamplerDescriptor, val: GpuAddressMode); + + #[doc = "Get the `compare` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompareFunction`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "compare")] + pub fn get_compare(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `compare` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompareFunction`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "compare")] + pub fn set_compare(this: &GpuSamplerDescriptor, val: GpuCompareFunction); + + #[doc = "Get the `lodMaxClamp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "lodMaxClamp")] + pub fn get_lod_max_clamp(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `lodMaxClamp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "lodMaxClamp")] + pub fn set_lod_max_clamp(this: &GpuSamplerDescriptor, val: f32); + + #[doc = "Get the `lodMinClamp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "lodMinClamp")] + pub fn get_lod_min_clamp(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `lodMinClamp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "lodMinClamp")] + pub fn set_lod_min_clamp(this: &GpuSamplerDescriptor, val: f32); + + #[doc = "Get the `magFilter` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFilterMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "magFilter")] + pub fn get_mag_filter(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `magFilter` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFilterMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "magFilter")] + pub fn set_mag_filter(this: &GpuSamplerDescriptor, val: GpuFilterMode); + + #[doc = "Get the `maxAnisotropy` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "maxAnisotropy")] + pub fn get_max_anisotropy(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `maxAnisotropy` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "maxAnisotropy")] + pub fn set_max_anisotropy(this: &GpuSamplerDescriptor, val: u16); + + #[doc = "Get the `minFilter` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFilterMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "minFilter")] + pub fn get_min_filter(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `minFilter` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuFilterMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "minFilter")] + pub fn set_min_filter(this: &GpuSamplerDescriptor, val: GpuFilterMode); + + #[doc = "Get the `mipmapFilter` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMipmapFilterMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "mipmapFilter")] + pub fn get_mipmap_filter(this: &GpuSamplerDescriptor) -> Option; + + #[doc = "Change the `mipmapFilter` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuMipmapFilterMode`, `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "mipmapFilter")] + pub fn set_mipmap_filter(this: &GpuSamplerDescriptor, val: GpuMipmapFilterMode); +} + +impl GpuSamplerDescriptor { + #[doc = "Construct a new `GpuSamplerDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSamplerDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_address_mode_u()` instead."] + pub fn address_mode_u(&mut self, val: GpuAddressMode) -> &mut Self { + self.set_address_mode_u(val); + self + } + + #[deprecated = "Use `set_address_mode_v()` instead."] + pub fn address_mode_v(&mut self, val: GpuAddressMode) -> &mut Self { + self.set_address_mode_v(val); + self + } + + #[deprecated = "Use `set_address_mode_w()` instead."] + pub fn address_mode_w(&mut self, val: GpuAddressMode) -> &mut Self { + self.set_address_mode_w(val); + self + } + + #[deprecated = "Use `set_compare()` instead."] + pub fn compare(&mut self, val: GpuCompareFunction) -> &mut Self { + self.set_compare(val); + self + } + + #[deprecated = "Use `set_lod_max_clamp()` instead."] + pub fn lod_max_clamp(&mut self, val: f32) -> &mut Self { + self.set_lod_max_clamp(val); + self + } + + #[deprecated = "Use `set_lod_min_clamp()` instead."] + pub fn lod_min_clamp(&mut self, val: f32) -> &mut Self { + self.set_lod_min_clamp(val); + self + } + + #[deprecated = "Use `set_mag_filter()` instead."] + pub fn mag_filter(&mut self, val: GpuFilterMode) -> &mut Self { + self.set_mag_filter(val); + self + } + + #[deprecated = "Use `set_max_anisotropy()` instead."] + pub fn max_anisotropy(&mut self, val: u16) -> &mut Self { + self.set_max_anisotropy(val); + self + } + + #[deprecated = "Use `set_min_filter()` instead."] + pub fn min_filter(&mut self, val: GpuFilterMode) -> &mut Self { + self.set_min_filter(val); + self + } + + #[deprecated = "Use `set_mipmap_filter()` instead."] + pub fn mipmap_filter(&mut self, val: GpuMipmapFilterMode) -> &mut Self { + self.set_mipmap_filter(val); + self + } +} + +impl Default for GpuSamplerDescriptor { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuShaderModule.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuShaderModule.rs new file mode 100644 index 00000000..bbebbb42 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuShaderModule.rs @@ -0,0 +1,73 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUShaderModule , typescript_type = "GPUShaderModule")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuShaderModule` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUShaderModule)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuShaderModule; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUShaderModule" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUShaderModule/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuShaderModule) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUShaderModule" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUShaderModule/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuShaderModule, value: &str); + + # [wasm_bindgen (method , structural , js_class = "GPUShaderModule" , js_name = getCompilationInfo)] + #[doc = "The `getCompilationInfo()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUShaderModule/getCompilationInfo)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModule`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn get_compilation_info(this: &GpuShaderModule) -> ::js_sys::Promise; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuShaderModuleDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuShaderModuleDescriptor.rs new file mode 100644 index 00000000..e0d08610 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuShaderModuleDescriptor.rs @@ -0,0 +1,125 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUShaderModuleDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuShaderModuleDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModuleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuShaderModuleDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModuleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuShaderModuleDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModuleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuShaderModuleDescriptor, val: &str); + + #[doc = "Get the `code` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModuleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "code")] + pub fn get_code(this: &GpuShaderModuleDescriptor) -> ::alloc::string::String; + + #[doc = "Change the `code` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModuleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "code")] + pub fn set_code(this: &GpuShaderModuleDescriptor, val: &str); + + #[doc = "Get the `compilationHints` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModuleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "compilationHints")] + pub fn get_compilation_hints(this: &GpuShaderModuleDescriptor) -> Option<::js_sys::Array>; + + #[doc = "Change the `compilationHints` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModuleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "compilationHints")] + pub fn set_compilation_hints(this: &GpuShaderModuleDescriptor, val: &::wasm_bindgen::JsValue); +} + +impl GpuShaderModuleDescriptor { + #[doc = "Construct a new `GpuShaderModuleDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModuleDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(code: &str) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_code(code); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_code()` instead."] + pub fn code(&mut self, val: &str) -> &mut Self { + self.set_code(val); + self + } + + #[deprecated = "Use `set_compilation_hints()` instead."] + pub fn compilation_hints(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_compilation_hints(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStencilFaceState.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStencilFaceState.rs new file mode 100644 index 00000000..96fedd78 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStencilFaceState.rs @@ -0,0 +1,154 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUStencilFaceState)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuStencilFaceState` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStencilFaceState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuStencilFaceState; + + #[doc = "Get the `compare` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompareFunction`, `GpuStencilFaceState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "compare")] + pub fn get_compare(this: &GpuStencilFaceState) -> Option; + + #[doc = "Change the `compare` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuCompareFunction`, `GpuStencilFaceState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "compare")] + pub fn set_compare(this: &GpuStencilFaceState, val: GpuCompareFunction); + + #[doc = "Get the `depthFailOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStencilFaceState`, `GpuStencilOperation`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "depthFailOp")] + pub fn get_depth_fail_op(this: &GpuStencilFaceState) -> Option; + + #[doc = "Change the `depthFailOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStencilFaceState`, `GpuStencilOperation`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "depthFailOp")] + pub fn set_depth_fail_op(this: &GpuStencilFaceState, val: GpuStencilOperation); + + #[doc = "Get the `failOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStencilFaceState`, `GpuStencilOperation`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "failOp")] + pub fn get_fail_op(this: &GpuStencilFaceState) -> Option; + + #[doc = "Change the `failOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStencilFaceState`, `GpuStencilOperation`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "failOp")] + pub fn set_fail_op(this: &GpuStencilFaceState, val: GpuStencilOperation); + + #[doc = "Get the `passOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStencilFaceState`, `GpuStencilOperation`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "passOp")] + pub fn get_pass_op(this: &GpuStencilFaceState) -> Option; + + #[doc = "Change the `passOp` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStencilFaceState`, `GpuStencilOperation`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "passOp")] + pub fn set_pass_op(this: &GpuStencilFaceState, val: GpuStencilOperation); +} + +impl GpuStencilFaceState { + #[doc = "Construct a new `GpuStencilFaceState`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStencilFaceState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_compare()` instead."] + pub fn compare(&mut self, val: GpuCompareFunction) -> &mut Self { + self.set_compare(val); + self + } + + #[deprecated = "Use `set_depth_fail_op()` instead."] + pub fn depth_fail_op(&mut self, val: GpuStencilOperation) -> &mut Self { + self.set_depth_fail_op(val); + self + } + + #[deprecated = "Use `set_fail_op()` instead."] + pub fn fail_op(&mut self, val: GpuStencilOperation) -> &mut Self { + self.set_fail_op(val); + self + } + + #[deprecated = "Use `set_pass_op()` instead."] + pub fn pass_op(&mut self, val: GpuStencilOperation) -> &mut Self { + self.set_pass_op(val); + self + } +} + +impl Default for GpuStencilFaceState { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStencilOperation.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStencilOperation.rs new file mode 100644 index 00000000..f5137d89 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStencilOperation.rs @@ -0,0 +1,43 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuStencilOperation` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuStencilOperation`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuStencilOperation { + Keep = "keep", + Zero = "zero", + Replace = "replace", + Invert = "invert", + IncrementClamp = "increment-clamp", + DecrementClamp = "decrement-clamp", + IncrementWrap = "increment-wrap", + DecrementWrap = "decrement-wrap", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStorageTextureAccess.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStorageTextureAccess.rs new file mode 100644 index 00000000..43499d6d --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStorageTextureAccess.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuStorageTextureAccess` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuStorageTextureAccess`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuStorageTextureAccess { + WriteOnly = "write-only", + ReadOnly = "read-only", + ReadWrite = "read-write", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStorageTextureBindingLayout.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStorageTextureBindingLayout.rs new file mode 100644 index 00000000..f694316e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStorageTextureBindingLayout.rs @@ -0,0 +1,127 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUStorageTextureBindingLayout)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuStorageTextureBindingLayout` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStorageTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuStorageTextureBindingLayout; + + #[doc = "Get the `access` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStorageTextureAccess`, `GpuStorageTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "access")] + pub fn get_access(this: &GpuStorageTextureBindingLayout) -> Option; + + #[doc = "Change the `access` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStorageTextureAccess`, `GpuStorageTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "access")] + pub fn set_access(this: &GpuStorageTextureBindingLayout, val: GpuStorageTextureAccess); + + #[doc = "Get the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStorageTextureBindingLayout`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "format")] + pub fn get_format(this: &GpuStorageTextureBindingLayout) -> GpuTextureFormat; + + #[doc = "Change the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStorageTextureBindingLayout`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "format")] + pub fn set_format(this: &GpuStorageTextureBindingLayout, val: GpuTextureFormat); + + #[doc = "Get the `viewDimension` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStorageTextureBindingLayout`, `GpuTextureViewDimension`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "viewDimension")] + pub fn get_view_dimension( + this: &GpuStorageTextureBindingLayout, + ) -> Option; + + #[doc = "Change the `viewDimension` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStorageTextureBindingLayout`, `GpuTextureViewDimension`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "viewDimension")] + pub fn set_view_dimension(this: &GpuStorageTextureBindingLayout, val: GpuTextureViewDimension); +} + +impl GpuStorageTextureBindingLayout { + #[doc = "Construct a new `GpuStorageTextureBindingLayout`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuStorageTextureBindingLayout`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(format: GpuTextureFormat) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_format(format); + ret + } + + #[deprecated = "Use `set_access()` instead."] + pub fn access(&mut self, val: GpuStorageTextureAccess) -> &mut Self { + self.set_access(val); + self + } + + #[deprecated = "Use `set_format()` instead."] + pub fn format(&mut self, val: GpuTextureFormat) -> &mut Self { + self.set_format(val); + self + } + + #[deprecated = "Use `set_view_dimension()` instead."] + pub fn view_dimension(&mut self, val: GpuTextureViewDimension) -> &mut Self { + self.set_view_dimension(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStoreOp.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStoreOp.rs new file mode 100644 index 00000000..f32edb3f --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuStoreOp.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuStoreOp` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuStoreOp`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuStoreOp { + Store = "store", + Discard = "discard", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSupportedFeatures.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSupportedFeatures.rs new file mode 100644 index 00000000..fc70e2f1 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSupportedFeatures.rs @@ -0,0 +1,109 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUSupportedFeatures , typescript_type = "GPUSupportedFeatures")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuSupportedFeatures` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedFeatures)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuSupportedFeatures; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedFeatures" , js_name = size)] + #[doc = "Getter for the `size` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedFeatures/size)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn size(this: &GpuSupportedFeatures) -> u32; + + # [wasm_bindgen (method , structural , js_class = "GPUSupportedFeatures" , js_name = entries)] + #[doc = "The `entries()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedFeatures/entries)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn entries(this: &GpuSupportedFeatures) -> ::js_sys::Iterator; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUSupportedFeatures" , js_name = forEach)] + #[doc = "The `forEach()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedFeatures/forEach)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn for_each( + this: &GpuSupportedFeatures, + callback: &::js_sys::Function, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (method , structural , js_class = "GPUSupportedFeatures" , js_name = has)] + #[doc = "The `has()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedFeatures/has)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn has(this: &GpuSupportedFeatures, value: &str) -> bool; + + # [wasm_bindgen (method , structural , js_class = "GPUSupportedFeatures" , js_name = keys)] + #[doc = "The `keys()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedFeatures/keys)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn keys(this: &GpuSupportedFeatures) -> ::js_sys::Iterator; + + # [wasm_bindgen (method , structural , js_class = "GPUSupportedFeatures" , js_name = values)] + #[doc = "The `values()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedFeatures/values)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn values(this: &GpuSupportedFeatures) -> ::js_sys::Iterator; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSupportedLimits.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSupportedLimits.rs new file mode 100644 index 00000000..8323fe6e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuSupportedLimits.rs @@ -0,0 +1,381 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUSupportedLimits , typescript_type = "GPUSupportedLimits")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuSupportedLimits` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuSupportedLimits; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxTextureDimension1D)] + #[doc = "Getter for the `maxTextureDimension1D` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxTextureDimension1D)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_texture_dimension_1d(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxTextureDimension2D)] + #[doc = "Getter for the `maxTextureDimension2D` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxTextureDimension2D)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_texture_dimension_2d(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxTextureDimension3D)] + #[doc = "Getter for the `maxTextureDimension3D` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxTextureDimension3D)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_texture_dimension_3d(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxTextureArrayLayers)] + #[doc = "Getter for the `maxTextureArrayLayers` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxTextureArrayLayers)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_texture_array_layers(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxBindGroups)] + #[doc = "Getter for the `maxBindGroups` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxBindGroups)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_bind_groups(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxBindGroupsPlusVertexBuffers)] + #[doc = "Getter for the `maxBindGroupsPlusVertexBuffers` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxBindGroupsPlusVertexBuffers)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_bind_groups_plus_vertex_buffers(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxBindingsPerBindGroup)] + #[doc = "Getter for the `maxBindingsPerBindGroup` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxBindingsPerBindGroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_bindings_per_bind_group(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxDynamicUniformBuffersPerPipelineLayout)] + #[doc = "Getter for the `maxDynamicUniformBuffersPerPipelineLayout` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxDynamicUniformBuffersPerPipelineLayout)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_dynamic_uniform_buffers_per_pipeline_layout(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxDynamicStorageBuffersPerPipelineLayout)] + #[doc = "Getter for the `maxDynamicStorageBuffersPerPipelineLayout` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxDynamicStorageBuffersPerPipelineLayout)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_dynamic_storage_buffers_per_pipeline_layout(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxSampledTexturesPerShaderStage)] + #[doc = "Getter for the `maxSampledTexturesPerShaderStage` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxSampledTexturesPerShaderStage)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_sampled_textures_per_shader_stage(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxSamplersPerShaderStage)] + #[doc = "Getter for the `maxSamplersPerShaderStage` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxSamplersPerShaderStage)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_samplers_per_shader_stage(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxStorageBuffersPerShaderStage)] + #[doc = "Getter for the `maxStorageBuffersPerShaderStage` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxStorageBuffersPerShaderStage)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_storage_buffers_per_shader_stage(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxStorageTexturesPerShaderStage)] + #[doc = "Getter for the `maxStorageTexturesPerShaderStage` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxStorageTexturesPerShaderStage)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_storage_textures_per_shader_stage(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxUniformBuffersPerShaderStage)] + #[doc = "Getter for the `maxUniformBuffersPerShaderStage` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxUniformBuffersPerShaderStage)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_uniform_buffers_per_shader_stage(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxUniformBufferBindingSize)] + #[doc = "Getter for the `maxUniformBufferBindingSize` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxUniformBufferBindingSize)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_uniform_buffer_binding_size(this: &GpuSupportedLimits) -> f64; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxStorageBufferBindingSize)] + #[doc = "Getter for the `maxStorageBufferBindingSize` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxStorageBufferBindingSize)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_storage_buffer_binding_size(this: &GpuSupportedLimits) -> f64; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = minUniformBufferOffsetAlignment)] + #[doc = "Getter for the `minUniformBufferOffsetAlignment` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/minUniformBufferOffsetAlignment)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn min_uniform_buffer_offset_alignment(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = minStorageBufferOffsetAlignment)] + #[doc = "Getter for the `minStorageBufferOffsetAlignment` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/minStorageBufferOffsetAlignment)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn min_storage_buffer_offset_alignment(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxVertexBuffers)] + #[doc = "Getter for the `maxVertexBuffers` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxVertexBuffers)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_vertex_buffers(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxBufferSize)] + #[doc = "Getter for the `maxBufferSize` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxBufferSize)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_buffer_size(this: &GpuSupportedLimits) -> f64; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxVertexAttributes)] + #[doc = "Getter for the `maxVertexAttributes` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxVertexAttributes)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_vertex_attributes(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxVertexBufferArrayStride)] + #[doc = "Getter for the `maxVertexBufferArrayStride` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxVertexBufferArrayStride)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_vertex_buffer_array_stride(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxInterStageShaderVariables)] + #[doc = "Getter for the `maxInterStageShaderVariables` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxInterStageShaderVariables)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_inter_stage_shader_variables(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxColorAttachments)] + #[doc = "Getter for the `maxColorAttachments` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxColorAttachments)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_color_attachments(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxColorAttachmentBytesPerSample)] + #[doc = "Getter for the `maxColorAttachmentBytesPerSample` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxColorAttachmentBytesPerSample)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_color_attachment_bytes_per_sample(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxComputeWorkgroupStorageSize)] + #[doc = "Getter for the `maxComputeWorkgroupStorageSize` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxComputeWorkgroupStorageSize)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_compute_workgroup_storage_size(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxComputeInvocationsPerWorkgroup)] + #[doc = "Getter for the `maxComputeInvocationsPerWorkgroup` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxComputeInvocationsPerWorkgroup)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_compute_invocations_per_workgroup(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxComputeWorkgroupSizeX)] + #[doc = "Getter for the `maxComputeWorkgroupSizeX` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxComputeWorkgroupSizeX)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_compute_workgroup_size_x(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxComputeWorkgroupSizeY)] + #[doc = "Getter for the `maxComputeWorkgroupSizeY` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxComputeWorkgroupSizeY)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_compute_workgroup_size_y(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxComputeWorkgroupSizeZ)] + #[doc = "Getter for the `maxComputeWorkgroupSizeZ` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxComputeWorkgroupSizeZ)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_compute_workgroup_size_z(this: &GpuSupportedLimits) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUSupportedLimits" , js_name = maxComputeWorkgroupsPerDimension)] + #[doc = "Getter for the `maxComputeWorkgroupsPerDimension` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUSupportedLimits/maxComputeWorkgroupsPerDimension)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuSupportedLimits`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn max_compute_workgroups_per_dimension(this: &GpuSupportedLimits) -> u32; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyBufferInfo.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyBufferInfo.rs new file mode 100644 index 00000000..dad3f24b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyBufferInfo.rs @@ -0,0 +1,149 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUTexelCopyBufferInfo)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuTexelCopyBufferInfo` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuTexelCopyBufferInfo; + + #[doc = "Get the `bytesPerRow` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "bytesPerRow")] + pub fn get_bytes_per_row(this: &GpuTexelCopyBufferInfo) -> Option; + + #[doc = "Change the `bytesPerRow` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "bytesPerRow")] + pub fn set_bytes_per_row(this: &GpuTexelCopyBufferInfo, val: u32); + + #[doc = "Get the `offset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "offset")] + pub fn get_offset(this: &GpuTexelCopyBufferInfo) -> Option; + + #[doc = "Change the `offset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "offset")] + pub fn set_offset(this: &GpuTexelCopyBufferInfo, val: f64); + + #[doc = "Get the `rowsPerImage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "rowsPerImage")] + pub fn get_rows_per_image(this: &GpuTexelCopyBufferInfo) -> Option; + + #[doc = "Change the `rowsPerImage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "rowsPerImage")] + pub fn set_rows_per_image(this: &GpuTexelCopyBufferInfo, val: u32); + + #[doc = "Get the `buffer` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "buffer")] + pub fn get_buffer(this: &GpuTexelCopyBufferInfo) -> GpuBuffer; + + #[doc = "Change the `buffer` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "buffer")] + pub fn set_buffer(this: &GpuTexelCopyBufferInfo, val: &GpuBuffer); +} + +impl GpuTexelCopyBufferInfo { + #[doc = "Construct a new `GpuTexelCopyBufferInfo`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuBuffer`, `GpuTexelCopyBufferInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(buffer: &GpuBuffer) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_buffer(buffer); + ret + } + + #[deprecated = "Use `set_bytes_per_row()` instead."] + pub fn bytes_per_row(&mut self, val: u32) -> &mut Self { + self.set_bytes_per_row(val); + self + } + + #[deprecated = "Use `set_offset()` instead."] + pub fn offset(&mut self, val: f64) -> &mut Self { + self.set_offset(val); + self + } + + #[deprecated = "Use `set_rows_per_image()` instead."] + pub fn rows_per_image(&mut self, val: u32) -> &mut Self { + self.set_rows_per_image(val); + self + } + + #[deprecated = "Use `set_buffer()` instead."] + pub fn buffer(&mut self, val: &GpuBuffer) -> &mut Self { + self.set_buffer(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyBufferLayout.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyBufferLayout.rs new file mode 100644 index 00000000..26f9a4f3 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyBufferLayout.rs @@ -0,0 +1,130 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUTexelCopyBufferLayout)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuTexelCopyBufferLayout` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuTexelCopyBufferLayout; + + #[doc = "Get the `bytesPerRow` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "bytesPerRow")] + pub fn get_bytes_per_row(this: &GpuTexelCopyBufferLayout) -> Option; + + #[doc = "Change the `bytesPerRow` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "bytesPerRow")] + pub fn set_bytes_per_row(this: &GpuTexelCopyBufferLayout, val: u32); + + #[doc = "Get the `offset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "offset")] + pub fn get_offset(this: &GpuTexelCopyBufferLayout) -> Option; + + #[doc = "Change the `offset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "offset")] + pub fn set_offset(this: &GpuTexelCopyBufferLayout, val: f64); + + #[doc = "Get the `rowsPerImage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "rowsPerImage")] + pub fn get_rows_per_image(this: &GpuTexelCopyBufferLayout) -> Option; + + #[doc = "Change the `rowsPerImage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "rowsPerImage")] + pub fn set_rows_per_image(this: &GpuTexelCopyBufferLayout, val: u32); +} + +impl GpuTexelCopyBufferLayout { + #[doc = "Construct a new `GpuTexelCopyBufferLayout`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_bytes_per_row()` instead."] + pub fn bytes_per_row(&mut self, val: u32) -> &mut Self { + self.set_bytes_per_row(val); + self + } + + #[deprecated = "Use `set_offset()` instead."] + pub fn offset(&mut self, val: f64) -> &mut Self { + self.set_offset(val); + self + } + + #[deprecated = "Use `set_rows_per_image()` instead."] + pub fn rows_per_image(&mut self, val: u32) -> &mut Self { + self.set_rows_per_image(val); + self + } +} + +impl Default for GpuTexelCopyBufferLayout { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyTextureInfo.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyTextureInfo.rs new file mode 100644 index 00000000..e948503d --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexelCopyTextureInfo.rs @@ -0,0 +1,149 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUTexelCopyTextureInfo)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuTexelCopyTextureInfo` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuTexelCopyTextureInfo; + + #[doc = "Get the `aspect` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`, `GpuTextureAspect`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "aspect")] + pub fn get_aspect(this: &GpuTexelCopyTextureInfo) -> Option; + + #[doc = "Change the `aspect` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`, `GpuTextureAspect`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "aspect")] + pub fn set_aspect(this: &GpuTexelCopyTextureInfo, val: GpuTextureAspect); + + #[doc = "Get the `mipLevel` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "mipLevel")] + pub fn get_mip_level(this: &GpuTexelCopyTextureInfo) -> Option; + + #[doc = "Change the `mipLevel` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "mipLevel")] + pub fn set_mip_level(this: &GpuTexelCopyTextureInfo, val: u32); + + #[doc = "Get the `origin` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "origin")] + pub fn get_origin(this: &GpuTexelCopyTextureInfo) -> ::wasm_bindgen::JsValue; + + #[doc = "Change the `origin` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "origin")] + pub fn set_origin(this: &GpuTexelCopyTextureInfo, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `texture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`, `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "texture")] + pub fn get_texture(this: &GpuTexelCopyTextureInfo) -> GpuTexture; + + #[doc = "Change the `texture` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`, `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "texture")] + pub fn set_texture(this: &GpuTexelCopyTextureInfo, val: &GpuTexture); +} + +impl GpuTexelCopyTextureInfo { + #[doc = "Construct a new `GpuTexelCopyTextureInfo`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexelCopyTextureInfo`, `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(texture: &GpuTexture) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_texture(texture); + ret + } + + #[deprecated = "Use `set_aspect()` instead."] + pub fn aspect(&mut self, val: GpuTextureAspect) -> &mut Self { + self.set_aspect(val); + self + } + + #[deprecated = "Use `set_mip_level()` instead."] + pub fn mip_level(&mut self, val: u32) -> &mut Self { + self.set_mip_level(val); + self + } + + #[deprecated = "Use `set_origin()` instead."] + pub fn origin(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_origin(val); + self + } + + #[deprecated = "Use `set_texture()` instead."] + pub fn texture(&mut self, val: &GpuTexture) -> &mut Self { + self.set_texture(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexture.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexture.rs new file mode 100644 index 00000000..20c3eb8c --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTexture.rs @@ -0,0 +1,186 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUTexture , typescript_type = "GPUTexture")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuTexture` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuTexture; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTexture" , js_name = width)] + #[doc = "Getter for the `width` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/width)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn width(this: &GpuTexture) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTexture" , js_name = height)] + #[doc = "Getter for the `height` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/height)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn height(this: &GpuTexture) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTexture" , js_name = depthOrArrayLayers)] + #[doc = "Getter for the `depthOrArrayLayers` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/depthOrArrayLayers)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn depth_or_array_layers(this: &GpuTexture) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTexture" , js_name = mipLevelCount)] + #[doc = "Getter for the `mipLevelCount` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/mipLevelCount)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn mip_level_count(this: &GpuTexture) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTexture" , js_name = sampleCount)] + #[doc = "Getter for the `sampleCount` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/sampleCount)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn sample_count(this: &GpuTexture) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTexture" , js_name = dimension)] + #[doc = "Getter for the `dimension` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/dimension)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`, `GpuTextureDimension`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn dimension(this: &GpuTexture) -> GpuTextureDimension; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTexture" , js_name = format)] + #[doc = "Getter for the `format` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/format)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn format(this: &GpuTexture) -> GpuTextureFormat; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTexture" , js_name = usage)] + #[doc = "Getter for the `usage` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/usage)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn usage(this: &GpuTexture) -> u32; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTexture" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuTexture) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUTexture" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuTexture, value: &str); + + # [wasm_bindgen (catch , method , structural , js_class = "GPUTexture" , js_name = createView)] + #[doc = "The `createView()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/createView)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`, `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_view(this: &GpuTexture) -> Result; + + # [wasm_bindgen (catch , method , structural , js_class = "GPUTexture" , js_name = createView)] + #[doc = "The `createView()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/createView)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`, `GpuTextureView`, `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn create_view_with_descriptor( + this: &GpuTexture, + descriptor: &GpuTextureViewDescriptor, + ) -> Result; + + # [wasm_bindgen (method , structural , js_class = "GPUTexture" , js_name = destroy)] + #[doc = "The `destroy()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTexture/destroy)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTexture`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn destroy(this: &GpuTexture); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureAspect.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureAspect.rs new file mode 100644 index 00000000..c8d7739b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureAspect.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuTextureAspect` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuTextureAspect`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuTextureAspect { + All = "all", + StencilOnly = "stencil-only", + DepthOnly = "depth-only", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureBindingLayout.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureBindingLayout.rs new file mode 100644 index 00000000..2a7bd017 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureBindingLayout.rs @@ -0,0 +1,130 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUTextureBindingLayout)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuTextureBindingLayout` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuTextureBindingLayout; + + #[doc = "Get the `multisampled` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "multisampled")] + pub fn get_multisampled(this: &GpuTextureBindingLayout) -> Option; + + #[doc = "Change the `multisampled` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "multisampled")] + pub fn set_multisampled(this: &GpuTextureBindingLayout, val: bool); + + #[doc = "Get the `sampleType` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureBindingLayout`, `GpuTextureSampleType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "sampleType")] + pub fn get_sample_type(this: &GpuTextureBindingLayout) -> Option; + + #[doc = "Change the `sampleType` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureBindingLayout`, `GpuTextureSampleType`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "sampleType")] + pub fn set_sample_type(this: &GpuTextureBindingLayout, val: GpuTextureSampleType); + + #[doc = "Get the `viewDimension` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureBindingLayout`, `GpuTextureViewDimension`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "viewDimension")] + pub fn get_view_dimension(this: &GpuTextureBindingLayout) -> Option; + + #[doc = "Change the `viewDimension` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureBindingLayout`, `GpuTextureViewDimension`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "viewDimension")] + pub fn set_view_dimension(this: &GpuTextureBindingLayout, val: GpuTextureViewDimension); +} + +impl GpuTextureBindingLayout { + #[doc = "Construct a new `GpuTextureBindingLayout`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureBindingLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_multisampled()` instead."] + pub fn multisampled(&mut self, val: bool) -> &mut Self { + self.set_multisampled(val); + self + } + + #[deprecated = "Use `set_sample_type()` instead."] + pub fn sample_type(&mut self, val: GpuTextureSampleType) -> &mut Self { + self.set_sample_type(val); + self + } + + #[deprecated = "Use `set_view_dimension()` instead."] + pub fn view_dimension(&mut self, val: GpuTextureViewDimension) -> &mut Self { + self.set_view_dimension(val); + self + } +} + +impl Default for GpuTextureBindingLayout { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureDescriptor.rs new file mode 100644 index 00000000..dc6be017 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureDescriptor.rs @@ -0,0 +1,247 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUTextureDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuTextureDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuTextureDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuTextureDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuTextureDescriptor, val: &str); + + #[doc = "Get the `dimension` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`, `GpuTextureDimension`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "dimension")] + pub fn get_dimension(this: &GpuTextureDescriptor) -> Option; + + #[doc = "Change the `dimension` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`, `GpuTextureDimension`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "dimension")] + pub fn set_dimension(this: &GpuTextureDescriptor, val: GpuTextureDimension); + + #[doc = "Get the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "format")] + pub fn get_format(this: &GpuTextureDescriptor) -> GpuTextureFormat; + + #[doc = "Change the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "format")] + pub fn set_format(this: &GpuTextureDescriptor, val: GpuTextureFormat); + + #[doc = "Get the `mipLevelCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "mipLevelCount")] + pub fn get_mip_level_count(this: &GpuTextureDescriptor) -> Option; + + #[doc = "Change the `mipLevelCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "mipLevelCount")] + pub fn set_mip_level_count(this: &GpuTextureDescriptor, val: u32); + + #[doc = "Get the `sampleCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "sampleCount")] + pub fn get_sample_count(this: &GpuTextureDescriptor) -> Option; + + #[doc = "Change the `sampleCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "sampleCount")] + pub fn set_sample_count(this: &GpuTextureDescriptor, val: u32); + + #[doc = "Get the `size` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "size")] + pub fn get_size(this: &GpuTextureDescriptor) -> ::wasm_bindgen::JsValue; + + #[doc = "Change the `size` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "size")] + pub fn set_size(this: &GpuTextureDescriptor, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `usage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "usage")] + pub fn get_usage(this: &GpuTextureDescriptor) -> u32; + + #[doc = "Change the `usage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "usage")] + pub fn set_usage(this: &GpuTextureDescriptor, val: u32); + + #[doc = "Get the `viewFormats` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "viewFormats")] + pub fn get_view_formats(this: &GpuTextureDescriptor) -> Option<::js_sys::Array>; + + #[doc = "Change the `viewFormats` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "viewFormats")] + pub fn set_view_formats(this: &GpuTextureDescriptor, val: &::wasm_bindgen::JsValue); +} + +impl GpuTextureDescriptor { + #[doc = "Construct a new `GpuTextureDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureDescriptor`, `GpuTextureFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(format: GpuTextureFormat, size: &::wasm_bindgen::JsValue, usage: u32) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_format(format); + ret.set_size(size); + ret.set_usage(usage); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_dimension()` instead."] + pub fn dimension(&mut self, val: GpuTextureDimension) -> &mut Self { + self.set_dimension(val); + self + } + + #[deprecated = "Use `set_format()` instead."] + pub fn format(&mut self, val: GpuTextureFormat) -> &mut Self { + self.set_format(val); + self + } + + #[deprecated = "Use `set_mip_level_count()` instead."] + pub fn mip_level_count(&mut self, val: u32) -> &mut Self { + self.set_mip_level_count(val); + self + } + + #[deprecated = "Use `set_sample_count()` instead."] + pub fn sample_count(&mut self, val: u32) -> &mut Self { + self.set_sample_count(val); + self + } + + #[deprecated = "Use `set_size()` instead."] + pub fn size(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_size(val); + self + } + + #[deprecated = "Use `set_usage()` instead."] + pub fn usage(&mut self, val: u32) -> &mut Self { + self.set_usage(val); + self + } + + #[deprecated = "Use `set_view_formats()` instead."] + pub fn view_formats(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_view_formats(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureDimension.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureDimension.rs new file mode 100644 index 00000000..d860c9d1 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureDimension.rs @@ -0,0 +1,38 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuTextureDimension` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuTextureDimension`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuTextureDimension { + N1d = "1d", + N2d = "2d", + N3d = "3d", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureFormat.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureFormat.rs new file mode 100644 index 00000000..c861a82f --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureFormat.rs @@ -0,0 +1,130 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuTextureFormat` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuTextureFormat`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuTextureFormat { + R8unorm = "r8unorm", + R8snorm = "r8snorm", + R8uint = "r8uint", + R8sint = "r8sint", + R16uint = "r16uint", + R16sint = "r16sint", + R16float = "r16float", + Rg8unorm = "rg8unorm", + Rg8snorm = "rg8snorm", + Rg8uint = "rg8uint", + Rg8sint = "rg8sint", + R32uint = "r32uint", + R32sint = "r32sint", + R32float = "r32float", + Rg16uint = "rg16uint", + Rg16sint = "rg16sint", + Rg16float = "rg16float", + Rgba8unorm = "rgba8unorm", + Rgba8unormSrgb = "rgba8unorm-srgb", + Rgba8snorm = "rgba8snorm", + Rgba8uint = "rgba8uint", + Rgba8sint = "rgba8sint", + Bgra8unorm = "bgra8unorm", + Bgra8unormSrgb = "bgra8unorm-srgb", + Rgb9e5ufloat = "rgb9e5ufloat", + Rgb10a2uint = "rgb10a2uint", + Rgb10a2unorm = "rgb10a2unorm", + Rg11b10ufloat = "rg11b10ufloat", + Rg32uint = "rg32uint", + Rg32sint = "rg32sint", + Rg32float = "rg32float", + Rgba16uint = "rgba16uint", + Rgba16sint = "rgba16sint", + Rgba16float = "rgba16float", + Rgba32uint = "rgba32uint", + Rgba32sint = "rgba32sint", + Rgba32float = "rgba32float", + Stencil8 = "stencil8", + Depth16unorm = "depth16unorm", + Depth24plus = "depth24plus", + Depth24plusStencil8 = "depth24plus-stencil8", + Depth32float = "depth32float", + Depth32floatStencil8 = "depth32float-stencil8", + Bc1RgbaUnorm = "bc1-rgba-unorm", + Bc1RgbaUnormSrgb = "bc1-rgba-unorm-srgb", + Bc2RgbaUnorm = "bc2-rgba-unorm", + Bc2RgbaUnormSrgb = "bc2-rgba-unorm-srgb", + Bc3RgbaUnorm = "bc3-rgba-unorm", + Bc3RgbaUnormSrgb = "bc3-rgba-unorm-srgb", + Bc4RUnorm = "bc4-r-unorm", + Bc4RSnorm = "bc4-r-snorm", + Bc5RgUnorm = "bc5-rg-unorm", + Bc5RgSnorm = "bc5-rg-snorm", + Bc6hRgbUfloat = "bc6h-rgb-ufloat", + Bc6hRgbFloat = "bc6h-rgb-float", + Bc7RgbaUnorm = "bc7-rgba-unorm", + Bc7RgbaUnormSrgb = "bc7-rgba-unorm-srgb", + Etc2Rgb8unorm = "etc2-rgb8unorm", + Etc2Rgb8unormSrgb = "etc2-rgb8unorm-srgb", + Etc2Rgb8a1unorm = "etc2-rgb8a1unorm", + Etc2Rgb8a1unormSrgb = "etc2-rgb8a1unorm-srgb", + Etc2Rgba8unorm = "etc2-rgba8unorm", + Etc2Rgba8unormSrgb = "etc2-rgba8unorm-srgb", + EacR11unorm = "eac-r11unorm", + EacR11snorm = "eac-r11snorm", + EacRg11unorm = "eac-rg11unorm", + EacRg11snorm = "eac-rg11snorm", + Astc4x4Unorm = "astc-4x4-unorm", + Astc4x4UnormSrgb = "astc-4x4-unorm-srgb", + Astc5x4Unorm = "astc-5x4-unorm", + Astc5x4UnormSrgb = "astc-5x4-unorm-srgb", + Astc5x5Unorm = "astc-5x5-unorm", + Astc5x5UnormSrgb = "astc-5x5-unorm-srgb", + Astc6x5Unorm = "astc-6x5-unorm", + Astc6x5UnormSrgb = "astc-6x5-unorm-srgb", + Astc6x6Unorm = "astc-6x6-unorm", + Astc6x6UnormSrgb = "astc-6x6-unorm-srgb", + Astc8x5Unorm = "astc-8x5-unorm", + Astc8x5UnormSrgb = "astc-8x5-unorm-srgb", + Astc8x6Unorm = "astc-8x6-unorm", + Astc8x6UnormSrgb = "astc-8x6-unorm-srgb", + Astc8x8Unorm = "astc-8x8-unorm", + Astc8x8UnormSrgb = "astc-8x8-unorm-srgb", + Astc10x5Unorm = "astc-10x5-unorm", + Astc10x5UnormSrgb = "astc-10x5-unorm-srgb", + Astc10x6Unorm = "astc-10x6-unorm", + Astc10x6UnormSrgb = "astc-10x6-unorm-srgb", + Astc10x8Unorm = "astc-10x8-unorm", + Astc10x8UnormSrgb = "astc-10x8-unorm-srgb", + Astc10x10Unorm = "astc-10x10-unorm", + Astc10x10UnormSrgb = "astc-10x10-unorm-srgb", + Astc12x10Unorm = "astc-12x10-unorm", + Astc12x10UnormSrgb = "astc-12x10-unorm-srgb", + Astc12x12Unorm = "astc-12x12-unorm", + Astc12x12UnormSrgb = "astc-12x12-unorm-srgb", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureSampleType.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureSampleType.rs new file mode 100644 index 00000000..04fdb2e3 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureSampleType.rs @@ -0,0 +1,40 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuTextureSampleType` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuTextureSampleType`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuTextureSampleType { + Float = "float", + UnfilterableFloat = "unfilterable-float", + Depth = "depth", + Sint = "sint", + Uint = "uint", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureView.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureView.rs new file mode 100644 index 00000000..6ece2949 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureView.rs @@ -0,0 +1,62 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUTextureView , typescript_type = "GPUTextureView")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuTextureView` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTextureView)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuTextureView; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUTextureView" , js_name = label)] + #[doc = "Getter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTextureView/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn label(this: &GpuTextureView) -> ::alloc::string::String; + + # [wasm_bindgen (structural , method , setter , js_class = "GPUTextureView" , js_name = label)] + #[doc = "Setter for the `label` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUTextureView/label)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureView`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn set_label(this: &GpuTextureView, value: &str); +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureViewDescriptor.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureViewDescriptor.rs new file mode 100644 index 00000000..49c642d3 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureViewDescriptor.rs @@ -0,0 +1,274 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUTextureViewDescriptor)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuTextureViewDescriptor` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuTextureViewDescriptor; + + #[doc = "Get the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "label")] + pub fn get_label(this: &GpuTextureViewDescriptor) -> Option<::alloc::string::String>; + + #[doc = "Change the `label` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "label")] + pub fn set_label(this: &GpuTextureViewDescriptor, val: &str); + + #[doc = "Get the `arrayLayerCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "arrayLayerCount")] + pub fn get_array_layer_count(this: &GpuTextureViewDescriptor) -> Option; + + #[doc = "Change the `arrayLayerCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "arrayLayerCount")] + pub fn set_array_layer_count(this: &GpuTextureViewDescriptor, val: u32); + + #[doc = "Get the `aspect` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureAspect`, `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "aspect")] + pub fn get_aspect(this: &GpuTextureViewDescriptor) -> Option; + + #[doc = "Change the `aspect` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureAspect`, `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "aspect")] + pub fn set_aspect(this: &GpuTextureViewDescriptor, val: GpuTextureAspect); + + #[doc = "Get the `baseArrayLayer` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "baseArrayLayer")] + pub fn get_base_array_layer(this: &GpuTextureViewDescriptor) -> Option; + + #[doc = "Change the `baseArrayLayer` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "baseArrayLayer")] + pub fn set_base_array_layer(this: &GpuTextureViewDescriptor, val: u32); + + #[doc = "Get the `baseMipLevel` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "baseMipLevel")] + pub fn get_base_mip_level(this: &GpuTextureViewDescriptor) -> Option; + + #[doc = "Change the `baseMipLevel` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "baseMipLevel")] + pub fn set_base_mip_level(this: &GpuTextureViewDescriptor, val: u32); + + #[doc = "Get the `dimension` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`, `GpuTextureViewDimension`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "dimension")] + pub fn get_dimension(this: &GpuTextureViewDescriptor) -> Option; + + #[doc = "Change the `dimension` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`, `GpuTextureViewDimension`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "dimension")] + pub fn set_dimension(this: &GpuTextureViewDescriptor, val: GpuTextureViewDimension); + + #[doc = "Get the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureFormat`, `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "format")] + pub fn get_format(this: &GpuTextureViewDescriptor) -> Option; + + #[doc = "Change the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureFormat`, `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "format")] + pub fn set_format(this: &GpuTextureViewDescriptor, val: GpuTextureFormat); + + #[doc = "Get the `mipLevelCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "mipLevelCount")] + pub fn get_mip_level_count(this: &GpuTextureViewDescriptor) -> Option; + + #[doc = "Change the `mipLevelCount` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "mipLevelCount")] + pub fn set_mip_level_count(this: &GpuTextureViewDescriptor, val: u32); + + #[doc = "Get the `usage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "usage")] + pub fn get_usage(this: &GpuTextureViewDescriptor) -> Option; + + #[doc = "Change the `usage` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "usage")] + pub fn set_usage(this: &GpuTextureViewDescriptor, val: u32); +} + +impl GpuTextureViewDescriptor { + #[doc = "Construct a new `GpuTextureViewDescriptor`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDescriptor`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new() -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret + } + + #[deprecated = "Use `set_label()` instead."] + pub fn label(&mut self, val: &str) -> &mut Self { + self.set_label(val); + self + } + + #[deprecated = "Use `set_array_layer_count()` instead."] + pub fn array_layer_count(&mut self, val: u32) -> &mut Self { + self.set_array_layer_count(val); + self + } + + #[deprecated = "Use `set_aspect()` instead."] + pub fn aspect(&mut self, val: GpuTextureAspect) -> &mut Self { + self.set_aspect(val); + self + } + + #[deprecated = "Use `set_base_array_layer()` instead."] + pub fn base_array_layer(&mut self, val: u32) -> &mut Self { + self.set_base_array_layer(val); + self + } + + #[deprecated = "Use `set_base_mip_level()` instead."] + pub fn base_mip_level(&mut self, val: u32) -> &mut Self { + self.set_base_mip_level(val); + self + } + + #[deprecated = "Use `set_dimension()` instead."] + pub fn dimension(&mut self, val: GpuTextureViewDimension) -> &mut Self { + self.set_dimension(val); + self + } + + #[deprecated = "Use `set_format()` instead."] + pub fn format(&mut self, val: GpuTextureFormat) -> &mut Self { + self.set_format(val); + self + } + + #[deprecated = "Use `set_mip_level_count()` instead."] + pub fn mip_level_count(&mut self, val: u32) -> &mut Self { + self.set_mip_level_count(val); + self + } + + #[deprecated = "Use `set_usage()` instead."] + pub fn usage(&mut self, val: u32) -> &mut Self { + self.set_usage(val); + self + } +} + +impl Default for GpuTextureViewDescriptor { + fn default() -> Self { + Self::new() + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureViewDimension.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureViewDimension.rs new file mode 100644 index 00000000..57ccb036 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuTextureViewDimension.rs @@ -0,0 +1,41 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuTextureViewDimension` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuTextureViewDimension`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuTextureViewDimension { + N1d = "1d", + N2d = "2d", + N2dArray = "2d-array", + Cube = "cube", + CubeArray = "cube-array", + N3d = "3d", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuUncapturedErrorEvent.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuUncapturedErrorEvent.rs new file mode 100644 index 00000000..a59f1dbe --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuUncapturedErrorEvent.rs @@ -0,0 +1,65 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = Event , extends = :: js_sys :: Object , js_name = GPUUncapturedErrorEvent , typescript_type = "GPUUncapturedErrorEvent")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuUncapturedErrorEvent` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUUncapturedErrorEvent)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuUncapturedErrorEvent`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuUncapturedErrorEvent; + + # [wasm_bindgen (structural , method , getter , js_class = "GPUUncapturedErrorEvent" , js_name = error)] + #[doc = "Getter for the `error` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUUncapturedErrorEvent/error)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuError`, `GpuUncapturedErrorEvent`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn error(this: &GpuUncapturedErrorEvent) -> GpuError; + + #[wasm_bindgen(catch, constructor, js_class = "GPUUncapturedErrorEvent")] + #[doc = "The `new GpuUncapturedErrorEvent(..)` constructor, creating a new instance of `GpuUncapturedErrorEvent`."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUUncapturedErrorEvent/GPUUncapturedErrorEvent)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuUncapturedErrorEvent`, `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new( + type_: &str, + gpu_uncaptured_error_event_init_dict: &GpuUncapturedErrorEventInit, + ) -> Result; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuUncapturedErrorEventInit.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuUncapturedErrorEventInit.rs new file mode 100644 index 00000000..32597521 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuUncapturedErrorEventInit.rs @@ -0,0 +1,149 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUUncapturedErrorEventInit)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuUncapturedErrorEventInit` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuUncapturedErrorEventInit; + + #[doc = "Get the `bubbles` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "bubbles")] + pub fn get_bubbles(this: &GpuUncapturedErrorEventInit) -> Option; + + #[doc = "Change the `bubbles` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "bubbles")] + pub fn set_bubbles(this: &GpuUncapturedErrorEventInit, val: bool); + + #[doc = "Get the `cancelable` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "cancelable")] + pub fn get_cancelable(this: &GpuUncapturedErrorEventInit) -> Option; + + #[doc = "Change the `cancelable` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "cancelable")] + pub fn set_cancelable(this: &GpuUncapturedErrorEventInit, val: bool); + + #[doc = "Get the `composed` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "composed")] + pub fn get_composed(this: &GpuUncapturedErrorEventInit) -> Option; + + #[doc = "Change the `composed` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "composed")] + pub fn set_composed(this: &GpuUncapturedErrorEventInit, val: bool); + + #[doc = "Get the `error` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuError`, `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "error")] + pub fn get_error(this: &GpuUncapturedErrorEventInit) -> GpuError; + + #[doc = "Change the `error` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuError`, `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "error")] + pub fn set_error(this: &GpuUncapturedErrorEventInit, val: &GpuError); +} + +impl GpuUncapturedErrorEventInit { + #[doc = "Construct a new `GpuUncapturedErrorEventInit`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuError`, `GpuUncapturedErrorEventInit`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(error: &GpuError) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_error(error); + ret + } + + #[deprecated = "Use `set_bubbles()` instead."] + pub fn bubbles(&mut self, val: bool) -> &mut Self { + self.set_bubbles(val); + self + } + + #[deprecated = "Use `set_cancelable()` instead."] + pub fn cancelable(&mut self, val: bool) -> &mut Self { + self.set_cancelable(val); + self + } + + #[deprecated = "Use `set_composed()` instead."] + pub fn composed(&mut self, val: bool) -> &mut Self { + self.set_composed(val); + self + } + + #[deprecated = "Use `set_error()` instead."] + pub fn error(&mut self, val: &GpuError) -> &mut Self { + self.set_error(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuValidationError.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuValidationError.rs new file mode 100644 index 00000000..f29b3b86 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuValidationError.rs @@ -0,0 +1,51 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = GpuError , extends = :: js_sys :: Object , js_name = GPUValidationError , typescript_type = "GPUValidationError")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuValidationError` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUValidationError)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuValidationError`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuValidationError; + + #[wasm_bindgen(catch, constructor, js_class = "GPUValidationError")] + #[doc = "The `new GpuValidationError(..)` constructor, creating a new instance of `GpuValidationError`."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/GPUValidationError/GPUValidationError)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuValidationError`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(message: &str) -> Result; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexAttribute.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexAttribute.rs new file mode 100644 index 00000000..298b2134 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexAttribute.rs @@ -0,0 +1,127 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUVertexAttribute)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuVertexAttribute` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexAttribute`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuVertexAttribute; + + #[doc = "Get the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexAttribute`, `GpuVertexFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "format")] + pub fn get_format(this: &GpuVertexAttribute) -> GpuVertexFormat; + + #[doc = "Change the `format` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexAttribute`, `GpuVertexFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "format")] + pub fn set_format(this: &GpuVertexAttribute, val: GpuVertexFormat); + + #[doc = "Get the `offset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexAttribute`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "offset")] + pub fn get_offset(this: &GpuVertexAttribute) -> f64; + + #[doc = "Change the `offset` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexAttribute`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "offset")] + pub fn set_offset(this: &GpuVertexAttribute, val: f64); + + #[doc = "Get the `shaderLocation` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexAttribute`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "shaderLocation")] + pub fn get_shader_location(this: &GpuVertexAttribute) -> u32; + + #[doc = "Change the `shaderLocation` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexAttribute`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "shaderLocation")] + pub fn set_shader_location(this: &GpuVertexAttribute, val: u32); +} + +impl GpuVertexAttribute { + #[doc = "Construct a new `GpuVertexAttribute`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexAttribute`, `GpuVertexFormat`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(format: GpuVertexFormat, offset: f64, shader_location: u32) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_format(format); + ret.set_offset(offset); + ret.set_shader_location(shader_location); + ret + } + + #[deprecated = "Use `set_format()` instead."] + pub fn format(&mut self, val: GpuVertexFormat) -> &mut Self { + self.set_format(val); + self + } + + #[deprecated = "Use `set_offset()` instead."] + pub fn offset(&mut self, val: f64) -> &mut Self { + self.set_offset(val); + self + } + + #[deprecated = "Use `set_shader_location()` instead."] + pub fn shader_location(&mut self, val: u32) -> &mut Self { + self.set_shader_location(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexBufferLayout.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexBufferLayout.rs new file mode 100644 index 00000000..9ebf3e6e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexBufferLayout.rs @@ -0,0 +1,126 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUVertexBufferLayout)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuVertexBufferLayout` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuVertexBufferLayout; + + #[doc = "Get the `arrayStride` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "arrayStride")] + pub fn get_array_stride(this: &GpuVertexBufferLayout) -> f64; + + #[doc = "Change the `arrayStride` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "arrayStride")] + pub fn set_array_stride(this: &GpuVertexBufferLayout, val: f64); + + #[doc = "Get the `attributes` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "attributes")] + pub fn get_attributes(this: &GpuVertexBufferLayout) -> ::js_sys::Array; + + #[doc = "Change the `attributes` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "attributes")] + pub fn set_attributes(this: &GpuVertexBufferLayout, val: &::wasm_bindgen::JsValue); + + #[doc = "Get the `stepMode` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexBufferLayout`, `GpuVertexStepMode`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "stepMode")] + pub fn get_step_mode(this: &GpuVertexBufferLayout) -> Option; + + #[doc = "Change the `stepMode` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexBufferLayout`, `GpuVertexStepMode`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "stepMode")] + pub fn set_step_mode(this: &GpuVertexBufferLayout, val: GpuVertexStepMode); +} + +impl GpuVertexBufferLayout { + #[doc = "Construct a new `GpuVertexBufferLayout`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexBufferLayout`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(array_stride: f64, attributes: &::wasm_bindgen::JsValue) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_array_stride(array_stride); + ret.set_attributes(attributes); + ret + } + + #[deprecated = "Use `set_array_stride()` instead."] + pub fn array_stride(&mut self, val: f64) -> &mut Self { + self.set_array_stride(val); + self + } + + #[deprecated = "Use `set_attributes()` instead."] + pub fn attributes(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_attributes(val); + self + } + + #[deprecated = "Use `set_step_mode()` instead."] + pub fn step_mode(&mut self, val: GpuVertexStepMode) -> &mut Self { + self.set_step_mode(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexFormat.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexFormat.rs new file mode 100644 index 00000000..167c67a6 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexFormat.rs @@ -0,0 +1,76 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuVertexFormat` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuVertexFormat`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuVertexFormat { + Uint8 = "uint8", + Uint8x2 = "uint8x2", + Uint8x4 = "uint8x4", + Sint8 = "sint8", + Sint8x2 = "sint8x2", + Sint8x4 = "sint8x4", + Unorm8 = "unorm8", + Unorm8x2 = "unorm8x2", + Unorm8x4 = "unorm8x4", + Snorm8 = "snorm8", + Snorm8x2 = "snorm8x2", + Snorm8x4 = "snorm8x4", + Uint16 = "uint16", + Uint16x2 = "uint16x2", + Uint16x4 = "uint16x4", + Sint16 = "sint16", + Sint16x2 = "sint16x2", + Sint16x4 = "sint16x4", + Unorm16 = "unorm16", + Unorm16x2 = "unorm16x2", + Unorm16x4 = "unorm16x4", + Snorm16 = "snorm16", + Snorm16x2 = "snorm16x2", + Snorm16x4 = "snorm16x4", + Float16 = "float16", + Float16x2 = "float16x2", + Float16x4 = "float16x4", + Float32 = "float32", + Float32x2 = "float32x2", + Float32x3 = "float32x3", + Float32x4 = "float32x4", + Uint32 = "uint32", + Uint32x2 = "uint32x2", + Uint32x3 = "uint32x3", + Uint32x4 = "uint32x4", + Sint32 = "sint32", + Sint32x2 = "sint32x2", + Sint32x3 = "sint32x3", + Sint32x4 = "sint32x4", + Unorm1010102 = "unorm10-10-10-2", + Unorm8x4Bgra = "unorm8x4-bgra", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexState.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexState.rs new file mode 100644 index 00000000..a60ef9e6 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexState.rs @@ -0,0 +1,149 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = GPUVertexState)] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `GpuVertexState` dictionary."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type GpuVertexState; + + #[doc = "Get the `constants` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "constants")] + pub fn get_constants(this: &GpuVertexState) -> Option<::js_sys::Object>; + + #[doc = "Change the `constants` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "constants")] + pub fn set_constants(this: &GpuVertexState, val: &::js_sys::Object); + + #[doc = "Get the `entryPoint` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "entryPoint")] + pub fn get_entry_point(this: &GpuVertexState) -> Option<::alloc::string::String>; + + #[doc = "Change the `entryPoint` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "entryPoint")] + pub fn set_entry_point(this: &GpuVertexState, val: &str); + + #[doc = "Get the `module` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModule`, `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "module")] + pub fn get_module(this: &GpuVertexState) -> GpuShaderModule; + + #[doc = "Change the `module` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModule`, `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "module")] + pub fn set_module(this: &GpuVertexState, val: &GpuShaderModule); + + #[doc = "Get the `buffers` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, getter = "buffers")] + pub fn get_buffers(this: &GpuVertexState) -> Option<::js_sys::Array>; + + #[doc = "Change the `buffers` field of this object."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + #[wasm_bindgen(method, setter = "buffers")] + pub fn set_buffers(this: &GpuVertexState, val: &::wasm_bindgen::JsValue); +} + +impl GpuVertexState { + #[doc = "Construct a new `GpuVertexState`."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `GpuShaderModule`, `GpuVertexState`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn new(module: &GpuShaderModule) -> Self { + #[allow(unused_mut)] + let mut ret: Self = ::wasm_bindgen::JsCast::unchecked_into(::js_sys::Object::new()); + ret.set_module(module); + ret + } + + #[deprecated = "Use `set_constants()` instead."] + pub fn constants(&mut self, val: &::js_sys::Object) -> &mut Self { + self.set_constants(val); + self + } + + #[deprecated = "Use `set_entry_point()` instead."] + pub fn entry_point(&mut self, val: &str) -> &mut Self { + self.set_entry_point(val); + self + } + + #[deprecated = "Use `set_module()` instead."] + pub fn module(&mut self, val: &GpuShaderModule) -> &mut Self { + self.set_module(val); + self + } + + #[deprecated = "Use `set_buffers()` instead."] + pub fn buffers(&mut self, val: &::wasm_bindgen::JsValue) -> &mut Self { + self.set_buffers(val); + self + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexStepMode.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexStepMode.rs new file mode 100644 index 00000000..29a0b857 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_GpuVertexStepMode.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +#[doc = "The `GpuVertexStepMode` enum."] +#[doc = ""] +#[doc = "*This API requires the following crate features to be activated: `GpuVertexStepMode`*"] +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuVertexStepMode { + Vertex = "vertex", + Instance = "instance", +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_WgslLanguageFeatures.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_WgslLanguageFeatures.rs new file mode 100644 index 00000000..d6fe55d0 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_WgslLanguageFeatures.rs @@ -0,0 +1,109 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. +#![allow(unused_imports)] +#![allow(clippy::all)] +use super::*; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +extern "C" { + # [wasm_bindgen (extends = :: js_sys :: Object , js_name = WGSLLanguageFeatures , typescript_type = "WGSLLanguageFeatures")] + #[derive(Debug, Clone, PartialEq, Eq)] + #[doc = "The `WgslLanguageFeatures` class."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/WGSLLanguageFeatures)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `WgslLanguageFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub type WgslLanguageFeatures; + + # [wasm_bindgen (structural , method , getter , js_class = "WGSLLanguageFeatures" , js_name = size)] + #[doc = "Getter for the `size` field of this object."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/WGSLLanguageFeatures/size)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `WgslLanguageFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn size(this: &WgslLanguageFeatures) -> u32; + + # [wasm_bindgen (method , structural , js_class = "WGSLLanguageFeatures" , js_name = entries)] + #[doc = "The `entries()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/WGSLLanguageFeatures/entries)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `WgslLanguageFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn entries(this: &WgslLanguageFeatures) -> ::js_sys::Iterator; + + # [wasm_bindgen (catch , method , structural , js_class = "WGSLLanguageFeatures" , js_name = forEach)] + #[doc = "The `forEach()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/WGSLLanguageFeatures/forEach)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `WgslLanguageFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn for_each( + this: &WgslLanguageFeatures, + callback: &::js_sys::Function, + ) -> Result<(), JsValue>; + + # [wasm_bindgen (method , structural , js_class = "WGSLLanguageFeatures" , js_name = has)] + #[doc = "The `has()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/WGSLLanguageFeatures/has)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `WgslLanguageFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn has(this: &WgslLanguageFeatures, value: &str) -> bool; + + # [wasm_bindgen (method , structural , js_class = "WGSLLanguageFeatures" , js_name = keys)] + #[doc = "The `keys()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/WGSLLanguageFeatures/keys)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `WgslLanguageFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn keys(this: &WgslLanguageFeatures) -> ::js_sys::Iterator; + + # [wasm_bindgen (method , structural , js_class = "WGSLLanguageFeatures" , js_name = values)] + #[doc = "The `values()` method."] + #[doc = ""] + #[doc = "[MDN Documentation](https://developer.mozilla.org/en-US/docs/Web/API/WGSLLanguageFeatures/values)"] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `WgslLanguageFeatures`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub fn values(this: &WgslLanguageFeatures) -> ::js_sys::Iterator; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_gpu_map_mode.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_gpu_map_mode.rs new file mode 100644 index 00000000..4b620cb0 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/gen_gpu_map_mode.rs @@ -0,0 +1,47 @@ +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. + +#[doc = ""] +#[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] +#[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] +pub mod gpu_map_mode { + #![allow(unused_imports)] + #![allow(clippy::all)] + use super::super::*; + use wasm_bindgen::prelude::*; + + #[doc = "The `GPUMapMode.READ` const."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `gpu_map_mode`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub const READ: u32 = 1u64 as u32; + + #[doc = "The `GPUMapMode.WRITE` const."] + #[doc = ""] + #[doc = "*This API requires the following crate features to be activated: `gpu_map_mode`*"] + #[doc = ""] + #[doc = "*This API is unstable and requires `--cfg=web_sys_unstable_apis` to be activated, as"] + #[doc = "[described in the `wasm-bindgen` guide](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/unstable-apis.html)*"] + pub const WRITE: u32 = 2u64 as u32; +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/mod.rs b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/mod.rs new file mode 100644 index 00000000..49bcd32f --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/webgpu/webgpu_sys/mod.rs @@ -0,0 +1,281 @@ +//! Bindings to the WebGPU API. +//! +//! Internally vendored from the `web-sys` crate until the WebGPU binding are stabilized. +// DO NOT EDIT THIS FILE! +// +// This module part of a subset of web-sys that is used by wgpu's webgpu backend. +// +// These bindings are vendored into wgpu for the sole purpose of letting +// us pin the WebGPU backend to a specific version of the bindings, not +// to enable local changes. There are no provisions to preserve changes +// you make here the next time we re-vendor the bindings. +// +// The `web-sys` crate does not treat breaking changes to the WebGPU API +// as semver breaking changes, as WebGPU is "unstable". This means Cargo +// will not let us mix versions of `web-sys`, pinning WebGPU bindings to +// a specific version, while letting other bindings like WebGL get +// updated. Vendoring WebGPU was the workaround we chose. +// +// Vendoring also allows us to avoid building `web-sys` with +// `--cfg=web_sys_unstable_apis`, needed to get the WebGPU bindings. +// +// If you want to improve the generated code, please submit a PR to the https://github.com/wasm-bindgen/wasm-bindgen repository. +// +// This file was generated by the `cargo xtask vendor-web-sys --version f7e0467c4b4d88303eb79fba485320cfb4dfdd58` command. + +#![allow(unused_imports, non_snake_case)] +use web_sys::{Event, EventTarget}; +mod gen_Gpu; +pub use gen_Gpu::*; +mod gen_GpuAdapter; +pub use gen_GpuAdapter::*; +mod gen_GpuAdapterInfo; +pub use gen_GpuAdapterInfo::*; +mod gen_GpuAddressMode; +pub use gen_GpuAddressMode::*; +mod gen_GpuAutoLayoutMode; +pub use gen_GpuAutoLayoutMode::*; +mod gen_GpuBindGroup; +pub use gen_GpuBindGroup::*; +mod gen_GpuBindGroupDescriptor; +pub use gen_GpuBindGroupDescriptor::*; +mod gen_GpuBindGroupEntry; +pub use gen_GpuBindGroupEntry::*; +mod gen_GpuBindGroupLayout; +pub use gen_GpuBindGroupLayout::*; +mod gen_GpuBindGroupLayoutDescriptor; +pub use gen_GpuBindGroupLayoutDescriptor::*; +mod gen_GpuBindGroupLayoutEntry; +pub use gen_GpuBindGroupLayoutEntry::*; +mod gen_GpuBlendComponent; +pub use gen_GpuBlendComponent::*; +mod gen_GpuBlendFactor; +pub use gen_GpuBlendFactor::*; +mod gen_GpuBlendOperation; +pub use gen_GpuBlendOperation::*; +mod gen_GpuBlendState; +pub use gen_GpuBlendState::*; +mod gen_GpuBuffer; +pub use gen_GpuBuffer::*; +mod gen_GpuBufferBinding; +pub use gen_GpuBufferBinding::*; +mod gen_GpuBufferBindingLayout; +pub use gen_GpuBufferBindingLayout::*; +mod gen_GpuBufferBindingType; +pub use gen_GpuBufferBindingType::*; +mod gen_GpuBufferDescriptor; +pub use gen_GpuBufferDescriptor::*; +mod gen_GpuBufferMapState; +pub use gen_GpuBufferMapState::*; +mod gen_GpuCanvasAlphaMode; +pub use gen_GpuCanvasAlphaMode::*; +mod gen_GpuCanvasContext; +pub use gen_GpuCanvasContext::*; +mod gen_GpuCanvasConfiguration; +pub use gen_GpuCanvasConfiguration::*; +mod gen_GpuCanvasToneMapping; +pub use gen_GpuCanvasToneMapping::*; +mod gen_GpuCanvasToneMappingMode; +pub use gen_GpuCanvasToneMappingMode::*; +mod gen_GpuColorDict; +pub use gen_GpuColorDict::*; +mod gen_GpuColorTargetState; +pub use gen_GpuColorTargetState::*; +mod gen_GpuCommandBuffer; +pub use gen_GpuCommandBuffer::*; +mod gen_GpuCommandBufferDescriptor; +pub use gen_GpuCommandBufferDescriptor::*; +mod gen_GpuCommandEncoder; +pub use gen_GpuCommandEncoder::*; +mod gen_GpuCommandEncoderDescriptor; +pub use gen_GpuCommandEncoderDescriptor::*; +mod gen_GpuCompareFunction; +pub use gen_GpuCompareFunction::*; +mod gen_GpuCompilationInfo; +pub use gen_GpuCompilationInfo::*; +mod gen_GpuCompilationMessage; +pub use gen_GpuCompilationMessage::*; +mod gen_GpuCompilationMessageType; +pub use gen_GpuCompilationMessageType::*; +mod gen_GpuComputePassDescriptor; +pub use gen_GpuComputePassDescriptor::*; +mod gen_GpuComputePassEncoder; +pub use gen_GpuComputePassEncoder::*; +mod gen_GpuComputePassTimestampWrites; +pub use gen_GpuComputePassTimestampWrites::*; +mod gen_GpuComputePipeline; +pub use gen_GpuComputePipeline::*; +mod gen_GpuComputePipelineDescriptor; +pub use gen_GpuComputePipelineDescriptor::*; +mod gen_GpuCullMode; +pub use gen_GpuCullMode::*; +mod gen_GpuDepthStencilState; +pub use gen_GpuDepthStencilState::*; +mod gen_GpuDevice; +pub use gen_GpuDevice::*; +mod gen_GpuDeviceDescriptor; +pub use gen_GpuDeviceDescriptor::*; +mod gen_GpuDeviceLostInfo; +pub use gen_GpuDeviceLostInfo::*; +mod gen_GpuDeviceLostReason; +pub use gen_GpuDeviceLostReason::*; +mod gen_GpuError; +pub use gen_GpuError::*; +mod gen_GpuErrorFilter; +pub use gen_GpuErrorFilter::*; +mod gen_GpuExternalTexture; +pub use gen_GpuExternalTexture::*; +mod gen_GpuExternalTextureBindingLayout; +pub use gen_GpuExternalTextureBindingLayout::*; +mod gen_GpuExternalTextureDescriptor; +pub use gen_GpuExternalTextureDescriptor::*; +mod gen_GpuExtent3dDict; +pub use gen_GpuExtent3dDict::*; +mod gen_GpuFeatureName; +pub use gen_GpuFeatureName::*; +mod gen_GpuFilterMode; +pub use gen_GpuFilterMode::*; +mod gen_GpuFragmentState; +pub use gen_GpuFragmentState::*; +mod gen_GpuFrontFace; +pub use gen_GpuFrontFace::*; +mod gen_GpuTexelCopyBufferInfo; +pub use gen_GpuTexelCopyBufferInfo::*; +mod gen_GpuCopyExternalImageSourceInfo; +pub use gen_GpuCopyExternalImageSourceInfo::*; +mod gen_GpuTexelCopyTextureInfo; +pub use gen_GpuTexelCopyTextureInfo::*; +mod gen_GpuCopyExternalImageDestInfo; +pub use gen_GpuCopyExternalImageDestInfo::*; +mod gen_GpuTexelCopyBufferLayout; +pub use gen_GpuTexelCopyBufferLayout::*; +mod gen_GpuIndexFormat; +pub use gen_GpuIndexFormat::*; +mod gen_GpuLoadOp; +pub use gen_GpuLoadOp::*; +mod gen_gpu_map_mode; +pub use gen_gpu_map_mode::*; +mod gen_GpuMipmapFilterMode; +pub use gen_GpuMipmapFilterMode::*; +mod gen_GpuMultisampleState; +pub use gen_GpuMultisampleState::*; +mod gen_GpuObjectDescriptorBase; +pub use gen_GpuObjectDescriptorBase::*; +mod gen_GpuOrigin2dDict; +pub use gen_GpuOrigin2dDict::*; +mod gen_GpuOrigin3dDict; +pub use gen_GpuOrigin3dDict::*; +mod gen_GpuOutOfMemoryError; +pub use gen_GpuOutOfMemoryError::*; +mod gen_GpuPipelineDescriptorBase; +pub use gen_GpuPipelineDescriptorBase::*; +mod gen_GpuPipelineLayout; +pub use gen_GpuPipelineLayout::*; +mod gen_GpuPipelineLayoutDescriptor; +pub use gen_GpuPipelineLayoutDescriptor::*; +mod gen_GpuPowerPreference; +pub use gen_GpuPowerPreference::*; +mod gen_GpuPrimitiveState; +pub use gen_GpuPrimitiveState::*; +mod gen_GpuPrimitiveTopology; +pub use gen_GpuPrimitiveTopology::*; +mod gen_GpuProgrammableStage; +pub use gen_GpuProgrammableStage::*; +mod gen_GpuQuerySet; +pub use gen_GpuQuerySet::*; +mod gen_GpuQuerySetDescriptor; +pub use gen_GpuQuerySetDescriptor::*; +mod gen_GpuQueryType; +pub use gen_GpuQueryType::*; +mod gen_GpuQueue; +pub use gen_GpuQueue::*; +mod gen_GpuQueueDescriptor; +pub use gen_GpuQueueDescriptor::*; +mod gen_GpuRenderBundle; +pub use gen_GpuRenderBundle::*; +mod gen_GpuRenderBundleDescriptor; +pub use gen_GpuRenderBundleDescriptor::*; +mod gen_GpuRenderBundleEncoder; +pub use gen_GpuRenderBundleEncoder::*; +mod gen_GpuRenderBundleEncoderDescriptor; +pub use gen_GpuRenderBundleEncoderDescriptor::*; +mod gen_GpuRenderPassColorAttachment; +pub use gen_GpuRenderPassColorAttachment::*; +mod gen_GpuRenderPassDepthStencilAttachment; +pub use gen_GpuRenderPassDepthStencilAttachment::*; +mod gen_GpuRenderPassDescriptor; +pub use gen_GpuRenderPassDescriptor::*; +mod gen_GpuRenderPassEncoder; +pub use gen_GpuRenderPassEncoder::*; +mod gen_GpuRenderPassTimestampWrites; +pub use gen_GpuRenderPassTimestampWrites::*; +mod gen_GpuRenderPipeline; +pub use gen_GpuRenderPipeline::*; +mod gen_GpuRenderPipelineDescriptor; +pub use gen_GpuRenderPipelineDescriptor::*; +mod gen_GpuRequestAdapterOptions; +pub use gen_GpuRequestAdapterOptions::*; +mod gen_GpuSampler; +pub use gen_GpuSampler::*; +mod gen_GpuSamplerBindingLayout; +pub use gen_GpuSamplerBindingLayout::*; +mod gen_GpuSamplerBindingType; +pub use gen_GpuSamplerBindingType::*; +mod gen_GpuSamplerDescriptor; +pub use gen_GpuSamplerDescriptor::*; +mod gen_GpuShaderModule; +pub use gen_GpuShaderModule::*; +mod gen_GpuShaderModuleDescriptor; +pub use gen_GpuShaderModuleDescriptor::*; +mod gen_GpuStencilFaceState; +pub use gen_GpuStencilFaceState::*; +mod gen_GpuStencilOperation; +pub use gen_GpuStencilOperation::*; +mod gen_GpuStorageTextureAccess; +pub use gen_GpuStorageTextureAccess::*; +mod gen_GpuStorageTextureBindingLayout; +pub use gen_GpuStorageTextureBindingLayout::*; +mod gen_GpuStoreOp; +pub use gen_GpuStoreOp::*; +mod gen_GpuSupportedFeatures; +pub use gen_GpuSupportedFeatures::*; +mod gen_GpuSupportedLimits; +pub use gen_GpuSupportedLimits::*; +mod gen_GpuTexture; +pub use gen_GpuTexture::*; +mod gen_GpuTextureAspect; +pub use gen_GpuTextureAspect::*; +mod gen_GpuTextureBindingLayout; +pub use gen_GpuTextureBindingLayout::*; +mod gen_GpuTextureDescriptor; +pub use gen_GpuTextureDescriptor::*; +mod gen_GpuTextureDimension; +pub use gen_GpuTextureDimension::*; +mod gen_GpuTextureFormat; +pub use gen_GpuTextureFormat::*; +mod gen_GpuTextureSampleType; +pub use gen_GpuTextureSampleType::*; +mod gen_GpuTextureView; +pub use gen_GpuTextureView::*; +mod gen_GpuTextureViewDescriptor; +pub use gen_GpuTextureViewDescriptor::*; +mod gen_GpuTextureViewDimension; +pub use gen_GpuTextureViewDimension::*; +mod gen_GpuUncapturedErrorEvent; +pub use gen_GpuUncapturedErrorEvent::*; +mod gen_GpuUncapturedErrorEventInit; +pub use gen_GpuUncapturedErrorEventInit::*; +mod gen_GpuValidationError; +pub use gen_GpuValidationError::*; +mod gen_GpuVertexAttribute; +pub use gen_GpuVertexAttribute::*; +mod gen_GpuVertexBufferLayout; +pub use gen_GpuVertexBufferLayout::*; +mod gen_GpuVertexFormat; +pub use gen_GpuVertexFormat::*; +mod gen_GpuVertexState; +pub use gen_GpuVertexState::*; +mod gen_GpuVertexStepMode; +pub use gen_GpuVertexStepMode::*; +mod gen_WgslLanguageFeatures; +pub use gen_WgslLanguageFeatures::*; diff --git a/third_party/wgpu-29.0.4-patched/src/backend/wgpu_core.rs b/third_party/wgpu-29.0.4-patched/src/backend/wgpu_core.rs new file mode 100644 index 00000000..7db39662 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/wgpu_core.rs @@ -0,0 +1,4036 @@ +use alloc::{ + borrow::Cow::{self, Borrowed}, + boxed::Box, + format, + string::{String, ToString as _}, + sync::Arc, + vec, + vec::Vec, +}; +use core::{ + error::Error, + fmt, + future::ready, + ops::{Deref, Range}, + pin::Pin, + ptr::NonNull, + slice, +}; +use hashbrown::HashMap; + +use arrayvec::ArrayVec; +use smallvec::SmallVec; +use wgc::{ + command::bundle_ffi::*, error::ContextErrorSource, pipeline::CreateShaderModuleError, + resource::BlasPrepareCompactResult, +}; +use wgt::{ + error::{ErrorType, WebGpuError}, + WasmNotSendSync, +}; + +use crate::{ + api, + dispatch::{self, BlasCompactCallback, BufferMappedRangeInterface}, + BindingResource, Blas, BufferBinding, BufferDescriptor, CompilationInfo, CompilationMessage, + CompilationMessageType, ErrorSource, Features, Label, LoadOp, MapMode, Operations, + ShaderSource, SurfaceTargetUnsafe, TextureDescriptor, Tlas, WriteOnly, +}; +use crate::{dispatch::DispatchAdapter, util::Mutex}; + +mod thread_id; + +#[derive(Clone)] +pub struct ContextWgpuCore(Arc); + +impl Drop for ContextWgpuCore { + fn drop(&mut self) { + //nothing + } +} + +impl fmt::Debug for ContextWgpuCore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ContextWgpuCore") + .field("type", &"Native") + .finish() + } +} + +impl ContextWgpuCore { + pub unsafe fn from_hal_instance(hal_instance: A::Instance) -> Self { + Self(unsafe { + Arc::new(wgc::global::Global::from_hal_instance::( + "wgpu", + hal_instance, + )) + }) + } + + /// # Safety + /// + /// - The raw instance handle returned must not be manually destroyed. + pub unsafe fn instance_as_hal(&self) -> Option<&A::Instance> { + unsafe { self.0.instance_as_hal::() } + } + + pub unsafe fn from_core_instance(core_instance: wgc::instance::Instance) -> Self { + Self(unsafe { Arc::new(wgc::global::Global::from_instance(core_instance)) }) + } + + #[cfg(wgpu_core)] + pub fn enumerate_adapters(&self, backends: wgt::Backends) -> Vec { + self.0.enumerate_adapters(backends) + } + + pub unsafe fn create_adapter_from_hal( + &self, + hal_adapter: hal::ExposedAdapter, + ) -> wgc::id::AdapterId { + unsafe { self.0.create_adapter_from_hal(hal_adapter.into(), None) } + } + + pub unsafe fn adapter_as_hal( + &self, + adapter: &CoreAdapter, + ) -> Option + WasmNotSendSync> { + unsafe { self.0.adapter_as_hal::(adapter.id) } + } + + pub unsafe fn buffer_as_hal( + &self, + buffer: &CoreBuffer, + ) -> Option> { + unsafe { self.0.buffer_as_hal::(buffer.id) } + } + + #[doc(hidden)] + pub unsafe fn buffer_mark_external_write_initialized( + &self, + buffer: &CoreBuffer, + range: core::ops::Range, + ) -> Result<(), crate::ExternalWriteInitializationError> { + match unsafe { + self.0 + .buffer_mark_external_write_initialized(buffer.id, range) + } { + Ok(()) => Ok(()), + Err(error) => { + self.handle_error_nolabel( + &buffer.error_sink, + error, + "Buffer::mark_external_write_initialized", + ); + Err(crate::ExternalWriteInitializationError { _private: () }) + } + } + } + + pub unsafe fn create_device_from_hal( + &self, + adapter: &CoreAdapter, + hal_device: hal::OpenDevice, + desc: &crate::DeviceDescriptor<'_>, + ) -> Result<(CoreDevice, CoreQueue), crate::RequestDeviceError> { + let (device_id, queue_id) = unsafe { + self.0.create_device_from_hal( + adapter.id, + hal_device.into(), + &desc.map_label(|l| l.map(Borrowed)), + None, + None, + ) + }?; + let error_sink = Arc::new(Mutex::new(ErrorSinkRaw::new())); + let device = CoreDevice { + context: self.clone(), + id: device_id, + error_sink: error_sink.clone(), + features: desc.required_features, + }; + let queue = CoreQueue { + context: self.clone(), + id: queue_id, + error_sink, + }; + Ok((device, queue)) + } + + pub unsafe fn create_texture_from_hal( + &self, + hal_texture: A::Texture, + device: &CoreDevice, + desc: &TextureDescriptor<'_>, + ) -> CoreTexture { + let descriptor = desc.map_label_and_view_formats(|l| l.map(Borrowed), |v| v.to_vec()); + let (id, error) = unsafe { + self.0 + .create_texture_from_hal(Box::new(hal_texture), device.id, &descriptor, None) + }; + if let Some(cause) = error { + self.handle_error( + &device.error_sink, + cause, + desc.label, + "Device::create_texture_from_hal", + ); + } + CoreTexture { + context: self.clone(), + id, + error_sink: Arc::clone(&device.error_sink), + } + } + + /// # Safety + /// + /// - `hal_buffer` must be created from `device`. + /// - `hal_buffer` must be created respecting `desc` + /// - `hal_buffer` must be initialized + /// - `hal_buffer` must not have zero size. + pub unsafe fn create_buffer_from_hal( + &self, + hal_buffer: A::Buffer, + device: &CoreDevice, + desc: &BufferDescriptor<'_>, + ) -> CoreBuffer { + let (id, error) = unsafe { + self.0.create_buffer_from_hal::( + hal_buffer, + device.id, + &desc.map_label(|l| l.map(Borrowed)), + None, + ) + }; + if let Some(cause) = error { + self.handle_error( + &device.error_sink, + cause, + desc.label, + "Device::create_buffer_from_hal", + ); + } + CoreBuffer { + context: self.clone(), + id, + error_sink: Arc::clone(&device.error_sink), + } + } + + pub unsafe fn device_as_hal( + &self, + device: &CoreDevice, + ) -> Option> { + unsafe { self.0.device_as_hal::(device.id) } + } + + pub unsafe fn surface_as_hal( + &self, + surface: &CoreSurface, + ) -> Option> { + unsafe { self.0.surface_as_hal::(surface.id) } + } + + pub unsafe fn texture_as_hal( + &self, + texture: &CoreTexture, + ) -> Option> { + unsafe { self.0.texture_as_hal::(texture.id) } + } + + pub unsafe fn texture_view_as_hal( + &self, + texture_view: &CoreTextureView, + ) -> Option> { + unsafe { self.0.texture_view_as_hal::(texture_view.id) } + } + + /// This method will start the wgpu_core level command recording. + pub unsafe fn command_encoder_as_hal_mut< + A: hal::Api, + F: FnOnce(Option<&mut A::CommandEncoder>) -> R, + R, + >( + &self, + command_encoder: &CoreCommandEncoder, + hal_command_encoder_callback: F, + ) -> R { + unsafe { + self.0.command_encoder_as_hal_mut::( + command_encoder.id, + hal_command_encoder_callback, + ) + } + } + + pub unsafe fn blas_as_hal( + &self, + blas: &CoreBlas, + ) -> Option> { + unsafe { self.0.blas_as_hal::(blas.id) } + } + + pub unsafe fn tlas_as_hal( + &self, + tlas: &CoreTlas, + ) -> Option> { + unsafe { self.0.tlas_as_hal::(tlas.id) } + } + + pub fn generate_report(&self) -> wgc::global::GlobalReport { + self.0.generate_report() + } + + #[cold] + #[track_caller] + #[inline(never)] + fn handle_error_inner( + &self, + sink_mutex: &Mutex, + error_type: ErrorType, + source: ContextErrorSource, + label: Label<'_>, + fn_ident: &'static str, + ) { + let source: ErrorSource = Box::new(wgc::error::ContextError { + fn_ident, + source, + label: label.unwrap_or_default().to_string(), + }); + let final_error_handling = { + let mut sink = sink_mutex.lock(); + let description = || self.format_error(&*source); + let error = match error_type { + ErrorType::Internal => { + let description = description(); + crate::Error::Internal { + source, + description, + } + } + ErrorType::OutOfMemory => crate::Error::OutOfMemory { source }, + ErrorType::Validation => { + let description = description(); + crate::Error::Validation { + source, + description, + } + } + ErrorType::DeviceLost => return, // will be surfaced via callback + }; + sink.handle_error_or_return_handler(error) + }; + + if let Some(f) = final_error_handling { + // If the user has provided their own `uncaptured_handler` callback, invoke it now, + // having released our lock on `sink_mutex`. See the comments on + // `handle_error_or_return_handler` for details. + f(); + } + } + + #[inline] + #[track_caller] + fn handle_error( + &self, + sink_mutex: &Mutex, + source: impl WebGpuError + WasmNotSendSync + 'static, + label: Label<'_>, + fn_ident: &'static str, + ) { + let error_type = source.webgpu_error_type(); + self.handle_error_inner(sink_mutex, error_type, Box::new(source), label, fn_ident) + } + + #[inline] + #[track_caller] + fn handle_error_nolabel( + &self, + sink_mutex: &Mutex, + source: impl WebGpuError + WasmNotSendSync + 'static, + fn_ident: &'static str, + ) { + let error_type = source.webgpu_error_type(); + self.handle_error_inner(sink_mutex, error_type, Box::new(source), None, fn_ident) + } + + #[track_caller] + #[cold] + fn handle_error_fatal( + &self, + cause: impl Error + WasmNotSendSync + 'static, + operation: &'static str, + ) -> ! { + panic!("Error in {operation}: {f}", f = self.format_error(&cause)); + } + + #[inline(never)] + fn format_error(&self, err: &(dyn Error + 'static)) -> String { + let mut output = String::new(); + let mut level = 1; + + fn print_tree(output: &mut String, level: &mut usize, e: &(dyn Error + 'static)) { + let mut print = |e: &(dyn Error + 'static)| { + use core::fmt::Write; + writeln!(output, "{}{}", " ".repeat(*level * 2), e).unwrap(); + + if let Some(e) = e.source() { + *level += 1; + print_tree(output, level, e); + *level -= 1; + } + }; + if let Some(multi) = e.downcast_ref::() { + for e in multi.errors() { + print(e); + } + } else { + print(e); + } + } + + print_tree(&mut output, &mut level, err); + + format!("Validation Error\n\nCaused by:\n{output}") + } + + pub unsafe fn queue_as_hal( + &self, + queue: &CoreQueue, + ) -> Option + WasmNotSendSync> { + unsafe { self.0.queue_as_hal::(queue.id) } + } +} + +fn map_buffer_copy_view( + view: crate::TexelCopyBufferInfo<'_>, +) -> wgt::TexelCopyBufferInfo { + wgt::TexelCopyBufferInfo { + buffer: view.buffer.inner.as_core().id, + layout: view.layout, + } +} + +fn map_texture_copy_view( + view: crate::TexelCopyTextureInfo<'_>, +) -> wgt::TexelCopyTextureInfo { + wgt::TexelCopyTextureInfo { + texture: view.texture.inner.as_core().id, + mip_level: view.mip_level, + origin: view.origin, + aspect: view.aspect, + } +} + +#[cfg_attr(not(webgl), expect(unused))] +fn map_texture_tagged_copy_view( + view: crate::CopyExternalImageDestInfo<&api::Texture>, +) -> wgt::CopyExternalImageDestInfo { + wgt::CopyExternalImageDestInfo { + texture: view.texture.inner.as_core().id, + mip_level: view.mip_level, + origin: view.origin, + aspect: view.aspect, + color_space: view.color_space, + premultiplied_alpha: view.premultiplied_alpha, + } +} + +fn map_load_op(load: &LoadOp) -> LoadOp> { + match *load { + LoadOp::Clear(clear_value) => LoadOp::Clear(Some(clear_value)), + LoadOp::DontCare(token) => LoadOp::DontCare(token), + LoadOp::Load => LoadOp::Load, + } +} + +fn map_pass_channel(ops: Option<&Operations>) -> wgc::command::PassChannel> { + match ops { + Some(&Operations { load, store }) => wgc::command::PassChannel { + load_op: Some(map_load_op(&load)), + store_op: Some(store), + read_only: false, + }, + None => wgc::command::PassChannel { + load_op: None, + store_op: None, + read_only: true, + }, + } +} + +#[derive(Debug)] +pub struct CoreSurface { + pub(crate) context: ContextWgpuCore, + id: wgc::id::SurfaceId, + /// Configured device is needed to know which backend + /// code to execute when acquiring a new frame. + configured_device: Mutex>, + /// The error sink with which to report errors. + /// `None` if the surface has not been configured. + error_sink: Mutex>, +} + +#[derive(Debug)] +pub struct CoreAdapter { + pub(crate) context: ContextWgpuCore, + pub(crate) id: wgc::id::AdapterId, +} + +#[derive(Debug)] +pub struct CoreDevice { + pub(crate) context: ContextWgpuCore, + id: wgc::id::DeviceId, + error_sink: ErrorSink, + features: Features, +} + +#[derive(Debug)] +pub struct CoreBuffer { + pub(crate) context: ContextWgpuCore, + id: wgc::id::BufferId, + error_sink: ErrorSink, +} + +#[derive(Debug)] +pub struct CoreShaderModule { + pub(crate) context: ContextWgpuCore, + id: wgc::id::ShaderModuleId, + compilation_info: CompilationInfo, +} + +#[derive(Debug)] +pub struct CoreBindGroupLayout { + pub(crate) context: ContextWgpuCore, + id: wgc::id::BindGroupLayoutId, +} + +#[derive(Debug)] +pub struct CoreBindGroup { + pub(crate) context: ContextWgpuCore, + id: wgc::id::BindGroupId, +} + +#[derive(Debug)] +pub struct CoreTexture { + pub(crate) context: ContextWgpuCore, + id: wgc::id::TextureId, + error_sink: ErrorSink, +} + +#[derive(Debug)] +pub struct CoreTextureView { + pub(crate) context: ContextWgpuCore, + id: wgc::id::TextureViewId, +} + +#[derive(Debug)] +pub struct CoreExternalTexture { + pub(crate) context: ContextWgpuCore, + id: wgc::id::ExternalTextureId, +} + +#[derive(Debug)] +pub struct CoreSampler { + pub(crate) context: ContextWgpuCore, + id: wgc::id::SamplerId, +} + +#[derive(Debug)] +pub struct CoreQuerySet { + pub(crate) context: ContextWgpuCore, + id: wgc::id::QuerySetId, +} + +#[derive(Debug)] +pub struct CorePipelineLayout { + pub(crate) context: ContextWgpuCore, + id: wgc::id::PipelineLayoutId, +} + +#[derive(Debug)] +pub struct CorePipelineCache { + pub(crate) context: ContextWgpuCore, + id: wgc::id::PipelineCacheId, +} + +#[derive(Debug)] +pub struct CoreCommandBuffer { + pub(crate) context: ContextWgpuCore, + id: wgc::id::CommandBufferId, +} + +#[derive(Debug)] +pub struct CoreRenderBundleEncoder { + pub(crate) context: ContextWgpuCore, + encoder: wgc::command::RenderBundleEncoder, + id: crate::cmp::Identifier, +} + +#[derive(Debug)] +pub struct CoreRenderBundle { + context: ContextWgpuCore, + id: wgc::id::RenderBundleId, +} + +#[derive(Debug)] +pub struct CoreQueue { + pub(crate) context: ContextWgpuCore, + id: wgc::id::QueueId, + error_sink: ErrorSink, +} + +#[derive(Debug)] +pub struct CoreComputePipeline { + pub(crate) context: ContextWgpuCore, + id: wgc::id::ComputePipelineId, + error_sink: ErrorSink, +} + +#[derive(Debug)] +pub struct CoreRenderPipeline { + pub(crate) context: ContextWgpuCore, + id: wgc::id::RenderPipelineId, + error_sink: ErrorSink, +} + +#[derive(Debug)] +pub struct CoreComputePass { + pub(crate) context: ContextWgpuCore, + pass: wgc::command::ComputePass, + error_sink: ErrorSink, + id: crate::cmp::Identifier, +} + +#[derive(Debug)] +pub struct CoreRenderPass { + pub(crate) context: ContextWgpuCore, + pass: wgc::command::RenderPass, + error_sink: ErrorSink, + id: crate::cmp::Identifier, +} + +#[derive(Debug)] +pub struct CoreCommandEncoder { + pub(crate) context: ContextWgpuCore, + id: wgc::id::CommandEncoderId, + error_sink: ErrorSink, +} + +#[derive(Debug)] +pub struct CoreBlas { + pub(crate) context: ContextWgpuCore, + id: wgc::id::BlasId, + error_sink: ErrorSink, +} + +#[derive(Debug)] +pub struct CoreTlas { + pub(crate) context: ContextWgpuCore, + id: wgc::id::TlasId, + // error_sink: ErrorSink, +} + +#[derive(Debug)] +pub struct CoreSurfaceOutputDetail { + context: ContextWgpuCore, + surface_id: wgc::id::SurfaceId, + error_sink: ErrorSink, +} + +type ErrorSink = Arc>; + +struct ErrorScope { + error: Option, + filter: crate::ErrorFilter, +} + +struct ErrorSinkRaw { + scopes: HashMap>, + uncaptured_handler: Option>, +} + +impl ErrorSinkRaw { + fn new() -> ErrorSinkRaw { + ErrorSinkRaw { + scopes: HashMap::new(), + uncaptured_handler: None, + } + } + + /// Deliver the error to + /// + /// * the innermost error scope, if any, or + /// * the uncaptured error handler, if there is one, or + /// * [`default_error_handler()`]. + /// + /// If a closure is returned, the caller should call it immediately after dropping the + /// [`ErrorSink`] mutex guard. This makes sure that the user callback is not called with + /// a wgpu mutex held. + #[track_caller] + #[must_use] + fn handle_error_or_return_handler(&mut self, err: crate::Error) -> Option { + let filter = match err { + crate::Error::OutOfMemory { .. } => crate::ErrorFilter::OutOfMemory, + crate::Error::Validation { .. } => crate::ErrorFilter::Validation, + crate::Error::Internal { .. } => crate::ErrorFilter::Internal, + }; + let thread_id = thread_id::ThreadId::current(); + let scopes = self.scopes.entry(thread_id).or_default(); + match scopes.iter_mut().rev().find(|scope| scope.filter == filter) { + Some(scope) => { + if scope.error.is_none() { + scope.error = Some(err); + } + None + } + None => { + if let Some(custom_handler) = &self.uncaptured_handler { + let custom_handler = Arc::clone(custom_handler); + Some(move || (custom_handler)(err)) + } else { + // direct call preserves #[track_caller] where dyn can't + default_error_handler(err) + } + } + } + } +} + +impl fmt::Debug for ErrorSinkRaw { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "ErrorSink") + } +} + +#[track_caller] +fn default_error_handler(err: crate::Error) -> ! { + log::error!("Handling wgpu errors as fatal by default"); + panic!("wgpu error: {err}\n"); +} + +impl From for CompilationInfo { + fn from(value: CreateShaderModuleError) -> Self { + match value { + #[cfg(feature = "wgsl")] + CreateShaderModuleError::Parsing(v) => v.into(), + #[cfg(feature = "glsl")] + CreateShaderModuleError::ParsingGlsl(v) => v.into(), + #[cfg(feature = "spirv")] + CreateShaderModuleError::ParsingSpirV(v) => v.into(), + CreateShaderModuleError::Validation(v) => v.into(), + // Device errors are reported through the error sink, and are not compilation errors. + // Same goes for native shader module generation errors. + CreateShaderModuleError::Device(_) | CreateShaderModuleError::Generation => { + CompilationInfo { + messages: Vec::new(), + } + } + // Everything else is an error message without location information. + _ => CompilationInfo { + messages: vec![CompilationMessage { + message: value.to_string(), + message_type: CompilationMessageType::Error, + location: None, + }], + }, + } + } +} + +#[derive(Debug)] +pub struct CoreQueueWriteBuffer { + buffer_id: wgc::id::StagingBufferId, + mapping: CoreBufferMappedRange, +} + +#[derive(Debug)] +pub struct CoreBufferMappedRange { + ptr: NonNull, + size: usize, +} + +#[cfg(send_sync)] +unsafe impl Send for CoreBufferMappedRange {} +#[cfg(send_sync)] +unsafe impl Sync for CoreBufferMappedRange {} + +impl Drop for CoreBufferMappedRange { + fn drop(&mut self) { + // Intentionally left blank so that `BufferMappedRange` still + // implements `Drop`, to match the web backend + } +} + +crate::cmp::impl_eq_ord_hash_arc_address!(ContextWgpuCore => .0); +crate::cmp::impl_eq_ord_hash_proxy!(CoreAdapter => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreDevice => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreQueue => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreShaderModule => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreBindGroupLayout => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreBindGroup => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreTextureView => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreSampler => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreBuffer => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreTexture => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreExternalTexture => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreBlas => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreTlas => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreQuerySet => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CorePipelineLayout => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreRenderPipeline => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreComputePipeline => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CorePipelineCache => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreCommandEncoder => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreComputePass => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreRenderPass => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreCommandBuffer => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreRenderBundleEncoder => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreRenderBundle => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreSurface => .id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreSurfaceOutputDetail => .surface_id); +crate::cmp::impl_eq_ord_hash_proxy!(CoreQueueWriteBuffer => .mapping.ptr); +crate::cmp::impl_eq_ord_hash_proxy!(CoreBufferMappedRange => .ptr); + +impl dispatch::InstanceInterface for ContextWgpuCore { + fn new(desc: wgt::InstanceDescriptor) -> Self + where + Self: Sized, + { + Self(Arc::new(wgc::global::Global::new("wgpu", desc, None))) + } + + unsafe fn create_surface( + &self, + target: crate::api::SurfaceTargetUnsafe, + ) -> Result { + let id = match target { + SurfaceTargetUnsafe::RawHandle { + raw_display_handle, + raw_window_handle, + } => unsafe { + self.0 + .instance_create_surface(raw_display_handle, raw_window_handle, None) + }, + + #[cfg(all( + unix, + not(target_vendor = "apple"), + not(target_family = "wasm"), + not(target_os = "netbsd") + ))] + SurfaceTargetUnsafe::Drm { + fd, + plane, + connector_id, + width, + height, + refresh_rate, + } => unsafe { + self.0.instance_create_surface_from_drm( + fd, + plane, + connector_id, + width, + height, + refresh_rate, + None, + ) + }, + + #[cfg(metal)] + SurfaceTargetUnsafe::CoreAnimationLayer(layer) => unsafe { + self.0.instance_create_surface_metal(layer, None) + }, + + #[cfg(target_os = "netbsd")] + SurfaceTargetUnsafe::Drm { .. } => Err( + wgc::instance::CreateSurfaceError::BackendNotEnabled(wgt::Backend::Vulkan), + ), + + #[cfg(dx12)] + SurfaceTargetUnsafe::CompositionVisual(visual) => unsafe { + self.0.instance_create_surface_from_visual(visual, None) + }, + + #[cfg(dx12)] + SurfaceTargetUnsafe::SurfaceHandle(surface_handle) => unsafe { + self.0 + .instance_create_surface_from_surface_handle(surface_handle, None) + }, + + #[cfg(dx12)] + SurfaceTargetUnsafe::SwapChainPanel(swap_chain_panel) => unsafe { + self.0 + .instance_create_surface_from_swap_chain_panel(swap_chain_panel, None) + }, + }?; + + Ok(CoreSurface { + context: self.clone(), + id, + configured_device: Mutex::default(), + error_sink: Mutex::default(), + } + .into()) + } + + fn request_adapter( + &self, + options: &crate::api::RequestAdapterOptions<'_, '_>, + ) -> Pin> { + let id = self.0.request_adapter( + &wgc::instance::RequestAdapterOptions { + power_preference: options.power_preference, + force_fallback_adapter: options.force_fallback_adapter, + compatible_surface: options + .compatible_surface + .map(|surface| surface.inner.as_core().id), + }, + wgt::Backends::all(), + None, + ); + let adapter = id.map(|id| { + let core = CoreAdapter { + context: self.clone(), + id, + }; + let generic: dispatch::DispatchAdapter = core.into(); + generic + }); + Box::pin(ready(adapter)) + } + + fn poll_all_devices(&self, force_wait: bool) -> bool { + match self.0.poll_all_devices(force_wait) { + Ok(all_queue_empty) => all_queue_empty, + Err(err) => self.handle_error_fatal(err, "Instance::poll_all_devices"), + } + } + + #[cfg(feature = "wgsl")] + fn wgsl_language_features(&self) -> crate::WgslLanguageFeatures { + use wgc::naga::front::wgsl::ImplementedLanguageExtension; + ImplementedLanguageExtension::all().iter().copied().fold( + crate::WgslLanguageFeatures::empty(), + |acc, wle| { + acc | match wle { + ImplementedLanguageExtension::ReadOnlyAndReadWriteStorageTextures => { + crate::WgslLanguageFeatures::ReadOnlyAndReadWriteStorageTextures + } + ImplementedLanguageExtension::Packed4x8IntegerDotProduct => { + crate::WgslLanguageFeatures::Packed4x8IntegerDotProduct + } + ImplementedLanguageExtension::PointerCompositeAccess => { + crate::WgslLanguageFeatures::PointerCompositeAccess + } + } + }, + ) + } + + fn enumerate_adapters( + &self, + backends: crate::Backends, + ) -> Pin> { + let adapters: Vec = self + .enumerate_adapters(backends) + .into_iter() + .map(|adapter| { + let core = crate::backend::wgpu_core::CoreAdapter { + context: self.clone(), + id: adapter, + }; + core.into() + }) + .collect(); + Box::pin(ready(adapters)) + } +} + +impl dispatch::AdapterInterface for CoreAdapter { + fn request_device( + &self, + desc: &crate::DeviceDescriptor<'_>, + ) -> Pin> { + let res = self.context.0.adapter_request_device( + self.id, + &desc.map_label(|l| l.map(Borrowed)), + None, + None, + ); + let (device_id, queue_id) = match res { + Ok(ids) => ids, + Err(err) => { + return Box::pin(ready(Err(err.into()))); + } + }; + let error_sink = Arc::new(Mutex::new(ErrorSinkRaw::new())); + let device = CoreDevice { + context: self.context.clone(), + id: device_id, + error_sink: error_sink.clone(), + features: desc.required_features, + }; + let queue = CoreQueue { + context: self.context.clone(), + id: queue_id, + error_sink, + }; + Box::pin(ready(Ok((device.into(), queue.into())))) + } + + fn is_surface_supported(&self, surface: &dispatch::DispatchSurface) -> bool { + let surface = surface.as_core(); + + self.context + .0 + .adapter_is_surface_supported(self.id, surface.id) + } + + fn features(&self) -> crate::Features { + self.context.0.adapter_features(self.id) + } + + fn limits(&self) -> crate::Limits { + self.context.0.adapter_limits(self.id) + } + + fn downlevel_capabilities(&self) -> crate::DownlevelCapabilities { + self.context.0.adapter_downlevel_capabilities(self.id) + } + + fn get_info(&self) -> crate::AdapterInfo { + self.context.0.adapter_get_info(self.id) + } + + fn get_texture_format_features( + &self, + format: crate::TextureFormat, + ) -> crate::TextureFormatFeatures { + self.context + .0 + .adapter_get_texture_format_features(self.id, format) + } + + fn get_presentation_timestamp(&self) -> crate::PresentationTimestamp { + self.context.0.adapter_get_presentation_timestamp(self.id) + } + + fn cooperative_matrix_properties(&self) -> Vec { + self.context + .0 + .adapter_cooperative_matrix_properties(self.id) + } +} + +impl Drop for CoreAdapter { + fn drop(&mut self) { + self.context.0.adapter_drop(self.id) + } +} + +impl dispatch::DeviceInterface for CoreDevice { + fn features(&self) -> crate::Features { + self.context.0.device_features(self.id) + } + + fn limits(&self) -> crate::Limits { + self.context.0.device_limits(self.id) + } + + fn adapter_info(&self) -> crate::AdapterInfo { + self.context.0.device_adapter_info(self.id) + } + + // If we have no way to create a shader module, we can't return one, and so most of the function is unreachable. + #[cfg_attr( + not(any( + feature = "spirv", + feature = "glsl", + feature = "wgsl", + feature = "naga-ir" + )), + expect(unused) + )] + fn create_shader_module( + &self, + desc: crate::ShaderModuleDescriptor<'_>, + shader_bound_checks: wgt::ShaderRuntimeChecks, + ) -> dispatch::DispatchShaderModule { + let descriptor = wgc::pipeline::ShaderModuleDescriptor { + label: desc.label.map(Borrowed), + runtime_checks: shader_bound_checks, + }; + let source = match desc.source { + #[cfg(feature = "spirv")] + ShaderSource::SpirV(ref spv) => { + // Parse the given shader code and store its representation. + let options = naga::front::spv::Options { + adjust_coordinate_space: false, // we require NDC_Y_UP feature + strict_capabilities: true, + block_ctx_dump_prefix: None, + }; + wgc::pipeline::ShaderModuleSource::SpirV(Borrowed(spv), options) + } + #[cfg(feature = "glsl")] + ShaderSource::Glsl { + ref shader, + stage, + defines, + } => { + let options = naga::front::glsl::Options { + stage, + defines: defines + .iter() + .map(|&(key, value)| (String::from(key), String::from(value))) + .collect(), + }; + wgc::pipeline::ShaderModuleSource::Glsl(Borrowed(shader), options) + } + #[cfg(feature = "wgsl")] + ShaderSource::Wgsl(ref code) => wgc::pipeline::ShaderModuleSource::Wgsl(Borrowed(code)), + #[cfg(feature = "naga-ir")] + ShaderSource::Naga(module) => wgc::pipeline::ShaderModuleSource::Naga(module), + ShaderSource::Dummy(_) => panic!("found `ShaderSource::Dummy`"), + }; + let (id, error) = + self.context + .0 + .device_create_shader_module(self.id, &descriptor, source, None); + let compilation_info = match error { + Some(cause) => { + self.context.handle_error( + &self.error_sink, + cause.clone(), + desc.label, + "Device::create_shader_module", + ); + CompilationInfo::from(cause) + } + None => CompilationInfo { messages: vec![] }, + }; + + CoreShaderModule { + context: self.context.clone(), + id, + compilation_info, + } + .into() + } + + unsafe fn create_shader_module_passthrough( + &self, + desc: &crate::ShaderModuleDescriptorPassthrough<'_>, + ) -> dispatch::DispatchShaderModule { + let desc = desc.map_label(|l| l.map(Cow::from)); + let (id, error) = unsafe { + self.context + .0 + .device_create_shader_module_passthrough(self.id, &desc, None) + }; + + let compilation_info = match error { + Some(cause) => { + self.context.handle_error( + &self.error_sink, + cause.clone(), + desc.label.as_deref(), + "Device::create_shader_module_passthrough", + ); + CompilationInfo::from(cause) + } + None => CompilationInfo { messages: vec![] }, + }; + + CoreShaderModule { + context: self.context.clone(), + id, + compilation_info, + } + .into() + } + + fn create_bind_group_layout( + &self, + desc: &crate::BindGroupLayoutDescriptor<'_>, + ) -> dispatch::DispatchBindGroupLayout { + let descriptor = wgc::binding_model::BindGroupLayoutDescriptor { + label: desc.label.map(Borrowed), + entries: Borrowed(desc.entries), + }; + let (id, error) = + self.context + .0 + .device_create_bind_group_layout(self.id, &descriptor, None); + if let Some(cause) = error { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_bind_group_layout", + ); + } + CoreBindGroupLayout { + context: self.context.clone(), + id, + } + .into() + } + + fn create_bind_group( + &self, + desc: &crate::BindGroupDescriptor<'_>, + ) -> dispatch::DispatchBindGroup { + use wgc::binding_model as bm; + + let mut arrayed_texture_views = Vec::new(); + let mut arrayed_samplers = Vec::new(); + if self.features.contains(Features::TEXTURE_BINDING_ARRAY) { + // gather all the array view IDs first + for entry in desc.entries.iter() { + if let BindingResource::TextureViewArray(array) = entry.resource { + arrayed_texture_views.extend(array.iter().map(|view| view.inner.as_core().id)); + } + if let BindingResource::SamplerArray(array) = entry.resource { + arrayed_samplers.extend(array.iter().map(|sampler| sampler.inner.as_core().id)); + } + } + } + let mut remaining_arrayed_texture_views = &arrayed_texture_views[..]; + let mut remaining_arrayed_samplers = &arrayed_samplers[..]; + + let mut arrayed_buffer_bindings = Vec::new(); + if self.features.contains(Features::BUFFER_BINDING_ARRAY) { + // gather all the buffers first + for entry in desc.entries.iter() { + if let BindingResource::BufferArray(array) = entry.resource { + arrayed_buffer_bindings.extend(array.iter().map(|binding| bm::BufferBinding { + buffer: binding.buffer.inner.as_core().id, + offset: binding.offset, + size: binding.size.map(wgt::BufferSize::get), + })); + } + } + } + let mut remaining_arrayed_buffer_bindings = &arrayed_buffer_bindings[..]; + + let mut arrayed_acceleration_structures = Vec::new(); + if self + .features + .contains(Features::ACCELERATION_STRUCTURE_BINDING_ARRAY) + { + // Gather all the TLAS IDs used by TLAS arrays first (same pattern as other arrayed resources). + for entry in desc.entries.iter() { + if let BindingResource::AccelerationStructureArray(array) = entry.resource { + arrayed_acceleration_structures + .extend(array.iter().map(|tlas| tlas.inner.as_core().id)); + } + } + } + let mut remaining_arrayed_acceleration_structures = &arrayed_acceleration_structures[..]; + + let entries = desc + .entries + .iter() + .map(|entry| bm::BindGroupEntry { + binding: entry.binding, + resource: match entry.resource { + BindingResource::Buffer(BufferBinding { + buffer, + offset, + size, + }) => bm::BindingResource::Buffer(bm::BufferBinding { + buffer: buffer.inner.as_core().id, + offset, + size: size.map(wgt::BufferSize::get), + }), + BindingResource::BufferArray(array) => { + let slice = &remaining_arrayed_buffer_bindings[..array.len()]; + remaining_arrayed_buffer_bindings = + &remaining_arrayed_buffer_bindings[array.len()..]; + bm::BindingResource::BufferArray(Borrowed(slice)) + } + BindingResource::Sampler(sampler) => { + bm::BindingResource::Sampler(sampler.inner.as_core().id) + } + BindingResource::SamplerArray(array) => { + let slice = &remaining_arrayed_samplers[..array.len()]; + remaining_arrayed_samplers = &remaining_arrayed_samplers[array.len()..]; + bm::BindingResource::SamplerArray(Borrowed(slice)) + } + BindingResource::TextureView(texture_view) => { + bm::BindingResource::TextureView(texture_view.inner.as_core().id) + } + BindingResource::TextureViewArray(array) => { + let slice = &remaining_arrayed_texture_views[..array.len()]; + remaining_arrayed_texture_views = + &remaining_arrayed_texture_views[array.len()..]; + bm::BindingResource::TextureViewArray(Borrowed(slice)) + } + BindingResource::AccelerationStructure(acceleration_structure) => { + bm::BindingResource::AccelerationStructure( + acceleration_structure.inner.as_core().id, + ) + } + BindingResource::AccelerationStructureArray(array) => { + let slice = &remaining_arrayed_acceleration_structures[..array.len()]; + remaining_arrayed_acceleration_structures = + &remaining_arrayed_acceleration_structures[array.len()..]; + bm::BindingResource::AccelerationStructureArray(Borrowed(slice)) + } + BindingResource::ExternalTexture(external_texture) => { + bm::BindingResource::ExternalTexture(external_texture.inner.as_core().id) + } + }, + }) + .collect::>(); + let descriptor = bm::BindGroupDescriptor { + label: desc.label.as_ref().map(|label| Borrowed(&label[..])), + layout: desc.layout.inner.as_core().id, + entries: Borrowed(&entries), + }; + + let (id, error) = self + .context + .0 + .device_create_bind_group(self.id, &descriptor, None); + if let Some(cause) = error { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_bind_group", + ); + } + CoreBindGroup { + context: self.context.clone(), + id, + } + .into() + } + + fn create_pipeline_layout( + &self, + desc: &crate::PipelineLayoutDescriptor<'_>, + ) -> dispatch::DispatchPipelineLayout { + // Limit is always less or equal to hal::MAX_BIND_GROUPS, so this is always right + // Guards following ArrayVec + assert!( + desc.bind_group_layouts.len() <= wgc::MAX_BIND_GROUPS, + "Bind group layout count {} exceeds device bind group limit {}", + desc.bind_group_layouts.len(), + wgc::MAX_BIND_GROUPS + ); + + let temp_layouts = desc + .bind_group_layouts + .iter() + .map(|bgl| bgl.map(|bgl| bgl.inner.as_core().id)) + .collect::>(); + let descriptor = wgc::binding_model::PipelineLayoutDescriptor { + label: desc.label.map(Borrowed), + bind_group_layouts: Borrowed(&temp_layouts), + immediate_size: desc.immediate_size, + }; + + let (id, error) = self + .context + .0 + .device_create_pipeline_layout(self.id, &descriptor, None); + if let Some(cause) = error { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_pipeline_layout", + ); + } + CorePipelineLayout { + context: self.context.clone(), + id, + } + .into() + } + + fn create_render_pipeline( + &self, + desc: &crate::RenderPipelineDescriptor<'_>, + ) -> dispatch::DispatchRenderPipeline { + use wgc::pipeline as pipe; + + let vertex_buffers: ArrayVec<_, { wgc::MAX_VERTEX_BUFFERS }> = desc + .vertex + .buffers + .iter() + .map(|vbuf| pipe::VertexBufferLayout { + array_stride: vbuf.array_stride, + step_mode: vbuf.step_mode, + attributes: Borrowed(vbuf.attributes), + }) + .collect(); + + let vert_constants = desc + .vertex + .compilation_options + .constants + .iter() + .map(|&(key, value)| (String::from(key), value)) + .collect(); + + let descriptor = pipe::RenderPipelineDescriptor { + label: desc.label.map(Borrowed), + layout: desc.layout.map(|layout| layout.inner.as_core().id), + vertex: pipe::VertexState { + stage: pipe::ProgrammableStageDescriptor { + module: desc.vertex.module.inner.as_core().id, + entry_point: desc.vertex.entry_point.map(Borrowed), + constants: vert_constants, + zero_initialize_workgroup_memory: desc + .vertex + .compilation_options + .zero_initialize_workgroup_memory, + }, + buffers: Borrowed(&vertex_buffers), + }, + primitive: desc.primitive, + depth_stencil: desc.depth_stencil.clone(), + multisample: desc.multisample, + fragment: desc.fragment.as_ref().map(|frag| { + let frag_constants = frag + .compilation_options + .constants + .iter() + .map(|&(key, value)| (String::from(key), value)) + .collect(); + pipe::FragmentState { + stage: pipe::ProgrammableStageDescriptor { + module: frag.module.inner.as_core().id, + entry_point: frag.entry_point.map(Borrowed), + constants: frag_constants, + zero_initialize_workgroup_memory: frag + .compilation_options + .zero_initialize_workgroup_memory, + }, + targets: Borrowed(frag.targets), + } + }), + multiview_mask: desc.multiview_mask, + cache: desc.cache.map(|cache| cache.inner.as_core().id), + }; + + let (id, error) = self + .context + .0 + .device_create_render_pipeline(self.id, &descriptor, None); + if let Some(cause) = error { + if let wgc::pipeline::CreateRenderPipelineError::Internal { stage, ref error } = cause { + log::error!("Shader translation error for stage {stage:?}: {error}"); + log::error!("Please report it to https://github.com/gfx-rs/wgpu"); + } + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_render_pipeline", + ); + } + CoreRenderPipeline { + context: self.context.clone(), + id, + error_sink: Arc::clone(&self.error_sink), + } + .into() + } + + fn create_mesh_pipeline( + &self, + desc: &crate::MeshPipelineDescriptor<'_>, + ) -> dispatch::DispatchRenderPipeline { + use wgc::pipeline as pipe; + + let mesh_constants = desc + .mesh + .compilation_options + .constants + .iter() + .map(|&(key, value)| (String::from(key), value)) + .collect(); + let descriptor = pipe::MeshPipelineDescriptor { + label: desc.label.map(Borrowed), + task: desc.task.as_ref().map(|task| { + let task_constants = task + .compilation_options + .constants + .iter() + .map(|&(key, value)| (String::from(key), value)) + .collect(); + pipe::TaskState { + stage: pipe::ProgrammableStageDescriptor { + module: task.module.inner.as_core().id, + entry_point: task.entry_point.map(Borrowed), + constants: task_constants, + zero_initialize_workgroup_memory: desc + .mesh + .compilation_options + .zero_initialize_workgroup_memory, + }, + } + }), + mesh: pipe::MeshState { + stage: pipe::ProgrammableStageDescriptor { + module: desc.mesh.module.inner.as_core().id, + entry_point: desc.mesh.entry_point.map(Borrowed), + constants: mesh_constants, + zero_initialize_workgroup_memory: desc + .mesh + .compilation_options + .zero_initialize_workgroup_memory, + }, + }, + layout: desc.layout.map(|layout| layout.inner.as_core().id), + primitive: desc.primitive, + depth_stencil: desc.depth_stencil.clone(), + multisample: desc.multisample, + fragment: desc.fragment.as_ref().map(|frag| { + let frag_constants = frag + .compilation_options + .constants + .iter() + .map(|&(key, value)| (String::from(key), value)) + .collect(); + pipe::FragmentState { + stage: pipe::ProgrammableStageDescriptor { + module: frag.module.inner.as_core().id, + entry_point: frag.entry_point.map(Borrowed), + constants: frag_constants, + zero_initialize_workgroup_memory: frag + .compilation_options + .zero_initialize_workgroup_memory, + }, + targets: Borrowed(frag.targets), + } + }), + multiview: desc.multiview, + cache: desc.cache.map(|cache| cache.inner.as_core().id), + }; + + let (id, error) = self + .context + .0 + .device_create_mesh_pipeline(self.id, &descriptor, None); + if let Some(cause) = error { + if let wgc::pipeline::CreateRenderPipelineError::Internal { stage, ref error } = cause { + log::error!("Shader translation error for stage {stage:?}: {error}"); + log::error!("Please report it to https://github.com/gfx-rs/wgpu"); + } + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_render_pipeline", + ); + } + CoreRenderPipeline { + context: self.context.clone(), + id, + error_sink: Arc::clone(&self.error_sink), + } + .into() + } + + fn create_compute_pipeline( + &self, + desc: &crate::ComputePipelineDescriptor<'_>, + ) -> dispatch::DispatchComputePipeline { + use wgc::pipeline as pipe; + + let constants = desc + .compilation_options + .constants + .iter() + .map(|&(key, value)| (String::from(key), value)) + .collect(); + + let descriptor = pipe::ComputePipelineDescriptor { + label: desc.label.map(Borrowed), + layout: desc.layout.map(|pll| pll.inner.as_core().id), + stage: pipe::ProgrammableStageDescriptor { + module: desc.module.inner.as_core().id, + entry_point: desc.entry_point.map(Borrowed), + constants, + zero_initialize_workgroup_memory: desc + .compilation_options + .zero_initialize_workgroup_memory, + }, + cache: desc.cache.map(|cache| cache.inner.as_core().id), + }; + + let (id, error) = self + .context + .0 + .device_create_compute_pipeline(self.id, &descriptor, None); + if let Some(cause) = error { + if let wgc::pipeline::CreateComputePipelineError::Internal(ref error) = cause { + log::error!( + "Shader translation error for stage {:?}: {}", + wgt::ShaderStages::COMPUTE, + error + ); + log::error!("Please report it to https://github.com/gfx-rs/wgpu"); + } + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_compute_pipeline", + ); + } + CoreComputePipeline { + context: self.context.clone(), + id, + error_sink: Arc::clone(&self.error_sink), + } + .into() + } + + unsafe fn create_pipeline_cache( + &self, + desc: &crate::PipelineCacheDescriptor<'_>, + ) -> dispatch::DispatchPipelineCache { + use wgc::pipeline as pipe; + + let descriptor = pipe::PipelineCacheDescriptor { + label: desc.label.map(Borrowed), + data: desc.data.map(Borrowed), + fallback: desc.fallback, + }; + let (id, error) = unsafe { + self.context + .0 + .device_create_pipeline_cache(self.id, &descriptor, None) + }; + if let Some(cause) = error { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::device_create_pipeline_cache_init", + ); + } + CorePipelineCache { + context: self.context.clone(), + id, + } + .into() + } + + fn create_buffer(&self, desc: &crate::BufferDescriptor<'_>) -> dispatch::DispatchBuffer { + let (id, error) = self.context.0.device_create_buffer( + self.id, + &desc.map_label(|l| l.map(Borrowed)), + None, + ); + if let Some(cause) = error { + self.context + .handle_error(&self.error_sink, cause, desc.label, "Device::create_buffer"); + } + + CoreBuffer { + context: self.context.clone(), + id, + error_sink: Arc::clone(&self.error_sink), + } + .into() + } + + fn create_texture(&self, desc: &crate::TextureDescriptor<'_>) -> dispatch::DispatchTexture { + let wgt_desc = desc.map_label_and_view_formats(|l| l.map(Borrowed), |v| v.to_vec()); + let (id, error) = self + .context + .0 + .device_create_texture(self.id, &wgt_desc, None); + if let Some(cause) = error { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_texture", + ); + } + + CoreTexture { + context: self.context.clone(), + id, + error_sink: Arc::clone(&self.error_sink), + } + .into() + } + + fn create_external_texture( + &self, + desc: &crate::ExternalTextureDescriptor<'_>, + planes: &[&crate::TextureView], + ) -> dispatch::DispatchExternalTexture { + let wgt_desc = desc.map_label(|l| l.map(Borrowed)); + let planes = planes + .iter() + .map(|plane| plane.inner.as_core().id) + .collect::>(); + let (id, error) = self + .context + .0 + .device_create_external_texture(self.id, &wgt_desc, &planes, None); + if let Some(cause) = error { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_external_texture", + ); + } + + CoreExternalTexture { + context: self.context.clone(), + id, + } + .into() + } + + fn create_blas( + &self, + desc: &crate::CreateBlasDescriptor<'_>, + sizes: crate::BlasGeometrySizeDescriptors, + ) -> (Option, dispatch::DispatchBlas) { + let global = &self.context.0; + let (id, handle, error) = + global.device_create_blas(self.id, &desc.map_label(|l| l.map(Borrowed)), sizes, None); + if let Some(cause) = error { + self.context + .handle_error(&self.error_sink, cause, desc.label, "Device::create_blas"); + } + ( + handle, + CoreBlas { + context: self.context.clone(), + id, + error_sink: Arc::clone(&self.error_sink), + } + .into(), + ) + } + + fn create_tlas(&self, desc: &crate::CreateTlasDescriptor<'_>) -> dispatch::DispatchTlas { + let global = &self.context.0; + let (id, error) = + global.device_create_tlas(self.id, &desc.map_label(|l| l.map(Borrowed)), None); + if let Some(cause) = error { + self.context + .handle_error(&self.error_sink, cause, desc.label, "Device::create_tlas"); + } + CoreTlas { + context: self.context.clone(), + id, + // error_sink: Arc::clone(&self.error_sink), + } + .into() + } + + fn create_sampler(&self, desc: &crate::SamplerDescriptor<'_>) -> dispatch::DispatchSampler { + let descriptor = wgc::resource::SamplerDescriptor { + label: desc.label.map(Borrowed), + address_modes: [ + desc.address_mode_u, + desc.address_mode_v, + desc.address_mode_w, + ], + mag_filter: desc.mag_filter, + min_filter: desc.min_filter, + mipmap_filter: desc.mipmap_filter, + lod_min_clamp: desc.lod_min_clamp, + lod_max_clamp: desc.lod_max_clamp, + compare: desc.compare, + anisotropy_clamp: desc.anisotropy_clamp, + border_color: desc.border_color, + }; + + let (id, error) = self + .context + .0 + .device_create_sampler(self.id, &descriptor, None); + if let Some(cause) = error { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_sampler", + ); + } + CoreSampler { + context: self.context.clone(), + id, + } + .into() + } + + fn create_query_set(&self, desc: &crate::QuerySetDescriptor<'_>) -> dispatch::DispatchQuerySet { + let (id, error) = self.context.0.device_create_query_set( + self.id, + &desc.map_label(|l| l.map(Borrowed)), + None, + ); + if let Some(cause) = error { + self.context + .handle_error_nolabel(&self.error_sink, cause, "Device::create_query_set"); + } + CoreQuerySet { + context: self.context.clone(), + id, + } + .into() + } + + fn create_command_encoder( + &self, + desc: &crate::CommandEncoderDescriptor<'_>, + ) -> dispatch::DispatchCommandEncoder { + let (id, error) = self.context.0.device_create_command_encoder( + self.id, + &desc.map_label(|l| l.map(Borrowed)), + None, + ); + if let Some(cause) = error { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "Device::create_command_encoder", + ); + } + + CoreCommandEncoder { + context: self.context.clone(), + id, + error_sink: Arc::clone(&self.error_sink), + } + .into() + } + + fn create_render_bundle_encoder( + &self, + desc: &crate::RenderBundleEncoderDescriptor<'_>, + ) -> dispatch::DispatchRenderBundleEncoder { + let descriptor = wgc::command::RenderBundleEncoderDescriptor { + label: desc.label.map(Borrowed), + color_formats: Borrowed(desc.color_formats), + depth_stencil: desc.depth_stencil, + sample_count: desc.sample_count, + multiview: desc.multiview, + }; + let encoder = match wgc::command::RenderBundleEncoder::new(&descriptor, self.id) { + Ok(encoder) => encoder, + Err(e) => panic!("Error in Device::create_render_bundle_encoder: {e}"), + }; + + CoreRenderBundleEncoder { + context: self.context.clone(), + encoder, + id: crate::cmp::Identifier::create(), + } + .into() + } + + fn set_device_lost_callback(&self, device_lost_callback: dispatch::BoxDeviceLostCallback) { + self.context + .0 + .device_set_device_lost_closure(self.id, device_lost_callback); + } + + fn on_uncaptured_error(&self, handler: Arc) { + let mut error_sink = self.error_sink.lock(); + error_sink.uncaptured_handler = Some(handler); + } + + fn push_error_scope(&self, filter: crate::ErrorFilter) -> u32 { + let mut error_sink = self.error_sink.lock(); + let thread_id = thread_id::ThreadId::current(); + let scopes = error_sink.scopes.entry(thread_id).or_default(); + let index = scopes + .len() + .try_into() + .expect("Greater than 2^32 nested error scopes"); + scopes.push(ErrorScope { + error: None, + filter, + }); + index + } + + fn pop_error_scope(&self, index: u32) -> Pin> { + let mut error_sink = self.error_sink.lock(); + + // We go out of our way to avoid panicking while unwinding, because that would abort the process, + // and we are supposed to just drop the error scope on the floor. + let is_panicking = crate::util::is_panicking(); + let thread_id = thread_id::ThreadId::current(); + let err = "Mismatched pop_error_scope call: no error scope for this thread. Error scopes are thread-local."; + let scopes = match error_sink.scopes.get_mut(&thread_id) { + Some(s) => s, + None => { + if !is_panicking { + panic!("{err}"); + } else { + return Box::pin(ready(None)); + } + } + }; + if scopes.is_empty() && !is_panicking { + panic!("{err}"); + } + if index as usize != scopes.len() - 1 && !is_panicking { + panic!( + "Mismatched pop_error_scope call: error scopes must be popped in reverse order." + ); + } + + // It would be more correct in this case to use `remove` here so that when unwinding is occurring + // we would remove the correct error scope, but we don't have such a primitive on the web + // and having consistent behavior here is more important. If you are unwinding and it unwinds + // the guards in the wrong order, it's totally reasonable to have incorrect behavior. + let scope = match scopes.pop() { + Some(s) => s, + None if !is_panicking => unreachable!(), + None => return Box::pin(ready(None)), + }; + + Box::pin(ready(scope.error)) + } + + unsafe fn start_graphics_debugger_capture(&self) { + unsafe { + self.context + .0 + .device_start_graphics_debugger_capture(self.id) + }; + } + + unsafe fn stop_graphics_debugger_capture(&self) { + unsafe { + self.context + .0 + .device_stop_graphics_debugger_capture(self.id) + }; + } + + fn poll(&self, poll_type: wgt::PollType) -> Result { + match self.context.0.device_poll(self.id, poll_type) { + Ok(status) => Ok(status), + Err(err) => { + if let Some(poll_error) = err.to_poll_error() { + return Err(poll_error); + } + + self.context.handle_error_fatal(err, "Device::poll") + } + } + } + + fn get_internal_counters(&self) -> crate::InternalCounters { + self.context.0.device_get_internal_counters(self.id) + } + + fn generate_allocator_report(&self) -> Option { + self.context.0.device_generate_allocator_report(self.id) + } + + fn destroy(&self) { + self.context.0.device_destroy(self.id); + } +} + +impl Drop for CoreDevice { + fn drop(&mut self) { + self.context.0.device_drop(self.id) + } +} + +impl dispatch::QueueInterface for CoreQueue { + fn write_buffer( + &self, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + data: &[u8], + ) { + let buffer = buffer.as_core(); + + match self + .context + .0 + .queue_write_buffer(self.id, buffer.id, offset, data) + { + Ok(()) => (), + Err(err) => { + self.context + .handle_error_nolabel(&self.error_sink, err, "Queue::write_buffer") + } + } + } + + fn create_staging_buffer( + &self, + size: crate::BufferSize, + ) -> Option { + match self + .context + .0 + .queue_create_staging_buffer(self.id, size, None) + { + Ok((buffer_id, ptr)) => Some( + CoreQueueWriteBuffer { + buffer_id, + mapping: CoreBufferMappedRange { + ptr, + size: size.get() as usize, + }, + } + .into(), + ), + Err(err) => { + self.context.handle_error_nolabel( + &self.error_sink, + err, + "Queue::write_buffer_with", + ); + None + } + } + } + + fn validate_write_buffer( + &self, + buffer: &dispatch::DispatchBuffer, + offset: wgt::BufferAddress, + size: wgt::BufferSize, + ) -> Option<()> { + let buffer = buffer.as_core(); + + match self + .context + .0 + .queue_validate_write_buffer(self.id, buffer.id, offset, size) + { + Ok(()) => Some(()), + Err(err) => { + self.context.handle_error_nolabel( + &self.error_sink, + err, + "Queue::write_buffer_with", + ); + None + } + } + } + + fn write_staging_buffer( + &self, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + staging_buffer: &dispatch::DispatchQueueWriteBuffer, + ) { + let buffer = buffer.as_core(); + let staging_buffer = staging_buffer.as_core(); + + match self.context.0.queue_write_staging_buffer( + self.id, + buffer.id, + offset, + staging_buffer.buffer_id, + ) { + Ok(()) => (), + Err(err) => { + self.context.handle_error_nolabel( + &self.error_sink, + err, + "Queue::write_buffer_with", + ); + } + } + } + + fn write_texture( + &self, + texture: crate::TexelCopyTextureInfo<'_>, + data: &[u8], + data_layout: crate::TexelCopyBufferLayout, + size: crate::Extent3d, + ) { + match self.context.0.queue_write_texture( + self.id, + &map_texture_copy_view(texture), + data, + &data_layout, + &size, + ) { + Ok(()) => (), + Err(err) => { + self.context + .handle_error_nolabel(&self.error_sink, err, "Queue::write_texture") + } + } + } + + // This method needs to exist if either webgpu or webgl is enabled, + // but we only actually have an implementation if webgl is enabled. + #[cfg(web)] + #[cfg_attr(not(webgl), expect(unused_variables))] + fn copy_external_image_to_texture( + &self, + source: &crate::CopyExternalImageSourceInfo, + dest: crate::CopyExternalImageDestInfo<&crate::api::Texture>, + size: crate::Extent3d, + ) { + #[cfg(webgl)] + match self.context.0.queue_copy_external_image_to_texture( + self.id, + source, + map_texture_tagged_copy_view(dest), + size, + ) { + Ok(()) => (), + Err(err) => self.context.handle_error_nolabel( + &self.error_sink, + err, + "Queue::copy_external_image_to_texture", + ), + } + } + + fn submit( + &self, + command_buffers: &mut dyn Iterator, + ) -> u64 { + let temp_command_buffers = command_buffers.collect::>(); + let command_buffer_ids = temp_command_buffers + .iter() + .map(|cmdbuf| cmdbuf.as_core().id) + .collect::>(); + + let index = match self.context.0.queue_submit(self.id, &command_buffer_ids) { + Ok(index) => index, + Err((index, err)) => { + self.context + .handle_error_nolabel(&self.error_sink, err, "Queue::submit"); + index + } + }; + + drop(temp_command_buffers); + + index + } + + fn get_timestamp_period(&self) -> f32 { + self.context.0.queue_get_timestamp_period(self.id) + } + + fn on_submitted_work_done(&self, callback: dispatch::BoxSubmittedWorkDoneCallback) { + self.context + .0 + .queue_on_submitted_work_done(self.id, callback); + } + + fn compact_blas(&self, blas: &dispatch::DispatchBlas) -> (Option, dispatch::DispatchBlas) { + let (id, handle, error) = + self.context + .0 + .queue_compact_blas(self.id, blas.as_core().id, None); + + if let Some(cause) = error { + self.context + .handle_error_nolabel(&self.error_sink, cause, "Queue::compact_blas"); + } + ( + handle, + CoreBlas { + context: self.context.clone(), + id, + error_sink: Arc::clone(&self.error_sink), + } + .into(), + ) + } +} + +impl Drop for CoreQueue { + fn drop(&mut self) { + self.context.0.queue_drop(self.id) + } +} + +impl dispatch::ShaderModuleInterface for CoreShaderModule { + fn get_compilation_info(&self) -> Pin> { + Box::pin(ready(self.compilation_info.clone())) + } +} + +impl Drop for CoreShaderModule { + fn drop(&mut self) { + self.context.0.shader_module_drop(self.id) + } +} + +impl dispatch::BindGroupLayoutInterface for CoreBindGroupLayout {} + +impl Drop for CoreBindGroupLayout { + fn drop(&mut self) { + self.context.0.bind_group_layout_drop(self.id) + } +} + +impl dispatch::BindGroupInterface for CoreBindGroup {} + +impl Drop for CoreBindGroup { + fn drop(&mut self) { + self.context.0.bind_group_drop(self.id) + } +} + +impl dispatch::TextureViewInterface for CoreTextureView {} + +impl Drop for CoreTextureView { + fn drop(&mut self) { + self.context.0.texture_view_drop(self.id); + } +} + +impl dispatch::ExternalTextureInterface for CoreExternalTexture { + fn destroy(&self) { + self.context.0.external_texture_destroy(self.id); + } +} + +impl Drop for CoreExternalTexture { + fn drop(&mut self) { + self.context.0.external_texture_drop(self.id); + } +} + +impl dispatch::SamplerInterface for CoreSampler {} + +impl Drop for CoreSampler { + fn drop(&mut self) { + self.context.0.sampler_drop(self.id) + } +} + +impl dispatch::BufferInterface for CoreBuffer { + fn map_async( + &self, + mode: crate::MapMode, + range: Range, + callback: dispatch::BufferMapCallback, + ) { + let operation = wgc::resource::BufferMapOperation { + host: match mode { + MapMode::Read => wgc::device::HostMap::Read, + MapMode::Write => wgc::device::HostMap::Write, + }, + callback: Some(Box::new(|status| { + let res = status.map_err(|_| crate::BufferAsyncError); + callback(res); + })), + }; + + match self.context.0.buffer_map_async( + self.id, + range.start, + Some(range.end - range.start), + operation, + ) { + Ok(_) => (), + Err(cause) => { + self.context + .handle_error_nolabel(&self.error_sink, cause, "Buffer::map_async") + } + } + } + + fn get_mapped_range( + &self, + sub_range: Range, + ) -> dispatch::DispatchBufferMappedRange { + let size = sub_range.end - sub_range.start; + match self + .context + .0 + .buffer_get_mapped_range(self.id, sub_range.start, Some(size)) + { + Ok((ptr, size)) => CoreBufferMappedRange { + ptr, + size: size as usize, + } + .into(), + Err(err) => self + .context + .handle_error_fatal(err, "Buffer::get_mapped_range"), + } + } + + fn unmap(&self) { + match self.context.0.buffer_unmap(self.id) { + Ok(()) => (), + Err(cause) => { + self.context + .handle_error_nolabel(&self.error_sink, cause, "Buffer::buffer_unmap") + } + } + } + + fn destroy(&self) { + self.context.0.buffer_destroy(self.id); + } +} + +impl Drop for CoreBuffer { + fn drop(&mut self) { + self.context.0.buffer_drop(self.id) + } +} + +impl dispatch::TextureInterface for CoreTexture { + fn create_view( + &self, + desc: &crate::TextureViewDescriptor<'_>, + ) -> dispatch::DispatchTextureView { + let descriptor = wgc::resource::TextureViewDescriptor { + label: desc.label.map(Borrowed), + format: desc.format, + dimension: desc.dimension, + usage: desc.usage, + range: wgt::ImageSubresourceRange { + aspect: desc.aspect, + base_mip_level: desc.base_mip_level, + mip_level_count: desc.mip_level_count, + base_array_layer: desc.base_array_layer, + array_layer_count: desc.array_layer_count, + }, + }; + let (id, error) = self + .context + .0 + .texture_create_view(self.id, &descriptor, None); + if let Some(cause) = error { + self.context + .handle_error(&self.error_sink, cause, desc.label, "Texture::create_view"); + } + CoreTextureView { + context: self.context.clone(), + id, + } + .into() + } + + fn destroy(&self) { + self.context.0.texture_destroy(self.id); + } +} + +impl Drop for CoreTexture { + fn drop(&mut self) { + self.context.0.texture_drop(self.id) + } +} + +impl dispatch::BlasInterface for CoreBlas { + fn prepare_compact_async(&self, callback: BlasCompactCallback) { + let callback: Option = + Some(Box::new(|status: BlasPrepareCompactResult| { + let res = status.map_err(|_| crate::BlasAsyncError); + callback(res); + })); + + match self.context.0.blas_prepare_compact_async(self.id, callback) { + Ok(_) => (), + Err(cause) => self.context.handle_error_nolabel( + &self.error_sink, + cause, + "Blas::prepare_compact_async", + ), + } + } + + fn ready_for_compaction(&self) -> bool { + match self.context.0.ready_for_compaction(self.id) { + Ok(ready) => ready, + Err(cause) => { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "Blas::ready_for_compaction", + ); + // A BLAS is definitely not ready for compaction if it's not valid + false + } + } + } +} + +impl Drop for CoreBlas { + fn drop(&mut self) { + self.context.0.blas_drop(self.id) + } +} + +impl dispatch::TlasInterface for CoreTlas {} + +impl Drop for CoreTlas { + fn drop(&mut self) { + self.context.0.tlas_drop(self.id) + } +} + +impl dispatch::QuerySetInterface for CoreQuerySet {} + +impl Drop for CoreQuerySet { + fn drop(&mut self) { + self.context.0.query_set_drop(self.id) + } +} + +impl dispatch::PipelineLayoutInterface for CorePipelineLayout {} + +impl Drop for CorePipelineLayout { + fn drop(&mut self) { + self.context.0.pipeline_layout_drop(self.id) + } +} + +impl dispatch::RenderPipelineInterface for CoreRenderPipeline { + fn get_bind_group_layout(&self, index: u32) -> dispatch::DispatchBindGroupLayout { + let (id, error) = self + .context + .0 + .render_pipeline_get_bind_group_layout(self.id, index, None); + if let Some(err) = error { + self.context.handle_error_nolabel( + &self.error_sink, + err, + "RenderPipeline::get_bind_group_layout", + ) + } + CoreBindGroupLayout { + context: self.context.clone(), + id, + } + .into() + } +} + +impl Drop for CoreRenderPipeline { + fn drop(&mut self) { + self.context.0.render_pipeline_drop(self.id) + } +} + +impl dispatch::ComputePipelineInterface for CoreComputePipeline { + fn get_bind_group_layout(&self, index: u32) -> dispatch::DispatchBindGroupLayout { + let (id, error) = self + .context + .0 + .compute_pipeline_get_bind_group_layout(self.id, index, None); + if let Some(err) = error { + self.context.handle_error_nolabel( + &self.error_sink, + err, + "ComputePipeline::get_bind_group_layout", + ) + } + CoreBindGroupLayout { + context: self.context.clone(), + id, + } + .into() + } +} + +impl Drop for CoreComputePipeline { + fn drop(&mut self) { + self.context.0.compute_pipeline_drop(self.id) + } +} + +impl dispatch::PipelineCacheInterface for CorePipelineCache { + fn get_data(&self) -> Option> { + self.context.0.pipeline_cache_get_data(self.id) + } +} + +impl Drop for CorePipelineCache { + fn drop(&mut self) { + self.context.0.pipeline_cache_drop(self.id) + } +} + +impl dispatch::CommandEncoderInterface for CoreCommandEncoder { + fn copy_buffer_to_buffer( + &self, + source: &dispatch::DispatchBuffer, + source_offset: crate::BufferAddress, + destination: &dispatch::DispatchBuffer, + destination_offset: crate::BufferAddress, + copy_size: Option, + ) { + let source = source.as_core(); + let destination = destination.as_core(); + + if let Err(cause) = self.context.0.command_encoder_copy_buffer_to_buffer( + self.id, + source.id, + source_offset, + destination.id, + destination_offset, + copy_size, + ) { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::copy_buffer_to_buffer", + ); + } + } + + fn copy_buffer_to_texture( + &self, + source: crate::TexelCopyBufferInfo<'_>, + destination: crate::TexelCopyTextureInfo<'_>, + copy_size: crate::Extent3d, + ) { + if let Err(cause) = self.context.0.command_encoder_copy_buffer_to_texture( + self.id, + &map_buffer_copy_view(source), + &map_texture_copy_view(destination), + ©_size, + ) { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::copy_buffer_to_texture", + ); + } + } + + fn copy_texture_to_buffer( + &self, + source: crate::TexelCopyTextureInfo<'_>, + destination: crate::TexelCopyBufferInfo<'_>, + copy_size: crate::Extent3d, + ) { + if let Err(cause) = self.context.0.command_encoder_copy_texture_to_buffer( + self.id, + &map_texture_copy_view(source), + &map_buffer_copy_view(destination), + ©_size, + ) { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::copy_texture_to_buffer", + ); + } + } + + fn copy_texture_to_texture( + &self, + source: crate::TexelCopyTextureInfo<'_>, + destination: crate::TexelCopyTextureInfo<'_>, + copy_size: crate::Extent3d, + ) { + if let Err(cause) = self.context.0.command_encoder_copy_texture_to_texture( + self.id, + &map_texture_copy_view(source), + &map_texture_copy_view(destination), + ©_size, + ) { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::copy_texture_to_texture", + ); + } + } + + fn begin_compute_pass( + &self, + desc: &crate::ComputePassDescriptor<'_>, + ) -> dispatch::DispatchComputePass { + let timestamp_writes = + desc.timestamp_writes + .as_ref() + .map(|tw| wgc::command::PassTimestampWrites { + query_set: tw.query_set.inner.as_core().id, + beginning_of_pass_write_index: tw.beginning_of_pass_write_index, + end_of_pass_write_index: tw.end_of_pass_write_index, + }); + + let (pass, err) = self.context.0.command_encoder_begin_compute_pass( + self.id, + &wgc::command::ComputePassDescriptor { + label: desc.label.map(Borrowed), + timestamp_writes, + }, + ); + + if let Some(cause) = err { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "CommandEncoder::begin_compute_pass", + ); + } + + CoreComputePass { + context: self.context.clone(), + pass, + error_sink: self.error_sink.clone(), + id: crate::cmp::Identifier::create(), + } + .into() + } + + fn begin_render_pass( + &self, + desc: &crate::RenderPassDescriptor<'_>, + ) -> dispatch::DispatchRenderPass { + let colors = desc + .color_attachments + .iter() + .map(|ca| { + ca.as_ref() + .map(|at| wgc::command::RenderPassColorAttachment { + view: at.view.inner.as_core().id, + depth_slice: at.depth_slice, + resolve_target: at.resolve_target.map(|view| view.inner.as_core().id), + load_op: at.ops.load, + store_op: at.ops.store, + }) + }) + .collect::>(); + + let depth_stencil = desc.depth_stencil_attachment.as_ref().map(|dsa| { + wgc::command::RenderPassDepthStencilAttachment { + view: dsa.view.inner.as_core().id, + depth: map_pass_channel(dsa.depth_ops.as_ref()), + stencil: map_pass_channel(dsa.stencil_ops.as_ref()), + } + }); + + let timestamp_writes = + desc.timestamp_writes + .as_ref() + .map(|tw| wgc::command::PassTimestampWrites { + query_set: tw.query_set.inner.as_core().id, + beginning_of_pass_write_index: tw.beginning_of_pass_write_index, + end_of_pass_write_index: tw.end_of_pass_write_index, + }); + + let (pass, err) = self.context.0.command_encoder_begin_render_pass( + self.id, + &wgc::command::RenderPassDescriptor { + label: desc.label.map(Borrowed), + timestamp_writes: timestamp_writes.as_ref(), + color_attachments: Borrowed(&colors), + depth_stencil_attachment: depth_stencil.as_ref(), + occlusion_query_set: desc.occlusion_query_set.map(|qs| qs.inner.as_core().id), + multiview_mask: desc.multiview_mask, + }, + ); + + if let Some(cause) = err { + self.context.handle_error( + &self.error_sink, + cause, + desc.label, + "CommandEncoder::begin_render_pass", + ); + } + + CoreRenderPass { + context: self.context.clone(), + pass, + error_sink: self.error_sink.clone(), + id: crate::cmp::Identifier::create(), + } + .into() + } + + fn finish(&mut self) -> dispatch::DispatchCommandBuffer { + let descriptor = wgt::CommandBufferDescriptor::default(); + let (id, opt_label_and_error) = + self.context + .0 + .command_encoder_finish(self.id, &descriptor, None); + if let Some((label, cause)) = opt_label_and_error { + self.context + .handle_error(&self.error_sink, cause, Some(&label), "a CommandEncoder"); + } + CoreCommandBuffer { + context: self.context.clone(), + id, + } + .into() + } + + fn clear_texture( + &self, + texture: &dispatch::DispatchTexture, + subresource_range: &crate::ImageSubresourceRange, + ) { + let texture = texture.as_core(); + + if let Err(cause) = + self.context + .0 + .command_encoder_clear_texture(self.id, texture.id, subresource_range) + { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::clear_texture", + ); + } + } + + fn clear_buffer( + &self, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_core(); + + if let Err(cause) = self + .context + .0 + .command_encoder_clear_buffer(self.id, buffer.id, offset, size) + { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::fill_buffer", + ); + } + } + + fn insert_debug_marker(&self, label: &str) { + if let Err(cause) = self + .context + .0 + .command_encoder_insert_debug_marker(self.id, label) + { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::insert_debug_marker", + ); + } + } + + fn push_debug_group(&self, label: &str) { + if let Err(cause) = self + .context + .0 + .command_encoder_push_debug_group(self.id, label) + { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::push_debug_group", + ); + } + } + + fn pop_debug_group(&self) { + if let Err(cause) = self.context.0.command_encoder_pop_debug_group(self.id) { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::pop_debug_group", + ); + } + } + + fn write_timestamp(&self, query_set: &dispatch::DispatchQuerySet, query_index: u32) { + let query_set = query_set.as_core(); + + if let Err(cause) = + self.context + .0 + .command_encoder_write_timestamp(self.id, query_set.id, query_index) + { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::write_timestamp", + ); + } + } + + fn resolve_query_set( + &self, + query_set: &dispatch::DispatchQuerySet, + first_query: u32, + query_count: u32, + destination: &dispatch::DispatchBuffer, + destination_offset: crate::BufferAddress, + ) { + let query_set = query_set.as_core(); + let destination = destination.as_core(); + + if let Err(cause) = self.context.0.command_encoder_resolve_query_set( + self.id, + query_set.id, + first_query, + query_count, + destination.id, + destination_offset, + ) { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::resolve_query_set", + ); + } + } + + fn mark_acceleration_structures_built<'a>( + &self, + blas: &mut dyn Iterator, + tlas: &mut dyn Iterator, + ) { + let blas = blas + .map(|b| b.inner.as_core().id) + .collect::>(); + let tlas = tlas + .map(|t| t.inner.as_core().id) + .collect::>(); + if let Err(cause) = self + .context + .0 + .command_encoder_mark_acceleration_structures_built(self.id, &blas, &tlas) + { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::build_acceleration_structures_unsafe_tlas", + ); + } + } + + fn build_acceleration_structures<'a>( + &self, + blas: &mut dyn Iterator>, + tlas: &mut dyn Iterator, + ) { + let blas = blas.map(|e: &crate::BlasBuildEntry<'_>| { + let geometries = match e.geometry { + crate::BlasGeometries::TriangleGeometries(ref triangle_geometries) => { + let iter = triangle_geometries.iter().map(|tg| { + wgc::ray_tracing::BlasTriangleGeometry { + vertex_buffer: tg.vertex_buffer.inner.as_core().id, + index_buffer: tg.index_buffer.map(|buf| buf.inner.as_core().id), + transform_buffer: tg.transform_buffer.map(|buf| buf.inner.as_core().id), + size: tg.size, + transform_buffer_offset: tg.transform_buffer_offset, + first_vertex: tg.first_vertex, + vertex_stride: tg.vertex_stride, + first_index: tg.first_index, + } + }); + wgc::ray_tracing::BlasGeometries::TriangleGeometries(Box::new(iter)) + } + }; + wgc::ray_tracing::BlasBuildEntry { + blas_id: e.blas.inner.as_core().id, + geometries, + } + }); + + let tlas = tlas.into_iter().map(|e| { + let instances = e + .instances + .iter() + .map(|instance: &Option| { + instance + .as_ref() + .map(|instance| wgc::ray_tracing::TlasInstance { + blas_id: instance.blas.as_core().id, + transform: &instance.transform, + custom_data: instance.custom_data, + mask: instance.mask, + }) + }); + wgc::ray_tracing::TlasPackage { + tlas_id: e.inner.as_core().id, + instances: Box::new(instances), + lowest_unmodified: e.lowest_unmodified, + } + }); + + if let Err(cause) = self + .context + .0 + .command_encoder_build_acceleration_structures(self.id, blas, tlas) + { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::build_acceleration_structures_unsafe_tlas", + ); + } + } + + fn transition_resources<'a>( + &mut self, + buffer_transitions: &mut dyn Iterator< + Item = wgt::BufferTransition<&'a dispatch::DispatchBuffer>, + >, + texture_transitions: &mut dyn Iterator< + Item = wgt::TextureTransition<&'a dispatch::DispatchTexture>, + >, + ) { + let result = self.context.0.command_encoder_transition_resources( + self.id, + buffer_transitions.map(|t| wgt::BufferTransition { + buffer: t.buffer.as_core().id, + state: t.state, + }), + texture_transitions.map(|t| wgt::TextureTransition { + texture: t.texture.as_core().id, + selector: t.selector.clone(), + state: t.state, + }), + ); + + if let Err(cause) = result { + self.context.handle_error_nolabel( + &self.error_sink, + cause, + "CommandEncoder::transition_resources", + ); + } + } +} + +impl Drop for CoreCommandEncoder { + fn drop(&mut self) { + self.context.0.command_encoder_drop(self.id) + } +} + +impl dispatch::CommandBufferInterface for CoreCommandBuffer {} + +impl Drop for CoreCommandBuffer { + fn drop(&mut self) { + self.context.0.command_buffer_drop(self.id) + } +} + +impl dispatch::ComputePassInterface for CoreComputePass { + fn set_pipeline(&mut self, pipeline: &dispatch::DispatchComputePipeline) { + let pipeline = pipeline.as_core(); + + if let Err(cause) = self + .context + .0 + .compute_pass_set_pipeline(&mut self.pass, pipeline.id) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::set_pipeline", + ); + } + } + + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&dispatch::DispatchBindGroup>, + offsets: &[crate::DynamicOffset], + ) { + let bg = bind_group.map(|bg| bg.as_core().id); + + if let Err(cause) = + self.context + .0 + .compute_pass_set_bind_group(&mut self.pass, index, bg, offsets) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::set_bind_group", + ); + } + } + + fn set_immediates(&mut self, offset: u32, data: &[u8]) { + if let Err(cause) = self + .context + .0 + .compute_pass_set_immediates(&mut self.pass, offset, data) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::set_immediates", + ); + } + } + + fn insert_debug_marker(&mut self, label: &str) { + if let Err(cause) = + self.context + .0 + .compute_pass_insert_debug_marker(&mut self.pass, label, 0) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::insert_debug_marker", + ); + } + } + + fn push_debug_group(&mut self, group_label: &str) { + if let Err(cause) = + self.context + .0 + .compute_pass_push_debug_group(&mut self.pass, group_label, 0) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::push_debug_group", + ); + } + } + + fn pop_debug_group(&mut self) { + if let Err(cause) = self.context.0.compute_pass_pop_debug_group(&mut self.pass) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::pop_debug_group", + ); + } + } + + fn write_timestamp(&mut self, query_set: &dispatch::DispatchQuerySet, query_index: u32) { + let query_set = query_set.as_core(); + + if let Err(cause) = + self.context + .0 + .compute_pass_write_timestamp(&mut self.pass, query_set.id, query_index) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::write_timestamp", + ); + } + } + + fn begin_pipeline_statistics_query( + &mut self, + query_set: &dispatch::DispatchQuerySet, + query_index: u32, + ) { + let query_set = query_set.as_core(); + + if let Err(cause) = self.context.0.compute_pass_begin_pipeline_statistics_query( + &mut self.pass, + query_set.id, + query_index, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::begin_pipeline_statistics_query", + ); + } + } + + fn end_pipeline_statistics_query(&mut self) { + if let Err(cause) = self + .context + .0 + .compute_pass_end_pipeline_statistics_query(&mut self.pass) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::end_pipeline_statistics_query", + ); + } + } + + fn dispatch_workgroups(&mut self, x: u32, y: u32, z: u32) { + if let Err(cause) = self + .context + .0 + .compute_pass_dispatch_workgroups(&mut self.pass, x, y, z) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::dispatch_workgroups", + ); + } + } + + fn dispatch_workgroups_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let indirect_buffer = indirect_buffer.as_core(); + + if let Err(cause) = self.context.0.compute_pass_dispatch_workgroups_indirect( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::dispatch_workgroups_indirect", + ); + } + } +} + +impl Drop for CoreComputePass { + fn drop(&mut self) { + if let Err(cause) = self.context.0.compute_pass_end(&mut self.pass) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "ComputePass::end", + ); + } + } +} + +impl dispatch::RenderPassInterface for CoreRenderPass { + fn set_pipeline(&mut self, pipeline: &dispatch::DispatchRenderPipeline) { + let pipeline = pipeline.as_core(); + + if let Err(cause) = self + .context + .0 + .render_pass_set_pipeline(&mut self.pass, pipeline.id) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::set_pipeline", + ); + } + } + + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&dispatch::DispatchBindGroup>, + offsets: &[crate::DynamicOffset], + ) { + let bg = bind_group.map(|bg| bg.as_core().id); + + if let Err(cause) = + self.context + .0 + .render_pass_set_bind_group(&mut self.pass, index, bg, offsets) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::set_bind_group", + ); + } + } + + fn set_index_buffer( + &mut self, + buffer: &dispatch::DispatchBuffer, + index_format: crate::IndexFormat, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_core(); + + if let Err(cause) = self.context.0.render_pass_set_index_buffer( + &mut self.pass, + buffer.id, + index_format, + offset, + size, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::set_index_buffer", + ); + } + } + + fn set_vertex_buffer( + &mut self, + slot: u32, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_core(); + + if let Err(cause) = self.context.0.render_pass_set_vertex_buffer( + &mut self.pass, + slot, + buffer.id, + offset, + size, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::set_vertex_buffer", + ); + } + } + + fn set_immediates(&mut self, offset: u32, data: &[u8]) { + if let Err(cause) = self + .context + .0 + .render_pass_set_immediates(&mut self.pass, offset, data) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::set_immediates", + ); + } + } + + fn set_blend_constant(&mut self, color: crate::Color) { + if let Err(cause) = self + .context + .0 + .render_pass_set_blend_constant(&mut self.pass, color) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::set_blend_constant", + ); + } + } + + fn set_scissor_rect(&mut self, x: u32, y: u32, width: u32, height: u32) { + if let Err(cause) = + self.context + .0 + .render_pass_set_scissor_rect(&mut self.pass, x, y, width, height) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::set_scissor_rect", + ); + } + } + + fn set_viewport( + &mut self, + x: f32, + y: f32, + width: f32, + height: f32, + min_depth: f32, + max_depth: f32, + ) { + if let Err(cause) = self.context.0.render_pass_set_viewport( + &mut self.pass, + x, + y, + width, + height, + min_depth, + max_depth, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::set_viewport", + ); + } + } + + fn set_stencil_reference(&mut self, reference: u32) { + if let Err(cause) = self + .context + .0 + .render_pass_set_stencil_reference(&mut self.pass, reference) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::set_stencil_reference", + ); + } + } + + fn draw(&mut self, vertices: Range, instances: Range) { + if let Err(cause) = self.context.0.render_pass_draw( + &mut self.pass, + vertices.end - vertices.start, + instances.end - instances.start, + vertices.start, + instances.start, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::draw", + ); + } + } + + fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range) { + if let Err(cause) = self.context.0.render_pass_draw_indexed( + &mut self.pass, + indices.end - indices.start, + instances.end - instances.start, + indices.start, + base_vertex, + instances.start, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::draw_indexed", + ); + } + } + + fn draw_mesh_tasks(&mut self, group_count_x: u32, group_count_y: u32, group_count_z: u32) { + if let Err(cause) = self.context.0.render_pass_draw_mesh_tasks( + &mut self.pass, + group_count_x, + group_count_y, + group_count_z, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::draw_mesh_tasks", + ); + } + } + + fn draw_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let indirect_buffer = indirect_buffer.as_core(); + + if let Err(cause) = self.context.0.render_pass_draw_indirect( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::draw_indirect", + ); + } + } + + fn draw_indexed_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let indirect_buffer = indirect_buffer.as_core(); + + if let Err(cause) = self.context.0.render_pass_draw_indexed_indirect( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::draw_indexed_indirect", + ); + } + } + + fn draw_mesh_tasks_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let indirect_buffer = indirect_buffer.as_core(); + + if let Err(cause) = self.context.0.render_pass_draw_mesh_tasks_indirect( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::draw_mesh_tasks_indirect", + ); + } + } + + fn multi_draw_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + count: u32, + ) { + let indirect_buffer = indirect_buffer.as_core(); + + if let Err(cause) = self.context.0.render_pass_multi_draw_indirect( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + count, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::multi_draw_indirect", + ); + } + } + + fn multi_draw_indexed_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + count: u32, + ) { + let indirect_buffer = indirect_buffer.as_core(); + + if let Err(cause) = self.context.0.render_pass_multi_draw_indexed_indirect( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + count, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::multi_draw_indexed_indirect", + ); + } + } + + fn multi_draw_mesh_tasks_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + count: u32, + ) { + let indirect_buffer = indirect_buffer.as_core(); + + if let Err(cause) = self.context.0.render_pass_multi_draw_mesh_tasks_indirect( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + count, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::multi_draw_mesh_tasks_indirect", + ); + } + } + + fn multi_draw_indirect_count( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + count_buffer: &dispatch::DispatchBuffer, + count_buffer_offset: crate::BufferAddress, + max_count: u32, + ) { + let indirect_buffer = indirect_buffer.as_core(); + let count_buffer = count_buffer.as_core(); + + if let Err(cause) = self.context.0.render_pass_multi_draw_indirect_count( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + count_buffer.id, + count_buffer_offset, + max_count, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::multi_draw_indirect_count", + ); + } + } + + fn multi_draw_indexed_indirect_count( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + count_buffer: &dispatch::DispatchBuffer, + count_buffer_offset: crate::BufferAddress, + max_count: u32, + ) { + let indirect_buffer = indirect_buffer.as_core(); + let count_buffer = count_buffer.as_core(); + + if let Err(cause) = self + .context + .0 + .render_pass_multi_draw_indexed_indirect_count( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + count_buffer.id, + count_buffer_offset, + max_count, + ) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::multi_draw_indexed_indirect_count", + ); + } + } + + fn multi_draw_mesh_tasks_indirect_count( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + count_buffer: &dispatch::DispatchBuffer, + count_buffer_offset: crate::BufferAddress, + max_count: u32, + ) { + let indirect_buffer = indirect_buffer.as_core(); + let count_buffer = count_buffer.as_core(); + + if let Err(cause) = self + .context + .0 + .render_pass_multi_draw_mesh_tasks_indirect_count( + &mut self.pass, + indirect_buffer.id, + indirect_offset, + count_buffer.id, + count_buffer_offset, + max_count, + ) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::multi_draw_mesh_tasks_indirect_count", + ); + } + } + + fn insert_debug_marker(&mut self, label: &str) { + if let Err(cause) = self + .context + .0 + .render_pass_insert_debug_marker(&mut self.pass, label, 0) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::insert_debug_marker", + ); + } + } + + fn push_debug_group(&mut self, group_label: &str) { + if let Err(cause) = + self.context + .0 + .render_pass_push_debug_group(&mut self.pass, group_label, 0) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::push_debug_group", + ); + } + } + + fn pop_debug_group(&mut self) { + if let Err(cause) = self.context.0.render_pass_pop_debug_group(&mut self.pass) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::pop_debug_group", + ); + } + } + + fn write_timestamp(&mut self, query_set: &dispatch::DispatchQuerySet, query_index: u32) { + let query_set = query_set.as_core(); + + if let Err(cause) = + self.context + .0 + .render_pass_write_timestamp(&mut self.pass, query_set.id, query_index) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::write_timestamp", + ); + } + } + + fn begin_occlusion_query(&mut self, query_index: u32) { + if let Err(cause) = self + .context + .0 + .render_pass_begin_occlusion_query(&mut self.pass, query_index) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::begin_occlusion_query", + ); + } + } + + fn end_occlusion_query(&mut self) { + if let Err(cause) = self + .context + .0 + .render_pass_end_occlusion_query(&mut self.pass) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::end_occlusion_query", + ); + } + } + + fn begin_pipeline_statistics_query( + &mut self, + query_set: &dispatch::DispatchQuerySet, + query_index: u32, + ) { + let query_set = query_set.as_core(); + + if let Err(cause) = self.context.0.render_pass_begin_pipeline_statistics_query( + &mut self.pass, + query_set.id, + query_index, + ) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::begin_pipeline_statistics_query", + ); + } + } + + fn end_pipeline_statistics_query(&mut self) { + if let Err(cause) = self + .context + .0 + .render_pass_end_pipeline_statistics_query(&mut self.pass) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::end_pipeline_statistics_query", + ); + } + } + + fn execute_bundles( + &mut self, + render_bundles: &mut dyn Iterator, + ) { + let temp_render_bundles = render_bundles + .map(|rb| rb.as_core().id) + .collect::>(); + if let Err(cause) = self + .context + .0 + .render_pass_execute_bundles(&mut self.pass, &temp_render_bundles) + { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::execute_bundles", + ); + } + } +} + +impl Drop for CoreRenderPass { + fn drop(&mut self) { + if let Err(cause) = self.context.0.render_pass_end(&mut self.pass) { + self.context.handle_error( + &self.error_sink, + cause, + self.pass.label(), + "RenderPass::end", + ); + } + } +} + +impl dispatch::RenderBundleEncoderInterface for CoreRenderBundleEncoder { + fn set_pipeline(&mut self, pipeline: &dispatch::DispatchRenderPipeline) { + let pipeline = pipeline.as_core(); + + wgpu_render_bundle_set_pipeline(&mut self.encoder, pipeline.id) + } + + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&dispatch::DispatchBindGroup>, + offsets: &[crate::DynamicOffset], + ) { + let bg = bind_group.map(|bg| bg.as_core().id); + + unsafe { + wgpu_render_bundle_set_bind_group( + &mut self.encoder, + index, + bg, + offsets.as_ptr(), + offsets.len(), + ) + } + } + + fn set_index_buffer( + &mut self, + buffer: &dispatch::DispatchBuffer, + index_format: crate::IndexFormat, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_core(); + + self.encoder + .set_index_buffer(buffer.id, index_format, offset, size) + } + + fn set_vertex_buffer( + &mut self, + slot: u32, + buffer: &dispatch::DispatchBuffer, + offset: crate::BufferAddress, + size: Option, + ) { + let buffer = buffer.as_core(); + + wgpu_render_bundle_set_vertex_buffer(&mut self.encoder, slot, buffer.id, offset, size) + } + + fn set_immediates(&mut self, offset: u32, data: &[u8]) { + unsafe { + wgpu_render_bundle_set_immediates( + &mut self.encoder, + offset, + data.len().try_into().unwrap(), + data.as_ptr(), + ) + } + } + + fn draw(&mut self, vertices: Range, instances: Range) { + wgpu_render_bundle_draw( + &mut self.encoder, + vertices.end - vertices.start, + instances.end - instances.start, + vertices.start, + instances.start, + ) + } + + fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range) { + wgpu_render_bundle_draw_indexed( + &mut self.encoder, + indices.end - indices.start, + instances.end - instances.start, + indices.start, + base_vertex, + instances.start, + ) + } + + fn draw_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let indirect_buffer = indirect_buffer.as_core(); + + wgpu_render_bundle_draw_indirect(&mut self.encoder, indirect_buffer.id, indirect_offset) + } + + fn draw_indexed_indirect( + &mut self, + indirect_buffer: &dispatch::DispatchBuffer, + indirect_offset: crate::BufferAddress, + ) { + let indirect_buffer = indirect_buffer.as_core(); + + wgpu_render_bundle_draw_indexed_indirect( + &mut self.encoder, + indirect_buffer.id, + indirect_offset, + ) + } + + fn finish(self, desc: &crate::RenderBundleDescriptor<'_>) -> dispatch::DispatchRenderBundle + where + Self: Sized, + { + let (id, error) = self.context.0.render_bundle_encoder_finish( + self.encoder, + &desc.map_label(|l| l.map(Borrowed)), + None, + ); + if let Some(err) = error { + self.context + .handle_error_fatal(err, "RenderBundleEncoder::finish"); + } + CoreRenderBundle { + context: self.context.clone(), + id, + } + .into() + } +} + +impl dispatch::RenderBundleInterface for CoreRenderBundle {} + +impl Drop for CoreRenderBundle { + fn drop(&mut self) { + self.context.0.render_bundle_drop(self.id) + } +} + +impl dispatch::SurfaceInterface for CoreSurface { + fn get_capabilities(&self, adapter: &dispatch::DispatchAdapter) -> wgt::SurfaceCapabilities { + let adapter = adapter.as_core(); + + self.context + .0 + .surface_get_capabilities(self.id, adapter.id) + .unwrap_or_default() + } + + fn configure(&self, device: &dispatch::DispatchDevice, config: &crate::SurfaceConfiguration) { + let device = device.as_core(); + + let error = self.context.0.surface_configure(self.id, device.id, config); + if let Some(e) = error { + self.context + .handle_error_nolabel(&device.error_sink, e, "Surface::configure"); + } else { + *self.configured_device.lock() = Some(device.id); + *self.error_sink.lock() = Some(device.error_sink.clone()); + } + } + + fn get_current_texture( + &self, + ) -> ( + Option, + crate::SurfaceStatus, + dispatch::DispatchSurfaceOutputDetail, + ) { + let error_sink = if let Some(error_sink) = self.error_sink.lock().as_ref() { + error_sink.clone() + } else { + Arc::new(Mutex::new(ErrorSinkRaw::new())) + }; + + let output_detail = CoreSurfaceOutputDetail { + context: self.context.clone(), + surface_id: self.id, + error_sink: error_sink.clone(), + } + .into(); + + match self.context.0.surface_get_current_texture(self.id, None) { + Ok(wgc::present::SurfaceOutput { + status, + texture: texture_id, + }) => { + let data = texture_id + .map(|id| CoreTexture { + context: self.context.clone(), + id, + error_sink, + }) + .map(Into::into); + + (data, status, output_detail) + } + Err(err) => { + let error_sink = self.error_sink.lock(); + match error_sink.as_ref() { + Some(error_sink) => { + self.context.handle_error_nolabel( + error_sink, + err, + "Surface::get_current_texture_view", + ); + (None, crate::SurfaceStatus::Validation, output_detail) + } + None => self + .context + .handle_error_fatal(err, "Surface::get_current_texture_view"), + } + } + } + } +} + +impl Drop for CoreSurface { + fn drop(&mut self) { + self.context.0.surface_drop(self.id) + } +} + +impl dispatch::SurfaceOutputDetailInterface for CoreSurfaceOutputDetail { + fn present(&self) { + match self.context.0.surface_present(self.surface_id) { + Ok(_status) => (), + Err(err) => { + self.context + .handle_error_nolabel(&self.error_sink, err, "Surface::present"); + } + } + } + + fn texture_discard(&self) { + match self.context.0.surface_texture_discard(self.surface_id) { + Ok(_status) => (), + Err(err) => self + .context + .handle_error_fatal(err, "Surface::discard_texture"), + } + } +} +impl Drop for CoreSurfaceOutputDetail { + fn drop(&mut self) { + // Discard gets called by the api struct + + // no-op + } +} + +impl dispatch::QueueWriteBufferInterface for CoreQueueWriteBuffer { + #[inline] + fn len(&self) -> usize { + self.mapping.len() + } + + #[inline] + unsafe fn write_slice(&mut self) -> WriteOnly<'_, [u8]> { + unsafe { self.mapping.write_slice() } + } +} +impl Drop for CoreQueueWriteBuffer { + fn drop(&mut self) { + // The api struct calls queue.write_staging_buffer + + // no-op + } +} + +impl dispatch::BufferMappedRangeInterface for CoreBufferMappedRange { + #[inline] + fn len(&self) -> usize { + self.size + } + + #[inline] + unsafe fn read_slice(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.size) } + } + + #[inline] + unsafe fn write_slice(&mut self) -> WriteOnly<'_, [u8]> { + unsafe { WriteOnly::new(NonNull::slice_from_raw_parts(self.ptr, self.size)) } + } + + #[cfg(webgpu)] + fn as_uint8array(&self) -> &js_sys::Uint8Array { + panic!("Only available on WebGPU") + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/backend/wgpu_core/thread_id.rs b/third_party/wgpu-29.0.4-patched/src/backend/wgpu_core/thread_id.rs new file mode 100644 index 00000000..10bc9ae8 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/backend/wgpu_core/thread_id.rs @@ -0,0 +1,30 @@ +//! Implementation of thread IDs for error scope tracking. +//! +//! Supports both std and no_std environments, though +//! the no_std implementation is a stub that does not +//! actually distinguish between threads. + +#[cfg(feature = "std")] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ThreadId(std::thread::ThreadId); + +#[cfg(feature = "std")] +impl ThreadId { + pub fn current() -> Self { + ThreadId(std::thread::current().id()) + } +} + +#[cfg(not(feature = "std"))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ThreadId(()); + +#[cfg(not(feature = "std"))] +impl ThreadId { + pub fn current() -> Self { + // A simple stub implementation for non-std environments. On + // no_std but multithreaded platforms, this will work, but + // make error scope global rather than thread-local. + ThreadId(()) + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/cmp.rs b/third_party/wgpu-29.0.4-patched/src/cmp.rs new file mode 100644 index 00000000..c2ec1265 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/cmp.rs @@ -0,0 +1,109 @@ +//! We need to impl `PartialEq`, `Eq`, `PartialOrd`, `Ord`, and `Hash` for all handle types in wgpu. +//! +//! For types that have some already-unique property, we can use that property to implement these traits. +//! +//! For types (like WebGPU) that don't have such a property, we generate an identifier and use that. + +#[cfg(supports_64bit_atomics)] +pub use core::sync::atomic::AtomicU64; +#[cfg(not(supports_64bit_atomics))] +pub use portable_atomic::AtomicU64; + +use core::{num::NonZeroU64, sync::atomic::Ordering}; + +static NEXT_ID: AtomicU64 = AtomicU64::new(1); + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Identifier { + inner: NonZeroU64, +} + +impl Identifier { + pub fn create() -> Self { + let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); + // Safety: Will take 7000+ years of constant incrementing to overflow. It's fine. + let inner = unsafe { NonZeroU64::new_unchecked(id) }; + Self { inner } + } +} + +/// Implements `PartialEq`, `Eq`, `PartialOrd`, `Ord`, and `Hash` for a type by proxying the operations to a single field. +/// +/// ```ignore +/// impl_eq_ord_hash_proxy!(MyType => .field); +/// ``` +macro_rules! impl_eq_ord_hash_proxy { + ($type:ty => $($access:tt)*) => { + impl PartialEq for $type { + fn eq(&self, other: &Self) -> bool { + self $($access)* == other $($access)* + } + } + + impl Eq for $type {} + + impl PartialOrd for $type { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for $type { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self $($access)*.cmp(&other $($access)*) + } + } + + impl core::hash::Hash for $type { + fn hash(&self, state: &mut H) { + self $($access)*.hash(state) + } + } + }; +} + +/// Implements `PartialEq`, `Eq`, `PartialOrd`, `Ord`, and `Hash` for a type by comparing the addresses of the `Arc`s. +/// +/// ```ignore +/// impl_eq_ord_hash_arc_address!(MyType => .field); +/// ``` +#[cfg_attr(not(any(wgpu_core, custom)), expect(unused_macros))] +macro_rules! impl_eq_ord_hash_arc_address { + ($type:ty => $($access:tt)*) => { + impl PartialEq for $type { + fn eq(&self, other: &Self) -> bool { + let address_left = alloc::sync::Arc::as_ptr(&self $($access)*); + let address_right = alloc::sync::Arc::as_ptr(&other $($access)*); + + address_left == address_right + } + } + + impl Eq for $type {} + + impl PartialOrd for $type { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for $type { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + let address_left = alloc::sync::Arc::as_ptr(&self $($access)*); + let address_right = alloc::sync::Arc::as_ptr(&other $($access)*); + + address_left.cmp(&address_right) + } + } + + impl core::hash::Hash for $type { + fn hash(&self, state: &mut H) { + let address = alloc::sync::Arc::as_ptr(&self $($access)*); + address.hash(state) + } + } + }; +} + +#[cfg_attr(not(any(wgpu_core, custom)), expect(unused_imports))] +pub(crate) use {impl_eq_ord_hash_arc_address, impl_eq_ord_hash_proxy}; diff --git a/third_party/wgpu-29.0.4-patched/src/dispatch.rs b/third_party/wgpu-29.0.4-patched/src/dispatch.rs new file mode 100644 index 00000000..d2b2366e --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/dispatch.rs @@ -0,0 +1,985 @@ +//! Infrastructure for dispatching calls to the appropriate "backend". The "backends" are: +//! +//! - `wgpu_core`: An implementation of the the wgpu api on top of various native graphics APIs. +//! - `webgpu`: An implementation of the wgpu api which calls WebGPU directly. +//! +//! The interface traits are all object safe and listed in the `InterfaceTypes` trait. +//! +//! The method for dispatching should optimize well if only one backend is +//! compiled in, as-if there was no dispatching at all. See the comments on +//! [`dispatch_types`] for details. +//! +//! [`dispatch_types`]: macro.dispatch_types.html + +#![allow( + drop_bounds, + reason = "This exists to remind implementors to impl drop." +)] +#![allow(clippy::too_many_arguments, reason = "It's fine.")] +#![allow( + missing_docs, + clippy::missing_safety_doc, + reason = "Interfaces are not documented" +)] +#![allow( + clippy::len_without_is_empty, + reason = "trait is minimal, not ergonomic" +)] + +use crate::{Blas, Tlas, WasmNotSend, WasmNotSendSync, WriteOnly}; + +use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; +use core::{any::Any, fmt::Debug, future::Future, hash::Hash, ops::Range, pin::Pin}; + +#[cfg(custom)] +use crate::backend::custom::*; +#[cfg(webgpu)] +use crate::backend::webgpu::*; +#[cfg(wgpu_core)] +use crate::backend::wgpu_core::*; + +/// Create a single trait with the given supertraits and a blanket impl for all types that implement them. +/// +/// This is useful for creating a trait alias as a shorthand. +macro_rules! trait_alias { + ($name:ident: $($bound:tt)+) => { + pub trait $name: $($bound)+ {} + impl $name for T {} + }; +} + +// Various return futures in the API. +trait_alias!(RequestAdapterFuture: Future> + WasmNotSend + 'static); +trait_alias!(RequestDeviceFuture: Future> + WasmNotSend + 'static); +trait_alias!(PopErrorScopeFuture: Future> + WasmNotSend + 'static); +trait_alias!(ShaderCompilationInfoFuture: Future + WasmNotSend + 'static); +trait_alias!(EnumerateAdapterFuture: Future> + WasmNotSend + 'static); + +// We can't use trait aliases here, as you can't convert from a dyn Trait to dyn Supertrait _yet_. +#[cfg(send_sync)] +pub type BoxDeviceLostCallback = Box; +#[cfg(not(send_sync))] +pub type BoxDeviceLostCallback = Box; +#[cfg(send_sync)] +pub type BoxSubmittedWorkDoneCallback = Box; +#[cfg(not(send_sync))] +pub type BoxSubmittedWorkDoneCallback = Box; +#[cfg(send_sync)] +pub type BufferMapCallback = Box) + Send + 'static>; +#[cfg(not(send_sync))] +pub type BufferMapCallback = Box) + 'static>; + +#[cfg(send_sync)] +pub type BlasCompactCallback = Box) + Send + 'static>; +#[cfg(not(send_sync))] +pub type BlasCompactCallback = Box) + 'static>; + +// remove when rust 1.86 +#[cfg_attr(not(custom), expect(dead_code))] +pub trait AsAny { + fn as_any(&self) -> &dyn Any; +} + +impl AsAny for T { + fn as_any(&self) -> &dyn Any { + self + } +} + +// Common traits on all the interface traits +trait_alias!(CommonTraits: AsAny + Any + Debug + WasmNotSendSync); + +pub trait InstanceInterface: CommonTraits { + fn new(desc: crate::InstanceDescriptor) -> Self + where + Self: Sized; + + unsafe fn create_surface( + &self, + target: crate::SurfaceTargetUnsafe, + ) -> Result; + + fn request_adapter( + &self, + options: &crate::RequestAdapterOptions<'_, '_>, + ) -> Pin>; + + fn poll_all_devices(&self, force_wait: bool) -> bool; + + #[cfg(feature = "wgsl")] + fn wgsl_language_features(&self) -> crate::WgslLanguageFeatures; + + fn enumerate_adapters(&self, backends: crate::Backends) + -> Pin>; +} + +pub trait AdapterInterface: CommonTraits { + fn request_device( + &self, + desc: &crate::DeviceDescriptor<'_>, + ) -> Pin>; + + fn is_surface_supported(&self, surface: &DispatchSurface) -> bool; + + fn features(&self) -> crate::Features; + + fn limits(&self) -> crate::Limits; + + fn downlevel_capabilities(&self) -> crate::DownlevelCapabilities; + + fn get_info(&self) -> crate::AdapterInfo; + + fn get_texture_format_features( + &self, + format: crate::TextureFormat, + ) -> crate::TextureFormatFeatures; + + fn get_presentation_timestamp(&self) -> crate::PresentationTimestamp; + + fn cooperative_matrix_properties(&self) -> Vec; +} + +pub trait DeviceInterface: CommonTraits { + fn features(&self) -> crate::Features; + fn limits(&self) -> crate::Limits; + fn adapter_info(&self) -> crate::AdapterInfo; + + fn create_shader_module( + &self, + desc: crate::ShaderModuleDescriptor<'_>, + shader_bound_checks: crate::ShaderRuntimeChecks, + ) -> DispatchShaderModule; + + unsafe fn create_shader_module_passthrough( + &self, + desc: &crate::ShaderModuleDescriptorPassthrough<'_>, + ) -> DispatchShaderModule; + + fn create_bind_group_layout( + &self, + desc: &crate::BindGroupLayoutDescriptor<'_>, + ) -> DispatchBindGroupLayout; + fn create_bind_group(&self, desc: &crate::BindGroupDescriptor<'_>) -> DispatchBindGroup; + fn create_pipeline_layout( + &self, + desc: &crate::PipelineLayoutDescriptor<'_>, + ) -> DispatchPipelineLayout; + fn create_render_pipeline( + &self, + desc: &crate::RenderPipelineDescriptor<'_>, + ) -> DispatchRenderPipeline; + fn create_mesh_pipeline( + &self, + desc: &crate::MeshPipelineDescriptor<'_>, + ) -> DispatchRenderPipeline; + fn create_compute_pipeline( + &self, + desc: &crate::ComputePipelineDescriptor<'_>, + ) -> DispatchComputePipeline; + unsafe fn create_pipeline_cache( + &self, + desc: &crate::PipelineCacheDescriptor<'_>, + ) -> DispatchPipelineCache; + fn create_buffer(&self, desc: &crate::BufferDescriptor<'_>) -> DispatchBuffer; + fn create_texture(&self, desc: &crate::TextureDescriptor<'_>) -> DispatchTexture; + fn create_external_texture( + &self, + desc: &crate::ExternalTextureDescriptor<'_>, + planes: &[&crate::TextureView], + ) -> DispatchExternalTexture; + fn create_blas( + &self, + desc: &crate::CreateBlasDescriptor<'_>, + sizes: crate::BlasGeometrySizeDescriptors, + ) -> (Option, DispatchBlas); + fn create_tlas(&self, desc: &crate::CreateTlasDescriptor<'_>) -> DispatchTlas; + fn create_sampler(&self, desc: &crate::SamplerDescriptor<'_>) -> DispatchSampler; + fn create_query_set(&self, desc: &crate::QuerySetDescriptor<'_>) -> DispatchQuerySet; + fn create_command_encoder( + &self, + desc: &crate::CommandEncoderDescriptor<'_>, + ) -> DispatchCommandEncoder; + fn create_render_bundle_encoder( + &self, + desc: &crate::RenderBundleEncoderDescriptor<'_>, + ) -> DispatchRenderBundleEncoder; + + fn set_device_lost_callback(&self, device_lost_callback: BoxDeviceLostCallback); + + fn on_uncaptured_error(&self, handler: Arc); + // Returns index on the stack of the pushed error scope. + fn push_error_scope(&self, filter: crate::ErrorFilter) -> u32; + fn pop_error_scope(&self, index: u32) -> Pin>; + + unsafe fn start_graphics_debugger_capture(&self); + unsafe fn stop_graphics_debugger_capture(&self); + + fn poll(&self, poll_type: wgt::PollType) -> Result; + + fn get_internal_counters(&self) -> crate::InternalCounters; + fn generate_allocator_report(&self) -> Option; + + fn destroy(&self); +} + +pub trait QueueInterface: CommonTraits { + fn write_buffer(&self, buffer: &DispatchBuffer, offset: crate::BufferAddress, data: &[u8]); + + fn create_staging_buffer(&self, size: crate::BufferSize) -> Option; + fn validate_write_buffer( + &self, + buffer: &DispatchBuffer, + offset: crate::BufferAddress, + size: crate::BufferSize, + ) -> Option<()>; + fn write_staging_buffer( + &self, + buffer: &DispatchBuffer, + offset: crate::BufferAddress, + staging_buffer: &DispatchQueueWriteBuffer, + ); + + fn write_texture( + &self, + texture: crate::TexelCopyTextureInfo<'_>, + data: &[u8], + data_layout: crate::TexelCopyBufferLayout, + size: crate::Extent3d, + ); + #[cfg(web)] + fn copy_external_image_to_texture( + &self, + source: &crate::CopyExternalImageSourceInfo, + dest: crate::CopyExternalImageDestInfo<&crate::api::Texture>, + size: crate::Extent3d, + ); + + /// Submit must always drain the iterator, even in the case of error. + fn submit(&self, command_buffers: &mut dyn Iterator) -> u64; + + fn get_timestamp_period(&self) -> f32; + fn on_submitted_work_done(&self, callback: BoxSubmittedWorkDoneCallback); + + fn compact_blas(&self, blas: &DispatchBlas) -> (Option, DispatchBlas); +} + +pub trait ShaderModuleInterface: CommonTraits { + fn get_compilation_info(&self) -> Pin>; +} +pub trait BindGroupLayoutInterface: CommonTraits {} +pub trait BindGroupInterface: CommonTraits {} +pub trait TextureViewInterface: CommonTraits {} +pub trait SamplerInterface: CommonTraits {} +pub trait BufferInterface: CommonTraits { + fn map_async( + &self, + mode: crate::MapMode, + range: Range, + callback: BufferMapCallback, + ); + fn get_mapped_range(&self, sub_range: Range) + -> DispatchBufferMappedRange; + + fn unmap(&self); + + fn destroy(&self); +} +pub trait TextureInterface: CommonTraits { + fn create_view(&self, desc: &crate::TextureViewDescriptor<'_>) -> DispatchTextureView; + + fn destroy(&self); +} +pub trait ExternalTextureInterface: CommonTraits { + fn destroy(&self); +} +pub trait BlasInterface: CommonTraits { + fn prepare_compact_async(&self, callback: BlasCompactCallback); + fn ready_for_compaction(&self) -> bool; +} +pub trait TlasInterface: CommonTraits {} +pub trait QuerySetInterface: CommonTraits {} +pub trait PipelineLayoutInterface: CommonTraits {} +pub trait RenderPipelineInterface: CommonTraits { + fn get_bind_group_layout(&self, index: u32) -> DispatchBindGroupLayout; +} +pub trait ComputePipelineInterface: CommonTraits { + fn get_bind_group_layout(&self, index: u32) -> DispatchBindGroupLayout; +} +pub trait PipelineCacheInterface: CommonTraits { + fn get_data(&self) -> Option>; +} +pub trait CommandEncoderInterface: CommonTraits { + fn copy_buffer_to_buffer( + &self, + source: &DispatchBuffer, + source_offset: crate::BufferAddress, + destination: &DispatchBuffer, + destination_offset: crate::BufferAddress, + copy_size: Option, + ); + fn copy_buffer_to_texture( + &self, + source: crate::TexelCopyBufferInfo<'_>, + destination: crate::TexelCopyTextureInfo<'_>, + copy_size: crate::Extent3d, + ); + fn copy_texture_to_buffer( + &self, + source: crate::TexelCopyTextureInfo<'_>, + destination: crate::TexelCopyBufferInfo<'_>, + copy_size: crate::Extent3d, + ); + fn copy_texture_to_texture( + &self, + source: crate::TexelCopyTextureInfo<'_>, + destination: crate::TexelCopyTextureInfo<'_>, + copy_size: crate::Extent3d, + ); + + fn begin_compute_pass(&self, desc: &crate::ComputePassDescriptor<'_>) -> DispatchComputePass; + fn begin_render_pass(&self, desc: &crate::RenderPassDescriptor<'_>) -> DispatchRenderPass; + fn finish(&mut self) -> DispatchCommandBuffer; + + fn clear_texture( + &self, + texture: &DispatchTexture, + subresource_range: &crate::ImageSubresourceRange, + ); + fn clear_buffer( + &self, + buffer: &DispatchBuffer, + offset: crate::BufferAddress, + size: Option, + ); + + fn insert_debug_marker(&self, label: &str); + fn push_debug_group(&self, label: &str); + fn pop_debug_group(&self); + + fn write_timestamp(&self, query_set: &DispatchQuerySet, query_index: u32); + fn resolve_query_set( + &self, + query_set: &DispatchQuerySet, + first_query: u32, + query_count: u32, + destination: &DispatchBuffer, + destination_offset: crate::BufferAddress, + ); + fn mark_acceleration_structures_built<'a>( + &self, + blas: &mut dyn Iterator, + tlas: &mut dyn Iterator, + ); + + fn build_acceleration_structures<'a>( + &self, + blas: &mut dyn Iterator>, + tlas: &mut dyn Iterator, + ); + + fn transition_resources<'a>( + &mut self, + buffer_transitions: &mut dyn Iterator>, + texture_transitions: &mut dyn Iterator>, + ); +} +pub trait ComputePassInterface: CommonTraits + Drop { + fn set_pipeline(&mut self, pipeline: &DispatchComputePipeline); + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&DispatchBindGroup>, + offsets: &[crate::DynamicOffset], + ); + fn set_immediates(&mut self, offset: u32, data: &[u8]); + + fn insert_debug_marker(&mut self, label: &str); + fn push_debug_group(&mut self, group_label: &str); + fn pop_debug_group(&mut self); + + fn write_timestamp(&mut self, query_set: &DispatchQuerySet, query_index: u32); + fn begin_pipeline_statistics_query(&mut self, query_set: &DispatchQuerySet, query_index: u32); + fn end_pipeline_statistics_query(&mut self); + + fn dispatch_workgroups(&mut self, x: u32, y: u32, z: u32); + fn dispatch_workgroups_indirect( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + ); +} +pub trait RenderPassInterface: CommonTraits + Drop { + fn set_pipeline(&mut self, pipeline: &DispatchRenderPipeline); + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&DispatchBindGroup>, + offsets: &[crate::DynamicOffset], + ); + fn set_index_buffer( + &mut self, + buffer: &DispatchBuffer, + index_format: crate::IndexFormat, + offset: crate::BufferAddress, + size: Option, + ); + fn set_vertex_buffer( + &mut self, + slot: u32, + buffer: &DispatchBuffer, + offset: crate::BufferAddress, + size: Option, + ); + fn set_immediates(&mut self, offset: u32, data: &[u8]); + fn set_blend_constant(&mut self, color: crate::Color); + fn set_scissor_rect(&mut self, x: u32, y: u32, width: u32, height: u32); + fn set_viewport( + &mut self, + x: f32, + y: f32, + width: f32, + height: f32, + min_depth: f32, + max_depth: f32, + ); + fn set_stencil_reference(&mut self, reference: u32); + + fn draw(&mut self, vertices: Range, instances: Range); + fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range); + fn draw_mesh_tasks(&mut self, group_count_x: u32, group_count_y: u32, group_count_z: u32); + fn draw_indirect( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + ); + fn draw_indexed_indirect( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + ); + fn draw_mesh_tasks_indirect( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + ); + + fn multi_draw_indirect( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + count: u32, + ); + fn multi_draw_indexed_indirect( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + count: u32, + ); + fn multi_draw_indirect_count( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + count_buffer: &DispatchBuffer, + count_buffer_offset: crate::BufferAddress, + max_count: u32, + ); + fn multi_draw_mesh_tasks_indirect( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + count: u32, + ); + fn multi_draw_indexed_indirect_count( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + count_buffer: &DispatchBuffer, + count_buffer_offset: crate::BufferAddress, + max_count: u32, + ); + fn multi_draw_mesh_tasks_indirect_count( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + count_buffer: &DispatchBuffer, + count_buffer_offset: crate::BufferAddress, + max_count: u32, + ); + + fn insert_debug_marker(&mut self, label: &str); + fn push_debug_group(&mut self, group_label: &str); + fn pop_debug_group(&mut self); + + fn write_timestamp(&mut self, query_set: &DispatchQuerySet, query_index: u32); + fn begin_occlusion_query(&mut self, query_index: u32); + fn end_occlusion_query(&mut self); + fn begin_pipeline_statistics_query(&mut self, query_set: &DispatchQuerySet, query_index: u32); + fn end_pipeline_statistics_query(&mut self); + + fn execute_bundles(&mut self, render_bundles: &mut dyn Iterator); +} + +pub trait RenderBundleEncoderInterface: CommonTraits { + fn set_pipeline(&mut self, pipeline: &DispatchRenderPipeline); + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&DispatchBindGroup>, + offsets: &[crate::DynamicOffset], + ); + fn set_index_buffer( + &mut self, + buffer: &DispatchBuffer, + index_format: crate::IndexFormat, + offset: crate::BufferAddress, + size: Option, + ); + fn set_vertex_buffer( + &mut self, + slot: u32, + buffer: &DispatchBuffer, + offset: crate::BufferAddress, + size: Option, + ); + fn set_immediates(&mut self, offset: u32, data: &[u8]); + + fn draw(&mut self, vertices: Range, instances: Range); + fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range); + fn draw_indirect( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + ); + fn draw_indexed_indirect( + &mut self, + indirect_buffer: &DispatchBuffer, + indirect_offset: crate::BufferAddress, + ); + + fn finish(self, desc: &crate::RenderBundleDescriptor<'_>) -> DispatchRenderBundle + where + Self: Sized; +} + +pub trait CommandBufferInterface: CommonTraits {} +pub trait RenderBundleInterface: CommonTraits {} + +pub trait SurfaceInterface: CommonTraits { + fn get_capabilities(&self, adapter: &DispatchAdapter) -> crate::SurfaceCapabilities; + + fn configure(&self, device: &DispatchDevice, config: &crate::SurfaceConfiguration); + fn get_current_texture( + &self, + ) -> ( + Option, + crate::SurfaceStatus, + DispatchSurfaceOutputDetail, + ); +} + +pub trait SurfaceOutputDetailInterface: CommonTraits { + fn present(&self); + fn texture_discard(&self); +} + +pub trait QueueWriteBufferInterface: CommonTraits { + fn len(&self) -> usize; + + /// # Safety + /// + /// Must only be used on write, not read, mappings. + unsafe fn write_slice(&mut self) -> WriteOnly<'_, [u8]>; +} + +pub trait BufferMappedRangeInterface: CommonTraits { + fn len(&self) -> usize; + + /// # Safety + /// + /// Must only be used on read, not write, mappings. + unsafe fn read_slice(&self) -> &[u8]; + + /// # Safety + /// + /// Must only be used on write, not read, mappings. + unsafe fn write_slice(&mut self) -> WriteOnly<'_, [u8]>; + + #[cfg(webgpu)] + fn as_uint8array(&self) -> &js_sys::Uint8Array; +} + +/// Generates a dispatch type for some `wgpu` API type. +/// +/// Invocations of this macro take one of the following forms: +/// +/// ```ignore +/// dispatch_types! {mut type D: I = Core, Web, Dyn } +/// dispatch_types! {ref type D: I = Core, Web, Dyn } +/// ``` +/// +/// This defines `D` as a type that dereferences to a `dyn I` trait object. Most uses of +/// `D` in the rest of this crate just call the methods from the `dyn I` object, not from +/// `D` itself. +/// +/// Internally, `D` is an enum with up to three variants holding values of type `Core`, +/// `Web`, and `Dyn`, all of which must implement `I`. `Core`, `Web` and `Dyn` are the +/// types from the `wgpu_core`, `webgpu`, and `custom` submodules of `wgpu::backend` that +/// correspond to `D`. The macro generates `Deref` and `DerefMut` implementations that +/// match on this enum and produce a `dyn I` reference for each variant. +/// +/// The macro's `mut type` form defines `D` as the unique owner of the backend type, with +/// a `DerefMut` implementation, and `as_*_mut` methods that return `&mut` references. +/// This `D` does not implement `Clone`. +/// +/// The macro's `ref type` form defines `D` to hold an `Arc` pointing to the backend type, +/// permitting `Clone` and `Deref`, but losing exclusive, mutable access. +/// +/// For example: +/// +/// ```ignore +/// dispatch_types! {ref type DispatchBuffer: BufferInterface = +/// CoreBuffer, WebBuffer, DynBuffer} +/// ``` +/// +/// This defines `DispatchBuffer` as a type that dereferences to `&dyn BufferInterface`, +/// which has methods like `map_async` and `destroy`. The enum would be: +/// +/// ```ignore +/// pub enum DispatchBuffer { +/// #[cfg(wgpu_core)] +/// Core(Arc), +/// #[cfg(webgpu)] +/// WebGPU(WebBuffer), +/// #[cfg(custom)] +/// Custom(DynBuffer), +/// } +/// ``` +/// +/// This macro also defines `as_*` methods so that the backend implementations can +/// dereference other arguments. +/// +/// ## Devirtualization +/// +/// The dispatch types generated by this macro are carefully designed to allow the +/// compiler to completely devirtualize calls in most circumstances. +/// +/// Note that every variant of the enum generated by this macro is under a `#[cfg]`. +/// Naturally, the `match` expressions in the `Deref` and `DerefMut` implementations have +/// matching `#[cfg]` attributes on each match arm. +/// +/// In practice, when `wgpu`'s `"custom"` feature is not enabled, there is usually only +/// one variant in the `enum`, making it effectively a newtype around the sole variant's +/// data: it has no discriminant to branch on, and the `match` expressions are removed +/// entirely by the compiler. +/// +/// In this case, when we invoke a method from the interface trait `I` on a dispatch type, +/// the `Deref` and `DerefMut` implementations' `match` statements build a `&dyn I` for +/// the data, on which we immediately invoke a method. The vtable is a constant, allowing +/// the Rust compiler to turn the `dyn` method call into an ordinary method call. This +/// creates opportunities for inlining. +/// +/// Similarly, the `as_*` methods are free when there is only one backend. +macro_rules! dispatch_types { + ( + ref type $name:ident: $interface:ident = $core_type:ident,$webgpu_type:ident,$custom_type:ident + ) => { + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] + pub enum $name { + #[cfg(wgpu_core)] + Core(Arc<$core_type>), + #[cfg(webgpu)] + WebGPU($webgpu_type), + #[allow(clippy::allow_attributes, private_interfaces)] + #[cfg(custom)] + Custom($custom_type), + } + + impl $name { + #[cfg(wgpu_core)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_core(&self) -> &$core_type { + match self { + Self::Core(value) => value, + _ => panic!(concat!(stringify!($name), " is not core")), + } + } + + #[cfg(wgpu_core)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_core_opt(&self) -> Option<&$core_type> { + match self { + Self::Core(value) => Some(value), + _ => None, + } + } + + #[cfg(custom)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_custom(&self) -> Option<&T> { + match self { + Self::Custom(value) => value.downcast(), + _ => None, + } + } + + #[cfg(webgpu)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_webgpu(&self) -> &$webgpu_type { + match self { + Self::WebGPU(value) => value, + _ => panic!(concat!(stringify!($name), " is not webgpu")), + } + } + + #[cfg(webgpu)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_webgpu_opt(&self) -> Option<&$webgpu_type> { + match self { + Self::WebGPU(value) => Some(value), + _ => None, + } + } + + #[cfg(custom)] + #[inline] + pub fn custom(t: T) -> Self { + Self::Custom($custom_type::new(t)) + } + } + + #[cfg(wgpu_core)] + impl From<$core_type> for $name { + #[inline] + fn from(value: $core_type) -> Self { + Self::Core(Arc::new(value)) + } + } + + #[cfg(webgpu)] + impl From<$webgpu_type> for $name { + #[inline] + fn from(value: $webgpu_type) -> Self { + Self::WebGPU(value) + } + } + + impl core::ops::Deref for $name { + type Target = dyn $interface; + + #[inline] + fn deref(&self) -> &Self::Target { + match self { + #[cfg(wgpu_core)] + Self::Core(value) => value.as_ref(), + #[cfg(webgpu)] + Self::WebGPU(value) => value, + #[cfg(custom)] + Self::Custom(value) => value.deref(), + #[cfg(not(any(wgpu_core, webgpu)))] + _ => panic!("No context available. You need to enable one of wgpu's backend feature build flags."), + } + } + } + }; + ( + mut type $name:ident: $interface:ident = $core_type:ident,$webgpu_type:ident,$custom_type:ident + ) => { + #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] + pub enum $name { + #[cfg(wgpu_core)] + Core($core_type), + #[cfg(webgpu)] + WebGPU($webgpu_type), + #[allow(clippy::allow_attributes, private_interfaces)] + #[cfg(custom)] + Custom($custom_type), + } + + impl $name { + #[cfg(wgpu_core)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_core(&self) -> &$core_type { + match self { + Self::Core(value) => value, + _ => panic!(concat!(stringify!($name), " is not core")), + } + } + + #[cfg(wgpu_core)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_core_mut(&mut self) -> &mut $core_type { + match self { + Self::Core(value) => value, + _ => panic!(concat!(stringify!($name), " is not core")), + } + } + + #[cfg(wgpu_core)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_core_opt(&self) -> Option<&$core_type> { + match self { + Self::Core(value) => Some(value), + _ => None, + } + } + + #[cfg(wgpu_core)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_core_mut_opt( + &mut self, + ) -> Option<&mut $core_type> { + match self { + Self::Core(value) => Some(value), + _ => None, + } + } + + #[cfg(custom)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_custom(&self) -> Option<&T> { + match self { + Self::Custom(value) => value.downcast(), + _ => None, + } + } + + #[cfg(webgpu)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_webgpu(&self) -> &$webgpu_type { + match self { + Self::WebGPU(value) => value, + _ => panic!(concat!(stringify!($name), " is not webgpu")), + } + } + + #[cfg(webgpu)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_webgpu_mut(&mut self) -> &mut $webgpu_type { + match self { + Self::WebGPU(value) => value, + _ => panic!(concat!(stringify!($name), " is not webgpu")), + } + } + + #[cfg(webgpu)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_webgpu_opt(&self) -> Option<&$webgpu_type> { + match self { + Self::WebGPU(value) => Some(value), + _ => None, + } + } + + #[cfg(webgpu)] + #[inline] + #[allow(clippy::allow_attributes, unused)] + pub fn as_webgpu_mut_opt( + &mut self, + ) -> Option<&mut $webgpu_type> { + match self { + Self::WebGPU(value) => Some(value), + _ => None, + } + } + + #[cfg(custom)] + #[inline] + pub fn custom(t: T) -> Self { + Self::Custom($custom_type::new(t)) + } + } + + #[cfg(wgpu_core)] + impl From<$core_type> for $name { + #[inline] + fn from(value: $core_type) -> Self { + Self::Core(value) + } + } + + #[cfg(webgpu)] + impl From<$webgpu_type> for $name { + #[inline] + fn from(value: $webgpu_type) -> Self { + Self::WebGPU(value) + } + } + + impl core::ops::Deref for $name { + type Target = dyn $interface; + + #[inline] + fn deref(&self) -> &Self::Target { + match self { + #[cfg(wgpu_core)] + Self::Core(value) => value, + #[cfg(webgpu)] + Self::WebGPU(value) => value, + #[cfg(custom)] + Self::Custom(value) => value.deref(), + #[cfg(not(any(wgpu_core, webgpu)))] + _ => panic!("No context available. You need to enable one of wgpu's backend feature build flags."), + } + } + } + + impl core::ops::DerefMut for $name { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + match self { + #[cfg(wgpu_core)] + Self::Core(value) => value, + #[cfg(webgpu)] + Self::WebGPU(value) => value, + #[cfg(custom)] + Self::Custom(value) => value.deref_mut(), + #[cfg(not(any(wgpu_core, webgpu)))] + _ => panic!("No context available. You need to enable one of wgpu's backend feature build flags."), + } + } + } + }; +} + +dispatch_types! {ref type DispatchInstance: InstanceInterface = ContextWgpuCore, ContextWebGpu, DynContext} +dispatch_types! {ref type DispatchAdapter: AdapterInterface = CoreAdapter, WebAdapter, DynAdapter} +dispatch_types! {ref type DispatchDevice: DeviceInterface = CoreDevice, WebDevice, DynDevice} +dispatch_types! {ref type DispatchQueue: QueueInterface = CoreQueue, WebQueue, DynQueue} +dispatch_types! {ref type DispatchShaderModule: ShaderModuleInterface = CoreShaderModule, WebShaderModule, DynShaderModule} +dispatch_types! {ref type DispatchBindGroupLayout: BindGroupLayoutInterface = CoreBindGroupLayout, WebBindGroupLayout, DynBindGroupLayout} +dispatch_types! {ref type DispatchBindGroup: BindGroupInterface = CoreBindGroup, WebBindGroup, DynBindGroup} +dispatch_types! {ref type DispatchTextureView: TextureViewInterface = CoreTextureView, WebTextureView, DynTextureView} +dispatch_types! {ref type DispatchSampler: SamplerInterface = CoreSampler, WebSampler, DynSampler} +dispatch_types! {ref type DispatchBuffer: BufferInterface = CoreBuffer, WebBuffer, DynBuffer} +dispatch_types! {ref type DispatchTexture: TextureInterface = CoreTexture, WebTexture, DynTexture} +dispatch_types! {ref type DispatchExternalTexture: ExternalTextureInterface = CoreExternalTexture, WebExternalTexture, DynExternalTexture} +dispatch_types! {ref type DispatchBlas: BlasInterface = CoreBlas, WebBlas, DynBlas} +dispatch_types! {ref type DispatchTlas: TlasInterface = CoreTlas, WebTlas, DynTlas} +dispatch_types! {ref type DispatchQuerySet: QuerySetInterface = CoreQuerySet, WebQuerySet, DynQuerySet} +dispatch_types! {ref type DispatchPipelineLayout: PipelineLayoutInterface = CorePipelineLayout, WebPipelineLayout, DynPipelineLayout} +dispatch_types! {ref type DispatchRenderPipeline: RenderPipelineInterface = CoreRenderPipeline, WebRenderPipeline, DynRenderPipeline} +dispatch_types! {ref type DispatchComputePipeline: ComputePipelineInterface = CoreComputePipeline, WebComputePipeline, DynComputePipeline} +dispatch_types! {ref type DispatchPipelineCache: PipelineCacheInterface = CorePipelineCache, WebPipelineCache, DynPipelineCache} +dispatch_types! {mut type DispatchCommandEncoder: CommandEncoderInterface = CoreCommandEncoder, WebCommandEncoder, DynCommandEncoder} +dispatch_types! {mut type DispatchComputePass: ComputePassInterface = CoreComputePass, WebComputePassEncoder, DynComputePass} +dispatch_types! {mut type DispatchRenderPass: RenderPassInterface = CoreRenderPass, WebRenderPassEncoder, DynRenderPass} +dispatch_types! {mut type DispatchCommandBuffer: CommandBufferInterface = CoreCommandBuffer, WebCommandBuffer, DynCommandBuffer} +dispatch_types! {mut type DispatchRenderBundleEncoder: RenderBundleEncoderInterface = CoreRenderBundleEncoder, WebRenderBundleEncoder, DynRenderBundleEncoder} +dispatch_types! {ref type DispatchRenderBundle: RenderBundleInterface = CoreRenderBundle, WebRenderBundle, DynRenderBundle} +dispatch_types! {ref type DispatchSurface: SurfaceInterface = CoreSurface, WebSurface, DynSurface} +dispatch_types! {ref type DispatchSurfaceOutputDetail: SurfaceOutputDetailInterface = CoreSurfaceOutputDetail, WebSurfaceOutputDetail, DynSurfaceOutputDetail} +dispatch_types! {mut type DispatchQueueWriteBuffer: QueueWriteBufferInterface = CoreQueueWriteBuffer, WebQueueWriteBuffer, DynQueueWriteBuffer} +dispatch_types! {mut type DispatchBufferMappedRange: BufferMappedRangeInterface = CoreBufferMappedRange, WebBufferMappedRange, DynBufferMappedRange} diff --git a/third_party/wgpu-29.0.4-patched/src/lib.rs b/third_party/wgpu-29.0.4-patched/src/lib.rs new file mode 100644 index 00000000..34620502 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/lib.rs @@ -0,0 +1,190 @@ +//! `wgpu` is a cross-platform, safe, pure-Rust graphics API. It runs natively on +//! Vulkan, Metal, D3D12, and OpenGL; and on top of WebGL2 and WebGPU on wasm. +//! +//! The API is based on the [WebGPU standard][webgpu], but is a fully native Rust library. +//! It serves as the core of the WebGPU integration in Firefox, Servo, and Deno. +//! +//! [webgpu]: https://gpuweb.github.io/gpuweb/ +//! +//! ## Getting Started +//! +//! The main entry point to the API is the [`Instance`] type, from which you can create [`Adapter`], [`Device`], and [`Surface`]. +//! +//! If you are new to `wgpu` and graphics programming, we recommend starting with [Learn Wgpu]. +//! +//! +//! Additionally, [WebGPU Fundamentals] is a tutorial for WebGPU which is very similar to our API, minus differences between Rust and Javascript. +//! +//! We have a [wiki](https://github.com/gfx-rs/wgpu/wiki) which has information on useful architecture patterns, debugging tips, and more getting started information. +//! +//! There are examples for this version [available on GitHub](https://github.com/gfx-rs/wgpu/tree/v29/examples#readme). +//! +//! The API is refcounted, so all handles are cloneable, and if you create a resource which references another, +//! it will automatically keep dependent resources alive. +//! +//! `wgpu` uses the coordinate systems of D3D and Metal. Depth ranges from [0, 1]. +//! +//! | Render | Texture | +//! | --------------------- | ---------------------- | +//! | ![render_coordinates] | ![texture_coordinates] | +//! +//! `wgpu`'s MSRV is **1.87**. +//! +//! [Learn Wgpu]: https://sotrh.github.io/learn-wgpu/ +//! [WebGPU Fundamentals]: https://webgpufundamentals.org/ +//! [render_coordinates]: https://raw.githubusercontent.com/gfx-rs/wgpu/refs/heads/v29/docs/render_coordinates.png +//! [texture_coordinates]: https://raw.githubusercontent.com/gfx-rs/wgpu/refs/heads/v29/docs/texture_coordinates.png +//! +//! ## Extension Specifications +//! +//! While the core of `wgpu` is based on the WebGPU standard, we also support extensions that allow for features that the standard does not have yet. +//! For high-level documentation on how to use these extensions, see documentation on [`Features`] or the relevant specification: +//! +//! 🧪EXPERIMENTAL🧪 APIs are subject to change and may allow undefined behavior if used incorrectly. +//! +//! - 🧪EXPERIMENTAL🧪 [Ray Tracing](https://github.com/gfx-rs/wgpu/blob/v29/docs/api-specs/ray_tracing.md). +//! - 🧪EXPERIMENTAL🧪 [Mesh Shading](https://github.com/gfx-rs/wgpu/blob/v29/docs/api-specs/mesh_shading.md). +//! +//! ## Shader Support +//! +//! `wgpu` can consume shaders in [WGSL](https://gpuweb.github.io/gpuweb/wgsl/), SPIR-V, and GLSL. +//! Both [HLSL](https://github.com/Microsoft/DirectXShaderCompiler) and [GLSL](https://github.com/KhronosGroup/glslang) +//! have compilers to target SPIR-V. All of these shader languages can be used with any backend as we handle all of the conversions. Additionally, support for these shader inputs is not going away. +//! +//! While WebGPU does not support any shading language other than WGSL, we will automatically convert your +//! non-WGSL shaders if you're running on WebGPU. +//! +//! WGSL is always supported by default, but GLSL and SPIR-V need features enabled to compile in support. +//! +//! To enable WGSL shaders, enable the `wgsl` feature of `wgpu` (enabled by default). +//! To enable SPIR-V shaders, enable the `spirv` feature of `wgpu`. +//! To enable GLSL shaders, enable the `glsl` feature of `wgpu`. +//! +//! ## Feature flags +#![doc = document_features::document_features!()] +//! +//! ### Feature Aliases +//! +//! These features aren't actually features on the crate itself, but a convenient shorthand for +//! complicated cases. +//! +//! - **`wgpu_core`** --- Enabled when there is any non-webgpu backend enabled on the platform. +//! - **`naga`** --- Enabled when target `glsl` or `spirv` input is enabled, or when `wgpu_core` is enabled. +//! + +#![no_std] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc(html_logo_url = "https://raw.githubusercontent.com/gfx-rs/wgpu/trunk/logo.png")] +#![warn( + clippy::alloc_instead_of_core, + clippy::allow_attributes, + clippy::std_instead_of_alloc, + clippy::std_instead_of_core, + missing_docs, + rust_2018_idioms, + unsafe_op_in_unsafe_fn +)] +#![allow( + // We need to investiagate these. + clippy::large_enum_variant, + // These degrade readability significantly. + clippy::bool_assert_comparison, + clippy::bool_comparison, +)] +// NOTE: Keep this in sync with `wgpu-core`. +#![cfg_attr(not(send_sync), allow(clippy::arc_with_non_send_sync))] +#![cfg_attr(not(any(wgpu_core, webgpu)), allow(unused))] + +extern crate alloc; +#[cfg(any(std, test))] +extern crate std; +#[cfg(wgpu_core)] +pub extern crate wgpu_core as wgc; +#[cfg(wgpu_core)] +pub extern crate wgpu_hal as hal; +pub extern crate wgpu_types as wgt; + +// +// +// Modules +// +// + +mod api; +mod backend; +mod cmp; +mod dispatch; +mod macros; +pub mod util; + +// +// +// Public re-exports +// +// + +#[cfg(custom)] +pub use backend::custom; + +pub use api::*; +pub use wgt::{ + AdapterInfo, AddressMode, AllocatorReport, AstcBlock, AstcChannel, Backend, BackendOptions, + Backends, BindGroupLayoutEntry, BindingType, BlendComponent, BlendFactor, BlendOperation, + BlendState, BufferAddress, BufferBindingType, BufferSize, BufferTextureCopyInfo, + BufferTransition, BufferUsages, BufferUses, Color, ColorTargetState, ColorWrites, + CommandBufferDescriptor, CompareFunction, CompositeAlphaMode, CooperativeMatrixProperties, + CooperativeScalarType, CopyExternalImageDestInfo, CoreCounters, DepthBiasState, + DepthStencilState, DeviceLostReason, DeviceType, DownlevelCapabilities, DownlevelFlags, + DownlevelLimits, Dx12BackendOptions, Dx12Compiler, Dx12SwapchainKind, + Dx12UseFrameLatencyWaitableObject, DxcShaderModel, DynamicOffset, ExperimentalFeatures, + Extent3d, ExternalTextureFormat, ExternalTextureTransferFunction, Face, Features, FeaturesWGPU, + FeaturesWebGPU, FilterMode, ForceShaderModelToken, FrontFace, GlBackendOptions, GlDebugFns, + GlFenceBehavior, Gles3MinorVersion, HalCounters, ImageSubresourceRange, IndexFormat, + InstanceDescriptor, InstanceFlags, InternalCounters, Limits, LoadOpDontCare, + MemoryBudgetThresholds, MemoryHints, MipmapFilterMode, MultisampleState, NoopBackendOptions, + Origin2d, Origin3d, PipelineStatisticsTypes, PollError, PollStatus, PolygonMode, + PowerPreference, PredefinedColorSpace, PresentMode, PresentationTimestamp, PrimitiveState, + PrimitiveTopology, QueryType, RenderBundleDepthStencil, RequestAdapterError, + SamplerBindingType, SamplerBorderColor, ShaderLocation, ShaderModel, ShaderRuntimeChecks, + ShaderStages, StencilFaceState, StencilOperation, StencilState, StorageTextureAccess, + SurfaceCapabilities, SurfaceStatus, TexelCopyBufferLayout, TextureAspect, TextureChannel, + TextureDimension, TextureFormat, TextureFormatFeatureFlags, TextureFormatFeatures, + TextureSampleType, TextureTransition, TextureUsages, TextureUses, TextureViewDimension, Trace, + VertexAttribute, VertexFormat, VertexStepMode, WasmNotSend, WasmNotSendSync, WasmNotSync, + WriteOnly, WriteOnlyIter, COPY_BUFFER_ALIGNMENT, COPY_BYTES_PER_ROW_ALIGNMENT, + IMMEDIATE_DATA_ALIGNMENT, MAP_ALIGNMENT, MAXIMUM_SUBGROUP_MAX_SIZE, MINIMUM_SUBGROUP_MIN_SIZE, + QUERY_RESOLVE_BUFFER_ALIGNMENT, QUERY_SET_MAX_QUERIES, QUERY_SIZE, VERTEX_ALIGNMENT, +}; + +#[expect(deprecated)] +pub use wgt::VERTEX_STRIDE_ALIGNMENT; + +// wasm-only types, we try to keep as many types non-platform +// specific, but these need to depend on web-sys. +#[cfg(web)] +pub use wgt::{CopyExternalImageSourceInfo, ExternalImageSource}; + +/// Re-export of our `naga` dependency. +/// +#[cfg(wgpu_core)] +#[cfg_attr(docsrs, doc(cfg(any(wgpu_core, naga))))] +// We re-export wgpu-core's re-export of naga, as we may not have direct access to it. +pub use ::wgc::naga; +/// Re-export of our `naga` dependency. +/// +#[cfg(all(not(wgpu_core), naga))] +#[cfg_attr(docsrs, doc(cfg(any(wgpu_core, naga))))] +// If that's not available, we re-export our own. +pub use naga; + +/// Re-export of our `raw-window-handle` dependency. +/// +pub use raw_window_handle as rwh; + +/// Re-export of our `web-sys` dependency. +/// +#[cfg(web)] +pub use web_sys; + +#[doc(hidden)] +pub use macros::helpers as __macro_helpers; diff --git a/third_party/wgpu-29.0.4-patched/src/macros/be-aligned.spv b/third_party/wgpu-29.0.4-patched/src/macros/be-aligned.spv new file mode 100644 index 00000000..0e2d0657 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/macros/be-aligned.spv @@ -0,0 +1 @@ +#"3D \ No newline at end of file diff --git a/third_party/wgpu-29.0.4-patched/src/macros/le-aligned.spv b/third_party/wgpu-29.0.4-patched/src/macros/le-aligned.spv new file mode 100644 index 00000000..3f6aead0 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/macros/le-aligned.spv @@ -0,0 +1 @@ +#D3" \ No newline at end of file diff --git a/third_party/wgpu-29.0.4-patched/src/macros/mod.rs b/third_party/wgpu-29.0.4-patched/src/macros/mod.rs new file mode 100644 index 00000000..90bbeea8 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/macros/mod.rs @@ -0,0 +1,259 @@ +//! Convenience macros + +#[cfg(doc)] +use crate::{VertexAttribute, VertexBufferLayout, VertexFormat}; + +/// Macro to produce an array of [`VertexAttribute`]. +/// +/// The input is a sequence of pairs of shader locations (expression of type [`u32`]) and +/// variant names of [`VertexFormat`]. +/// +/// The return value has type `[VertexAttribute; N]`, where `N` is the number of inputs. +/// +/// Offsets are calculated automatically, +/// using the assumption that there is no padding or other data between attributes. +/// +/// # Example +/// +/// ``` +/// // Suppose that this is our vertex format: +/// #[repr(C, packed)] +/// struct Vertex { +/// foo: [f32; 2], +/// bar: f32, +/// baz: [u16; 4], +/// } +/// +/// // Then these attributes match it: +/// let attrs = wgpu::vertex_attr_array![ +/// 0 => Float32x2, +/// 1 => Float32, +/// 2 => Uint16x4, +/// ]; +/// +/// // Here's the full data structure the macro produced: +/// use wgpu::{VertexAttribute as A, VertexFormat as F}; +/// assert_eq!(attrs, [ +/// A { format: F::Float32x2, offset: 0, shader_location: 0, }, +/// A { format: F::Float32, offset: 8, shader_location: 1, }, +/// A { format: F::Uint16x4, offset: 12, shader_location: 2, }, +/// ]); +/// ``` +/// +/// See [`VertexBufferLayout`] for an example building on this one. +#[macro_export] +macro_rules! vertex_attr_array { + ($($location:expr => $format:ident),* $(,)?) => { + $crate::_vertex_attr_array_helper!([] ; 0; $($location => $format ,)*) + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! _vertex_attr_array_helper { + ([$($t:expr,)*] ; $off:expr ;) => { [$($t,)*] }; + ([$($t:expr,)*] ; $off:expr ; $location:expr => $format:ident, $($ll:expr => $ii:ident ,)*) => { + $crate::_vertex_attr_array_helper!( + [$($t,)* + $crate::VertexAttribute { + format: $crate::VertexFormat :: $format, + offset: $off, + shader_location: $location, + },]; + $off + $crate::VertexFormat :: $format.size(); + $($ll => $ii ,)* + ) + }; +} + +#[test] +fn test_vertex_attr_array() { + let attrs = vertex_attr_array![0 => Float32x2, 3 => Uint16x4]; + // VertexAttribute does not support PartialEq, so we cannot test directly + assert_eq!(attrs.len(), 2); + assert_eq!(attrs[0].offset, 0); + assert_eq!(attrs[0].shader_location, 0); + assert_eq!(attrs[1].offset, size_of::<(f32, f32)>() as u64); + assert_eq!(attrs[1].shader_location, 3); +} + +#[macro_export] +#[doc(hidden)] +macro_rules! include_spirv_source { + ($($token:tt)*) => { + { + const SPIRV_SOURCE: [ + u8; + $crate::__macro_helpers::include_bytes!($($token)*).len() + ] = *$crate::__macro_helpers::include_bytes!($($token)*); + const SPIRV_LEN: usize = SPIRV_SOURCE.len() / 4; + const SPIRV_WORDS: [u32; SPIRV_LEN] = $crate::util::make_spirv_const(SPIRV_SOURCE); + &SPIRV_WORDS + } + } +} + +#[test] +fn make_spirv_le_pass() { + static SPIRV: &[u32] = include_spirv_source!("le-aligned.spv"); + assert_eq!(SPIRV, &[0x07230203, 0x11223344]); +} + +#[test] +fn make_spirv_be_pass() { + static SPIRV: &[u32] = include_spirv_source!("be-aligned.spv"); + assert_eq!(SPIRV, &[0x07230203, 0x11223344]); +} + +/// Macro to load a SPIR-V module statically. +/// +/// It ensures the word alignment as well as the magic number. +/// +/// Return type: [`crate::ShaderModuleDescriptor`] +#[macro_export] +#[cfg(feature = "spirv")] +macro_rules! include_spirv { + ($($token:tt)*) => { + { + $crate::ShaderModuleDescriptor { + label: Some($($token)*), + source: $crate::ShaderSource::SpirV( + $crate::__macro_helpers::Cow::Borrowed($crate::include_spirv_source!($($token)*)) + ), + } + } + }; +} + +#[cfg(all(feature = "spirv", test))] +#[expect(dead_code)] +static SPIRV: crate::ShaderModuleDescriptor<'_> = include_spirv!("le-aligned.spv"); + +/// Macro to load raw SPIR-V data statically, for use with [`Features::PASSTHROUGH_SHADERS`]. +/// +/// It ensures the word alignment as well as the magic number. +/// +/// [`Features::PASSTHROUGH_SHADERS`]: crate::Features::PASSTHROUGH_SHADERS +#[macro_export] +macro_rules! include_spirv_raw { + ($($token:tt)*) => { + { + $crate::ShaderModuleDescriptorPassthrough { + label: $crate::__macro_helpers::Some($($token)*), + spirv: Some($crate::__macro_helpers::Cow::Borrowed($crate::include_spirv_source!($($token)*))), + // This is unused for SPIR-V + num_workgroups: (0, 0, 0), + dxil: None, + metallib: None, + msl: None, + hlsl: None, + glsl: None, + wgsl: None, + } + } + }; +} + +#[cfg(test)] +#[expect(dead_code)] +static SPIRV_RAW: crate::ShaderModuleDescriptorPassthrough<'_> = + include_spirv_raw!("le-aligned.spv"); + +/// Load WGSL source code from a file at compile time. +/// +/// The loaded path is relative to the path of the file containing the macro call, in the same way +/// as [`include_str!`] operates. +/// +/// ```ignore +/// fn main() { +/// let module: ShaderModuleDescriptor = include_wgsl!("shader.wgsl"); +/// } +/// ``` +#[macro_export] +macro_rules! include_wgsl { + ($($token:tt)*) => { + { + $crate::ShaderModuleDescriptor { + label: $crate::__macro_helpers::Some($($token)*), + source: $crate::ShaderSource::Wgsl($crate::__macro_helpers::Cow::Borrowed($crate::__macro_helpers::include_str!($($token)*))), + } + } + }; +} + +// Macros which help us generate the documentation of which hal types correspond to which backend. +// +// Because all backends are not compiled into the program, we cannot link to them in all situations, +// we need to only link to the types if the backend is compiled in. These are used in #[doc] attributes +// so cannot have more than one line, so cannot use internal cfgs. + +/// Helper macro to generate the documentation for dx12 hal methods, referencing the hal type. +#[cfg(dx12)] +macro_rules! hal_type_dx12 { + ($ty: literal) => { + concat!("- [`hal::api::Dx12`] uses [`hal::dx12::", $ty, "`]") + }; +} +/// Helper macro to generate the documentation for dx12 hal methods, referencing the hal type. +#[cfg(not(dx12))] +macro_rules! hal_type_dx12 { + ($ty: literal) => { + concat!("- `hal::api::Dx12` uses `hal::dx12::", $ty, "`") + }; +} +pub(crate) use hal_type_dx12; + +/// Helper macro to generate the documentation for metal hal methods, referencing the hal type. +#[cfg(metal)] +macro_rules! hal_type_metal { + ($ty: literal) => { + concat!("- [`hal::api::Metal`] uses [`hal::metal::", $ty, "`]") + }; +} +/// Helper macro to generate the documentation for metal hal methods, referencing the hal type. +#[cfg(not(metal))] +macro_rules! hal_type_metal { + ($ty: literal) => { + concat!("- `hal::api::Metal` uses `hal::metal::", $ty, "`") + }; +} +pub(crate) use hal_type_metal; + +/// Helper macro to generate the documentation for vulkan hal methods, referencing the hal type. +#[cfg(vulkan)] +macro_rules! hal_type_vulkan { + ($ty: literal) => { + concat!("- [`hal::api::Vulkan`] uses [`hal::vulkan::", $ty, "`]") + }; +} +/// Helper macro to generate the documentation for vulkan hal methods, referencing the hal type. +#[cfg(not(vulkan))] +macro_rules! hal_type_vulkan { + ($ty: literal) => { + concat!("- `hal::api::Vulkan` uses `hal::vulkan::", $ty, "`") + }; +} +pub(crate) use hal_type_vulkan; + +/// Helper macro to generate the documentation for gles hal methods, referencing the hal type. +#[cfg(gles)] +macro_rules! hal_type_gles { + ($ty: literal) => { + concat!("- [`hal::api::Gles`] uses [`hal::gles::", $ty, "`]") + }; +} +/// Helper macro to generate the documentation for gles hal methods, referencing the hal type. +#[cfg(not(gles))] +macro_rules! hal_type_gles { + ($ty: literal) => { + concat!("- `hal::api::Gles` uses `hal::gles::", $ty, "`") + }; +} +pub(crate) use hal_type_gles; + +#[doc(hidden)] +pub mod helpers { + pub use alloc::{borrow::Cow, string::String}; + pub use core::{include_bytes, include_str}; + pub use Some; +} diff --git a/third_party/wgpu-29.0.4-patched/src/util/belt.rs b/third_party/wgpu-29.0.4-patched/src/util/belt.rs new file mode 100644 index 00000000..29093e27 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/belt.rs @@ -0,0 +1,344 @@ +use crate::{ + util::align_to, Buffer, BufferAddress, BufferDescriptor, BufferSize, BufferSlice, BufferUsages, + BufferViewMut, CommandEncoder, Device, MapMode, +}; +use alloc::vec::Vec; +use core::fmt; +use std::sync::mpsc; +use wgt::Features; + +use crate::COPY_BUFFER_ALIGNMENT; + +/// Efficiently performs many buffer writes by sharing and reusing temporary buffers. +/// +/// Internally it uses a ring-buffer of staging buffers that are sub-allocated. +/// Its advantage over [`Queue::write_buffer_with()`] is that the individual allocations +/// are cheaper; `StagingBelt` is most useful when you are writing very many small pieces +/// of data. It can be understood as a sort of arena allocator. +/// +/// Using a staging belt is slightly complicated, and generally goes as follows: +/// 1. Use [`StagingBelt::write_buffer()`] or [`StagingBelt::allocate()`] to allocate +/// buffer slices, then write your data to them. +/// 2. Call [`StagingBelt::finish()`]. +/// 3. Submit all command encoders that were used in step 1. +/// 4. Call [`StagingBelt::recall()`]. +/// +/// [`Queue::write_buffer_with()`]: crate::Queue::write_buffer_with +pub struct StagingBelt { + device: Device, + chunk_size: BufferAddress, + /// User-specified [`BufferUsages`] used to create the chunk buffers are created. + /// + /// [`new`](Self::new) guarantees that this always contains + /// [`MAP_WRITE`](BufferUsages::MAP_WRITE). + buffer_usages: BufferUsages, + /// Chunks into which we are accumulating data to be transferred. + active_chunks: Vec, + /// Chunks that have scheduled transfers already; they are unmapped and some + /// command encoder has one or more commands with them as source. + closed_chunks: Vec, + /// Chunks that are back from the GPU and ready to be mapped for write and put + /// into `active_chunks`. + free_chunks: Vec, + /// When closed chunks are mapped again, the map callback sends them here. + sender: Exclusive>, + /// Free chunks are received here to be put on `self.free_chunks`. + receiver: Exclusive>, +} + +impl StagingBelt { + /// Create a new staging belt. + /// + /// The `chunk_size` is the unit of internal buffer allocation; writes will be + /// sub-allocated within each chunk. Therefore, for optimal use of memory, the + /// chunk size should be: + /// + /// * larger than the largest single [`StagingBelt::write_buffer()`] operation; + /// * 1-4 times less than the total amount of data uploaded per submission + /// (per [`StagingBelt::finish()`]); and + /// * bigger is better, within these bounds. + /// + /// The buffers returned by this [`StagingBelt`] will be have the buffer usages + /// [`COPY_SRC | MAP_WRITE`](crate::BufferUsages) + pub fn new(device: Device, chunk_size: BufferAddress) -> Self { + Self::new_with_buffer_usages(device, chunk_size, BufferUsages::COPY_SRC) + } + + /// Create a new staging belt. + /// + /// The `chunk_size` is the unit of internal buffer allocation; writes will be + /// sub-allocated within each chunk. Therefore, for optimal use of memory, the + /// chunk size should be: + /// + /// * larger than the largest single [`StagingBelt::write_buffer()`] operation; + /// * 1-4 times less than the total amount of data uploaded per submission + /// (per [`StagingBelt::finish()`]); and + /// * bigger is better, within these bounds. + /// + /// `buffer_usages` specifies the [`BufferUsages`] the staging buffers + /// will be created with. [`MAP_WRITE`](BufferUsages::MAP_WRITE) will be added + /// automatically. The method will panic if the combination of usages is not + /// supported. Because [`MAP_WRITE`](BufferUsages::MAP_WRITE) is implied, the allowed usages + /// depends on if [`Features::MAPPABLE_PRIMARY_BUFFERS`] is enabled. + /// - If enabled: any usage is valid. + /// - If disabled: only [`COPY_SRC`](BufferUsages::COPY_SRC) can be used. + #[track_caller] + pub fn new_with_buffer_usages( + device: Device, + chunk_size: BufferAddress, + mut buffer_usages: BufferUsages, + ) -> Self { + let (sender, receiver) = mpsc::channel(); + + // make sure anything other than MAP_WRITE | COPY_SRC is only allowed with MAPPABLE_PRIMARY_BUFFERS. + let extra_usages = + buffer_usages.difference(BufferUsages::MAP_WRITE | BufferUsages::COPY_SRC); + if !extra_usages.is_empty() + && !device + .features() + .contains(Features::MAPPABLE_PRIMARY_BUFFERS) + { + panic!("Only BufferUsages::COPY_SRC may be used when Features::MAPPABLE_PRIMARY_BUFFERS is not enabled. Specified buffer usages: {buffer_usages:?}"); + } + // always set MAP_WRITE + buffer_usages.insert(BufferUsages::MAP_WRITE); + + StagingBelt { + device, + chunk_size, + buffer_usages, + active_chunks: Vec::new(), + closed_chunks: Vec::new(), + free_chunks: Vec::new(), + sender: Exclusive::new(sender), + receiver: Exclusive::new(receiver), + } + } + + /// Allocate a staging belt slice of `size` to be copied into the `target` buffer + /// at the specified offset. + /// + /// `offset` and `size` must be multiples of [`COPY_BUFFER_ALIGNMENT`] + /// (as is required by the underlying buffer operations). + /// + /// The upload will be placed into the provided command encoder. This encoder + /// must be submitted after [`StagingBelt::finish()`] is called and before + /// [`StagingBelt::recall()`] is called. + /// + /// If the `size` is greater than the size of any free internal buffer, a new buffer + /// will be allocated for it. Therefore, the `chunk_size` passed to [`StagingBelt::new()`] + /// should ideally be larger than every such size. + #[track_caller] + pub fn write_buffer( + &mut self, + encoder: &mut CommandEncoder, + target: &Buffer, + offset: BufferAddress, + size: BufferSize, + ) -> BufferViewMut { + // Asserting this explicitly gives a usefully more specific, and more prompt, error than + // leaving it to regular API validation. + // We check only `offset`, not `size`, because `self.allocate()` will check the size. + assert!( + offset.is_multiple_of(COPY_BUFFER_ALIGNMENT), + "StagingBelt::write_buffer() offset {offset} must be a multiple of `COPY_BUFFER_ALIGNMENT`" + ); + + let slice_of_belt = self.allocate( + size, + const { BufferSize::new(crate::COPY_BUFFER_ALIGNMENT).unwrap() }, + ); + encoder.copy_buffer_to_buffer( + slice_of_belt.buffer(), + slice_of_belt.offset(), + target, + offset, + size.get(), + ); + slice_of_belt.get_mapped_range_mut() + } + + /// Allocate a staging belt slice with the given `size` and `alignment` and return it. + /// + /// `size` must be a multiple of [`COPY_BUFFER_ALIGNMENT`] + /// (as is required by the underlying buffer operations). + /// + /// To use this slice, call [`BufferSlice::get_mapped_range_mut()`] and write your data into + /// that [`BufferViewMut`]. + /// (The view must be dropped before [`StagingBelt::finish()`] is called.) + /// + /// You can then record your own GPU commands to perform with the slice, + /// such as copying it to a texture (whereas + /// [`StagingBelt::write_buffer()`] can only write to other buffers). + /// All commands involving this slice must be submitted after + /// [`StagingBelt::finish()`] is called and before [`StagingBelt::recall()`] is called. + /// + /// If the `size` is greater than the space available in any free internal buffer, a new buffer + /// will be allocated for it. Therefore, the `chunk_size` passed to [`StagingBelt::new()`] + /// should ideally be larger than every such size. + /// + /// The chosen slice will be positioned within the buffer at a multiple of `alignment`, + /// which may be used to meet alignment requirements for the operation you wish to perform + /// with the slice. This does not necessarily affect the alignment of the [`BufferViewMut`]. + #[track_caller] + pub fn allocate(&mut self, size: BufferSize, alignment: BufferSize) -> BufferSlice<'_> { + assert!( + size.get().is_multiple_of(COPY_BUFFER_ALIGNMENT), + "StagingBelt allocation size {size} must be a multiple of `COPY_BUFFER_ALIGNMENT`" + ); + assert!( + alignment.get().is_power_of_two(), + "alignment must be a power of two, not {alignment}" + ); + // At minimum, we must have alignment sufficient to map the buffer. + let alignment = alignment.get().max(crate::MAP_ALIGNMENT); + + let mut chunk = if let Some(index) = self + .active_chunks + .iter() + .position(|chunk| chunk.can_allocate(size, alignment)) + { + self.active_chunks.swap_remove(index) + } else { + self.receive_chunks(); // ensure self.free_chunks is up to date + + if let Some(index) = self + .free_chunks + .iter() + .position(|chunk| chunk.can_allocate(size, alignment)) + { + self.free_chunks.swap_remove(index) + } else { + Chunk { + buffer: self.device.create_buffer(&BufferDescriptor { + label: Some("(wgpu internal) StagingBelt staging buffer"), + size: self.chunk_size.max(size.get()), + usage: self.buffer_usages, + mapped_at_creation: true, + }), + offset: 0, + } + } + }; + + let allocation_offset = chunk.allocate(size, alignment); + + self.active_chunks.push(chunk); + let chunk = self.active_chunks.last().unwrap(); + + chunk + .buffer + .slice(allocation_offset..allocation_offset + size.get()) + } + + /// Prepare currently mapped buffers for use in a submission. + /// + /// This must be called before the command encoder(s) provided to + /// [`StagingBelt::write_buffer()`] are submitted. + /// + /// At this point, all the partially used staging buffers are closed (cannot be used for + /// further writes) until after [`StagingBelt::recall()`] is called *and* the GPU is done + /// copying the data from them. + pub fn finish(&mut self) { + for chunk in self.active_chunks.drain(..) { + chunk.buffer.unmap(); + self.closed_chunks.push(chunk); + } + } + + /// Recall all of the closed buffers back to be reused. + /// + /// This must only be called after the command encoder(s) provided to + /// [`StagingBelt::write_buffer()`] are submitted. Additional calls are harmless. + /// Not calling this as soon as possible may result in increased buffer memory usage. + pub fn recall(&mut self) { + self.receive_chunks(); + + for chunk in self.closed_chunks.drain(..) { + let sender = self.sender.get_mut().clone(); + chunk + .buffer + .clone() + .slice(..) + .map_async(MapMode::Write, move |_| { + let _ = sender.send(chunk); + }); + } + } + + /// Move all chunks that the GPU is done with (and are now mapped again) + /// from `self.receiver` to `self.free_chunks`. + fn receive_chunks(&mut self) { + while let Ok(mut chunk) = self.receiver.get_mut().try_recv() { + chunk.offset = 0; + self.free_chunks.push(chunk); + } + } +} + +impl fmt::Debug for StagingBelt { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { + device, + chunk_size, + buffer_usages, + active_chunks, + closed_chunks, + free_chunks, + sender: _, + receiver: _, + } = self; + f.debug_struct("StagingBelt") + .field("device", device) + .field("chunk_size", chunk_size) + .field("buffer_usages", buffer_usages) + .field("active_chunks", &active_chunks.len()) + .field("closed_chunks", &closed_chunks.len()) + .field("free_chunks", &free_chunks.len()) + .finish_non_exhaustive() + } +} + +struct Chunk { + buffer: Buffer, + offset: BufferAddress, +} + +impl Chunk { + fn can_allocate(&self, size: BufferSize, alignment: BufferAddress) -> bool { + let alloc_start = align_to(self.offset, alignment); + let alloc_end = alloc_start + size.get(); + + alloc_end <= self.buffer.size() + } + + fn allocate(&mut self, size: BufferSize, alignment: BufferAddress) -> BufferAddress { + let alloc_start = align_to(self.offset, alignment); + let alloc_end = alloc_start + size.get(); + + assert!(alloc_end <= self.buffer.size()); + self.offset = alloc_end; + alloc_start + } +} + +use exclusive::Exclusive; +mod exclusive { + /// `Sync` wrapper that works by providing only exclusive access. + /// + /// See + pub(super) struct Exclusive(T); + + /// Safety: `&Exclusive` has no operations. + unsafe impl Sync for Exclusive {} + + impl Exclusive { + pub fn new(value: T) -> Self { + Self(value) + } + + pub fn get_mut(&mut self) -> &mut T { + &mut self.0 + } + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/util/blit.wgsl b/third_party/wgpu-29.0.4-patched/src/util/blit.wgsl new file mode 100644 index 00000000..c73845c2 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/blit.wgsl @@ -0,0 +1,30 @@ +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) tex_coords: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vi: u32) -> VertexOutput { + var out: VertexOutput; + + out.tex_coords = vec2( + f32((vi << 1u) & 2u), + f32(vi & 2u), + ); + + out.position = vec4(out.tex_coords * 2.0 - 1.0, 0.0, 1.0); + + // Invert y so the texture is not upside down + out.tex_coords.y = 1.0 - out.tex_coords.y; + return out; +} + +@group(0) @binding(0) +var texture: texture_2d; +@group(0) @binding(1) +var texture_sampler: sampler; + +@fragment +fn fs_main(vs: VertexOutput) -> @location(0) vec4 { + return textureSample(texture, texture_sampler, vs.tex_coords); +} \ No newline at end of file diff --git a/third_party/wgpu-29.0.4-patched/src/util/device.rs b/third_party/wgpu-29.0.4-patched/src/util/device.rs new file mode 100644 index 00000000..540607b8 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/device.rs @@ -0,0 +1,167 @@ +use alloc::borrow::ToOwned as _; + +use wgt::TextureDataOrder; + +/// Describes a [Buffer](crate::Buffer) when allocating. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct BufferInitDescriptor<'a> { + /// Debug label of a buffer. This will show up in graphics debuggers for easy identification. + pub label: crate::Label<'a>, + /// Contents of a buffer on creation. + pub contents: &'a [u8], + /// Usages of a buffer. If the buffer is used in any way that isn't specified here, the operation + /// will panic. + pub usage: wgt::BufferUsages, +} + +/// Utility methods not meant to be in the main API. +pub trait DeviceExt { + /// Creates a [Buffer](crate::Buffer) with data to initialize it. + fn create_buffer_init(&self, desc: &BufferInitDescriptor<'_>) -> crate::Buffer; + + /// Upload an entire texture and its mipmaps from a source buffer. + /// + /// Expects all mipmaps to be tightly packed in the data buffer. + /// + /// See [`TextureDataOrder`] for the order in which the data is laid out in memory. + /// + /// Implicitly adds the `COPY_DST` usage if it is not present in the descriptor, + /// as it is required to be able to upload the data to the gpu. + fn create_texture_with_data( + &self, + queue: &crate::Queue, + desc: &crate::TextureDescriptor<'_>, + order: TextureDataOrder, + data: &[u8], + ) -> crate::Texture; +} + +impl DeviceExt for crate::Device { + fn create_buffer_init(&self, descriptor: &BufferInitDescriptor<'_>) -> crate::Buffer { + // Skip mapping if the buffer is zero sized + if descriptor.contents.is_empty() { + let wgt_descriptor = crate::BufferDescriptor { + label: descriptor.label, + size: 0, + usage: descriptor.usage, + mapped_at_creation: false, + }; + + self.create_buffer(&wgt_descriptor) + } else { + let unpadded_size = descriptor.contents.len() as crate::BufferAddress; + // Valid vulkan usage is + // 1. buffer size must be a multiple of COPY_BUFFER_ALIGNMENT. + // 2. buffer size must be greater than 0. + // Therefore we round the value up to the nearest multiple, and ensure it's at least COPY_BUFFER_ALIGNMENT. + let align_mask = crate::COPY_BUFFER_ALIGNMENT - 1; + let padded_size = + ((unpadded_size + align_mask) & !align_mask).max(crate::COPY_BUFFER_ALIGNMENT); + + let wgt_descriptor = crate::BufferDescriptor { + label: descriptor.label, + size: padded_size, + usage: descriptor.usage, + mapped_at_creation: true, + }; + + let buffer = self.create_buffer(&wgt_descriptor); + + buffer + .get_mapped_range_mut(..) + .slice(..unpadded_size as usize) + .copy_from_slice(descriptor.contents); + buffer.unmap(); + + buffer + } + } + + fn create_texture_with_data( + &self, + queue: &crate::Queue, + desc: &crate::TextureDescriptor<'_>, + order: TextureDataOrder, + data: &[u8], + ) -> crate::Texture { + // Implicitly add the COPY_DST usage + let mut desc = desc.to_owned(); + desc.usage |= crate::TextureUsages::COPY_DST; + let texture = self.create_texture(&desc); + + // Will return None only if it's a combined depth-stencil format + // If so, default to 4, validation will fail later anyway since the depth or stencil + // aspect needs to be written to individually + let block_size = desc.format.block_copy_size(None).unwrap_or(4); + let (block_width, block_height) = desc.format.block_dimensions(); + let layer_iterations = desc.array_layer_count(); + + let outer_iteration; + let inner_iteration; + match order { + TextureDataOrder::LayerMajor => { + outer_iteration = layer_iterations; + inner_iteration = desc.mip_level_count; + } + TextureDataOrder::MipMajor => { + outer_iteration = desc.mip_level_count; + inner_iteration = layer_iterations; + } + } + + let mut binary_offset = 0; + for outer in 0..outer_iteration { + for inner in 0..inner_iteration { + let (layer, mip) = match order { + TextureDataOrder::LayerMajor => (outer, inner), + TextureDataOrder::MipMajor => (inner, outer), + }; + + let mut mip_size = desc.mip_level_size(mip).unwrap(); + // copying layers separately + if desc.dimension != wgt::TextureDimension::D3 { + mip_size.depth_or_array_layers = 1; + } + + // When uploading mips of compressed textures and the mip is supposed to be + // a size that isn't a multiple of the block size, the mip needs to be uploaded + // as its "physical size" which is the size rounded up to the nearest block size. + let mip_physical = mip_size.physical_size(desc.format); + + // All these calculations are performed on the physical size as that's the + // data that exists in the buffer. + let width_blocks = mip_physical.width / block_width; + let height_blocks = mip_physical.height / block_height; + + let bytes_per_row = width_blocks * block_size; + let data_size = bytes_per_row * height_blocks * mip_size.depth_or_array_layers; + + let end_offset = binary_offset + data_size as usize; + + queue.write_texture( + crate::TexelCopyTextureInfo { + texture: &texture, + mip_level: mip, + origin: crate::Origin3d { + x: 0, + y: 0, + z: layer, + }, + aspect: wgt::TextureAspect::All, + }, + &data[binary_offset..end_offset], + crate::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(bytes_per_row), + rows_per_image: Some(height_blocks), + }, + mip_physical, + ); + + binary_offset = end_offset; + } + } + + texture + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/util/encoder.rs b/third_party/wgpu-29.0.4-patched/src/util/encoder.rs new file mode 100644 index 00000000..64d9b6c3 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/encoder.rs @@ -0,0 +1,197 @@ +use core::ops::Range; + +use wgt::{BufferAddress, DynamicOffset, IndexFormat}; + +use crate::{BindGroup, Buffer, BufferSlice, RenderBundleEncoder, RenderPass, RenderPipeline}; + +/// Methods shared by [`RenderPass`] and [`RenderBundleEncoder`]. +pub trait RenderEncoder<'a> { + /// Sets the active bind group for a given bind group index. The bind group layout + /// in the active pipeline when any `draw()` function is called must match the layout of this bind group. + /// + /// If the bind group have dynamic offsets, provide them in order of their declaration. + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&'a BindGroup>, + offsets: &[DynamicOffset], + ); + + /// Sets the active render pipeline. + /// + /// Subsequent draw calls will exhibit the behavior defined by `pipeline`. + fn set_pipeline(&mut self, pipeline: &'a RenderPipeline); + + /// Sets the active index buffer. + /// + /// Subsequent calls to [`draw_indexed`](RenderEncoder::draw_indexed) on this [`RenderEncoder`] will + /// use `buffer` as the source index buffer. + fn set_index_buffer(&mut self, buffer_slice: BufferSlice<'a>, index_format: IndexFormat); + + /// Assign a vertex buffer to a slot. + /// + /// Subsequent calls to [`draw`] and [`draw_indexed`] on this + /// [`RenderEncoder`] will use `buffer` as one of the source vertex buffers. + /// + /// The `slot` refers to the index of the matching descriptor in + /// [`VertexState::buffers`](crate::VertexState::buffers). + /// + /// [`draw`]: RenderEncoder::draw + /// [`draw_indexed`]: RenderEncoder::draw_indexed + fn set_vertex_buffer(&mut self, slot: u32, buffer_slice: BufferSlice<'a>); + + /// Draws primitives from the active vertex buffer(s). + /// + /// The active vertex buffers can be set with [`RenderEncoder::set_vertex_buffer`]. + fn draw(&mut self, vertices: Range, instances: Range); + + /// Draws indexed primitives using the active index buffer and the active vertex buffers. + /// + /// The active index buffer can be set with [`RenderEncoder::set_index_buffer`], while the active + /// vertex buffers can be set with [`RenderEncoder::set_vertex_buffer`]. + fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range); + + /// Draws primitives from the active vertex buffer(s) based on the contents of the `indirect_buffer`. + /// + /// The active vertex buffers can be set with [`RenderEncoder::set_vertex_buffer`]. + /// + /// The structure expected in `indirect_buffer` must conform to [`DrawIndirectArgs`](crate::util::DrawIndirectArgs). + fn draw_indirect(&mut self, indirect_buffer: &'a Buffer, indirect_offset: BufferAddress); + + /// Draws indexed primitives using the active index buffer and the active vertex buffers, + /// based on the contents of the `indirect_buffer`. + /// + /// The active index buffer can be set with [`RenderEncoder::set_index_buffer`], while the active + /// vertex buffers can be set with [`RenderEncoder::set_vertex_buffer`]. + /// + /// The structure expected in `indirect_buffer` must conform to [`DrawIndexedIndirectArgs`](crate::util::DrawIndexedIndirectArgs). + fn draw_indexed_indirect( + &mut self, + indirect_buffer: &'a Buffer, + indirect_offset: BufferAddress, + ); + + /// [`wgt::Features::IMMEDIATES`] must be enabled on the device in order to call this function. + /// + /// Set immediate data for subsequent draw calls. + /// + /// Write the bytes in `data` at offset `offset` within immediate data + /// storage. Both `offset` and the length of `data` must be + /// multiples of [`crate::IMMEDIATE_DATA_ALIGNMENT`], which is always 4. + /// + /// For example, if `offset` is `4` and `data` is eight bytes long, this + /// call will write `data` to bytes `4..12` of immediate data storage. + fn set_immediates(&mut self, offset: u32, data: &[u8]); +} + +impl<'a> RenderEncoder<'a> for RenderPass<'a> { + #[inline(always)] + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&'a BindGroup>, + offsets: &[DynamicOffset], + ) { + Self::set_bind_group(self, index, bind_group, offsets); + } + + #[inline(always)] + fn set_pipeline(&mut self, pipeline: &'a RenderPipeline) { + Self::set_pipeline(self, pipeline); + } + + #[inline(always)] + fn set_index_buffer(&mut self, buffer_slice: BufferSlice<'a>, index_format: IndexFormat) { + Self::set_index_buffer(self, buffer_slice, index_format); + } + + #[inline(always)] + fn set_vertex_buffer(&mut self, slot: u32, buffer_slice: BufferSlice<'a>) { + Self::set_vertex_buffer(self, slot, buffer_slice); + } + + #[inline(always)] + fn draw(&mut self, vertices: Range, instances: Range) { + Self::draw(self, vertices, instances); + } + + #[inline(always)] + fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range) { + Self::draw_indexed(self, indices, base_vertex, instances); + } + + #[inline(always)] + fn draw_indirect(&mut self, indirect_buffer: &'a Buffer, indirect_offset: BufferAddress) { + Self::draw_indirect(self, indirect_buffer, indirect_offset); + } + + #[inline(always)] + fn draw_indexed_indirect( + &mut self, + indirect_buffer: &'a Buffer, + indirect_offset: BufferAddress, + ) { + Self::draw_indexed_indirect(self, indirect_buffer, indirect_offset); + } + + #[inline(always)] + fn set_immediates(&mut self, offset: u32, data: &[u8]) { + Self::set_immediates(self, offset, data); + } +} + +impl<'a> RenderEncoder<'a> for RenderBundleEncoder<'a> { + #[inline(always)] + fn set_bind_group( + &mut self, + index: u32, + bind_group: Option<&'a BindGroup>, + offsets: &[DynamicOffset], + ) { + Self::set_bind_group(self, index, bind_group, offsets); + } + + #[inline(always)] + fn set_pipeline(&mut self, pipeline: &'a RenderPipeline) { + Self::set_pipeline(self, pipeline); + } + + #[inline(always)] + fn set_index_buffer(&mut self, buffer_slice: BufferSlice<'a>, index_format: IndexFormat) { + Self::set_index_buffer(self, buffer_slice, index_format); + } + + #[inline(always)] + fn set_vertex_buffer(&mut self, slot: u32, buffer_slice: BufferSlice<'a>) { + Self::set_vertex_buffer(self, slot, buffer_slice); + } + + #[inline(always)] + fn draw(&mut self, vertices: Range, instances: Range) { + Self::draw(self, vertices, instances); + } + + #[inline(always)] + fn draw_indexed(&mut self, indices: Range, base_vertex: i32, instances: Range) { + Self::draw_indexed(self, indices, base_vertex, instances); + } + + #[inline(always)] + fn draw_indirect(&mut self, indirect_buffer: &'a Buffer, indirect_offset: BufferAddress) { + Self::draw_indirect(self, indirect_buffer, indirect_offset); + } + + #[inline(always)] + fn draw_indexed_indirect( + &mut self, + indirect_buffer: &'a Buffer, + indirect_offset: BufferAddress, + ) { + Self::draw_indexed_indirect(self, indirect_buffer, indirect_offset); + } + + #[inline(always)] + fn set_immediates(&mut self, offset: u32, data: &[u8]) { + Self::set_immediates(self, offset, data); + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/util/init.rs b/third_party/wgpu-29.0.4-patched/src/util/init.rs new file mode 100644 index 00000000..0c3e2b8c --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/init.rs @@ -0,0 +1,128 @@ +use crate::{Adapter, Instance, RequestAdapterOptions, Surface}; + +#[cfg(doc)] +use crate::Backends; + +/// Initialize the adapter obeying the `WGPU_ADAPTER_NAME` environment variable. +#[cfg(wgpu_core)] +#[cfg_attr(not(std), expect(unused_variables, unreachable_code))] +pub async fn initialize_adapter_from_env( + instance: &Instance, + compatible_surface: Option<&Surface<'_>>, +) -> Result { + let desired_adapter_name: alloc::string::String = { + cfg_if::cfg_if! { + if #[cfg(std)] { + std::env::var("WGPU_ADAPTER_NAME") + .as_deref() + .map(str::to_lowercase) + .map_err(|_| wgt::RequestAdapterError::EnvNotSet)? + } else { + return Err(wgt::RequestAdapterError::EnvNotSet) + } + } + }; + + let adapters = instance.enumerate_adapters(crate::Backends::all()).await; + + let mut chosen_adapter = None; + for adapter in adapters { + let info = adapter.get_info(); + + if let Some(surface) = compatible_surface { + if !adapter.is_surface_supported(surface) { + continue; + } + } + + if info.name.to_lowercase().contains(&desired_adapter_name) { + chosen_adapter = Some(adapter); + break; + } + } + + Ok(chosen_adapter.expect("WGPU_ADAPTER_NAME set but no matching adapter found!")) +} + +/// Initialize the adapter obeying the `WGPU_ADAPTER_NAME` environment variable. +#[cfg(not(wgpu_core))] +pub async fn initialize_adapter_from_env( + _instance: &Instance, + _compatible_surface: Option<&Surface<'_>>, +) -> Result { + Err(wgt::RequestAdapterError::EnvNotSet) +} + +/// Initialize the adapter obeying the `WGPU_ADAPTER_NAME` environment variable and if it doesn't exist fall back on a default adapter. +pub async fn initialize_adapter_from_env_or_default( + instance: &Instance, + compatible_surface: Option<&Surface<'_>>, +) -> Result { + match initialize_adapter_from_env(instance, compatible_surface).await { + Ok(a) => Ok(a), + Err(_) => { + instance + .request_adapter(&RequestAdapterOptions { + power_preference: crate::PowerPreference::from_env().unwrap_or_default(), + force_fallback_adapter: false, + compatible_surface, + }) + .await + } + } +} + +/// Determines whether the [`Backends::BROWSER_WEBGPU`] backend is supported. +/// +/// The result can only be true if this is called from the main thread or a dedicated worker. +/// For convenience, this is also supported on non-wasm targets, always returning false there. +pub async fn is_browser_webgpu_supported() -> bool { + #[cfg(webgpu)] + { + // In theory it should be enough to check for the presence of the `gpu` property... + let gpu = crate::backend::get_browser_gpu_property(); + let Ok(Some(gpu)) = gpu else { + return false; + }; + + // ...but in practice, we also have to try to create an adapter, since as of writing + // Chrome on Linux has the `gpu` property but doesn't support WebGPU. + let adapter_promise = gpu.request_adapter(); + wasm_bindgen_futures::JsFuture::from(adapter_promise) + .await + .is_ok_and(|adapter| !adapter.is_undefined() && !adapter.is_null()) + } + #[cfg(not(webgpu))] + { + false + } +} + +/// Create an new instance of wgpu, but disabling [`Backends::BROWSER_WEBGPU`] if no WebGPU support was detected. +/// +/// If the instance descriptor enables [`Backends::BROWSER_WEBGPU`], +/// this checks via [`is_browser_webgpu_supported`] for WebGPU support before forwarding +/// the descriptor with or without [`Backends::BROWSER_WEBGPU`] respectively to [`Instance::new`]. +/// +/// You should prefer this method over [`Instance::new`] if you want to target WebGPU and automatically +/// fall back to WebGL if WebGPU is not available. +/// This is because WebGPU support has to be decided upon instance creation and [`Instance::new`] +/// (being a `sync` function) can't establish WebGPU support (details see [`is_browser_webgpu_supported`]). +/// +/// # Panics +/// +/// If no backend feature for the active target platform is enabled, +/// this method will panic, see [`Instance::enabled_backend_features()`]. +pub async fn new_instance_with_webgpu_detection( + mut instance_desc: wgt::InstanceDescriptor, +) -> crate::Instance { + if instance_desc + .backends + .contains(wgt::Backends::BROWSER_WEBGPU) + && !is_browser_webgpu_supported().await + { + instance_desc.backends.remove(wgt::Backends::BROWSER_WEBGPU); + } + + crate::Instance::new(instance_desc) +} diff --git a/third_party/wgpu-29.0.4-patched/src/util/mod.rs b/third_party/wgpu-29.0.4-patched/src/util/mod.rs new file mode 100644 index 00000000..06658caa --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/mod.rs @@ -0,0 +1,197 @@ +//! Utility structures and functions that are built on top of the main `wgpu` API. +//! +//! Nothing in this module is a part of the WebGPU API specification; +//! they are unique to the `wgpu` library. + +// TODO: For [`belt::StagingBelt`] to be available in `no_std` its usage of [`std::sync::mpsc`] +// must be replaced with an appropriate alternative. +#[cfg(std)] +mod belt; +mod device; +mod encoder; +mod init; +mod mutex; +mod panicking; +mod spirv; +mod texture_blitter; + +use alloc::{format, string::String}; + +#[cfg(std)] +pub use belt::StagingBelt; +pub use device::{BufferInitDescriptor, DeviceExt}; +pub use encoder::RenderEncoder; +pub use init::*; +pub use spirv::*; +#[cfg(feature = "wgsl")] +pub use texture_blitter::{TextureBlitter, TextureBlitterBuilder}; +pub use wgt::{ + math::*, DispatchIndirectArgs, DrawIndexedIndirectArgs, DrawIndirectArgs, TextureDataOrder, +}; + +pub(crate) use mutex::Mutex; +pub(crate) use panicking::is_panicking; + +use crate::dispatch; + +/// CPU accessible buffer used to download data back from the GPU. +pub struct DownloadBuffer { + _gpu_buffer: super::Buffer, + mapped_range: dispatch::DispatchBufferMappedRange, +} + +impl DownloadBuffer { + /// Asynchronously read the contents of a buffer. + pub fn read_buffer( + device: &super::Device, + queue: &super::Queue, + buffer: &super::BufferSlice<'_>, + callback: impl FnOnce(Result) + Send + 'static, + ) { + let size = buffer.size.into(); + + let download = device.create_buffer(&super::BufferDescriptor { + size, + usage: super::BufferUsages::COPY_DST | super::BufferUsages::MAP_READ, + mapped_at_creation: false, + label: None, + }); + + let mut encoder = + device.create_command_encoder(&super::CommandEncoderDescriptor { label: None }); + encoder.copy_buffer_to_buffer(buffer.buffer, buffer.offset, &download, 0, size); + let command_buffer: super::CommandBuffer = encoder.finish(); + queue.submit(Some(command_buffer)); + + download + .clone() + .slice(..) + .map_async(super::MapMode::Read, move |result| { + if let Err(e) = result { + callback(Err(e)); + return; + } + + let mapped_range = download.inner.get_mapped_range(0..size); + callback(Ok(Self { + _gpu_buffer: download, + mapped_range, + })); + }); + } +} + +impl core::ops::Deref for DownloadBuffer { + type Target = [u8]; + fn deref(&self) -> &[u8] { + // SAFETY: `self.mapped_range` is always a read mapping + unsafe { self.mapped_range.read_slice() } + } +} + +/// A recommended key for storing [`PipelineCache`]s for the adapter +/// associated with the given [`AdapterInfo`](wgt::AdapterInfo) +/// This key will define a class of adapters for which the same cache +/// might be valid. +/// +/// If this returns `None`, the adapter doesn't support [`PipelineCache`]. +/// This may be because the API doesn't support application managed caches +/// (such as browser WebGPU), or that `wgpu` hasn't implemented it for +/// that API yet. +/// +/// This key could be used as a filename, as seen in the example below. +/// +/// # Examples +/// +/// ```no_run +/// # use std::path::PathBuf; +/// use wgpu::PipelineCacheDescriptor; +/// # let adapter_info = todo!(); +/// # let device: wgpu::Device = todo!(); +/// let cache_dir: PathBuf = unimplemented!("Some reasonable platform-specific cache directory for your app."); +/// let filename = wgpu::util::pipeline_cache_key(&adapter_info); +/// let (pipeline_cache, cache_file) = if let Some(filename) = filename { +/// let cache_path = cache_dir.join(&filename); +/// // If we failed to read the cache, for whatever reason, treat the data as lost. +/// // In a real app, we'd probably avoid caching entirely unless the error was "file not found". +/// let cache_data = std::fs::read(&cache_path).ok(); +/// let pipeline_cache = unsafe { +/// device.create_pipeline_cache(&PipelineCacheDescriptor { +/// data: cache_data.as_deref(), +/// label: None, +/// fallback: true +/// }) +/// }; +/// (Some(pipeline_cache), Some(cache_path)) +/// } else { +/// (None, None) +/// }; +/// +/// // Run pipeline initialisation, making sure to set the `cache` +/// // fields of your `*PipelineDescriptor` to `pipeline_cache` +/// +/// // And then save the resulting cache (probably off the main thread). +/// if let (Some(pipeline_cache), Some(cache_file)) = (pipeline_cache, cache_file) { +/// let data = pipeline_cache.get_data(); +/// if let Some(data) = data { +/// let temp_file = cache_file.with_extension("temp"); +/// std::fs::write(&temp_file, &data)?; +/// std::fs::rename(&temp_file, &cache_file)?; +/// } +/// } +/// # Ok::<_, std::io::Error>(()) +/// ``` +/// +/// [`PipelineCache`]: super::PipelineCache +pub fn pipeline_cache_key(adapter_info: &wgt::AdapterInfo) -> Option { + match adapter_info.backend { + wgt::Backend::Vulkan => Some(format!( + // The vendor/device should uniquely define a driver + // We/the driver will also later validate that the vendor/device and driver + // version match, which may lead to clearing an outdated + // cache for the same device. + "wgpu_pipeline_cache_vulkan_{}_{}", + adapter_info.vendor, adapter_info.device + )), + _ => None, + } +} + +/// Adds extra conversion functions to `TextureFormat`. +pub trait TextureFormatExt { + /// Finds the [`TextureFormat`](wgt::TextureFormat) corresponding to the given + /// [`StorageFormat`](wgc::naga::StorageFormat). + /// + /// # Examples + /// ``` + /// use wgpu::util::TextureFormatExt; + /// assert_eq!(wgpu::TextureFormat::from_storage_format(wgpu::naga::StorageFormat::Bgra8Unorm), wgpu::TextureFormat::Bgra8Unorm); + /// ``` + #[cfg(wgpu_core)] + fn from_storage_format(storage_format: crate::naga::StorageFormat) -> Self; + + /// Finds the [`StorageFormat`](wgc::naga::StorageFormat) corresponding to the given [`TextureFormat`](wgt::TextureFormat). + /// Returns `None` if there is no matching storage format, + /// which typically indicates this format is not supported + /// for storage textures. + /// + /// # Examples + /// ``` + /// use wgpu::util::TextureFormatExt; + /// assert_eq!(wgpu::TextureFormat::Bgra8Unorm.to_storage_format(), Some(wgpu::naga::StorageFormat::Bgra8Unorm)); + /// ``` + #[cfg(wgpu_core)] + fn to_storage_format(&self) -> Option; +} + +impl TextureFormatExt for wgt::TextureFormat { + #[cfg(wgpu_core)] + fn from_storage_format(storage_format: crate::naga::StorageFormat) -> Self { + wgc::map_storage_format_from_naga(storage_format) + } + + #[cfg(wgpu_core)] + fn to_storage_format(&self) -> Option { + wgc::map_storage_format_to_naga(*self) + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/util/mutex.rs b/third_party/wgpu-29.0.4-patched/src/util/mutex.rs new file mode 100644 index 00000000..86dff533 --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/mutex.rs @@ -0,0 +1,60 @@ +//! Provides a [`Mutex`] for internal use based on what features are available. + +cfg_if::cfg_if! { + if #[cfg(feature = "parking_lot")] { + use parking_lot::Mutex as MutexInner; + } else if #[cfg(std)] { + use std::sync::Mutex as MutexInner; + } else { + use core::cell::RefCell as MutexInner; + } +} + +pub(crate) struct Mutex { + inner: MutexInner, +} + +impl core::fmt::Debug for Mutex +where + MutexInner: core::fmt::Debug, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + as core::fmt::Debug>::fmt(&self.inner, f) + } +} + +impl Default for Mutex { + fn default() -> Self { + Self::new(::default()) + } +} + +impl Mutex { + pub const fn new(value: T) -> Self { + Self { + inner: MutexInner::new(value), + } + } +} + +impl Mutex { + pub fn lock(&self) -> impl core::ops::DerefMut + '_ { + cfg_if::cfg_if! { + if #[cfg(feature = "parking_lot")] { + self.inner.lock() + } else if #[cfg(std)] { + self.inner.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + } else { + loop { + let Ok(lock) = self.inner.try_borrow_mut() else { + // Without `std` all we can do is spin until the current lock is released + core::hint::spin_loop(); + continue; + }; + + break lock; + } + } + } + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/util/panicking.rs b/third_party/wgpu-29.0.4-patched/src/util/panicking.rs new file mode 100644 index 00000000..12ef903b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/panicking.rs @@ -0,0 +1,9 @@ +#[cfg(feature = "std")] +pub fn is_panicking() -> bool { + std::thread::panicking() +} + +#[cfg(not(feature = "std"))] +pub fn is_panicking() -> bool { + false +} diff --git a/third_party/wgpu-29.0.4-patched/src/util/spirv.rs b/third_party/wgpu-29.0.4-patched/src/util/spirv.rs new file mode 100644 index 00000000..882cf17b --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/spirv.rs @@ -0,0 +1,221 @@ +//! Utilities for loading SPIR-V module data. + +use alloc::borrow::Cow; +use core::mem; + +#[cfg_attr(not(any(feature = "spirv", doc)), expect(unused_imports))] +use crate::ShaderSource; + +#[cfg(doc)] +use crate::Device; + +const SPIRV_MAGIC_NUMBER: u32 = 0x0723_0203; + +/// Treat the given byte slice as a SPIR-V module. +/// +/// # Panics +/// +/// This function panics if: +/// +/// - `data.len()` is not a multiple of 4 +/// - `data` does not begin with the SPIR-V magic number +/// +/// It does not check that the data is a valid SPIR-V module in any other way. +#[cfg(feature = "spirv")] // ShaderSource::SpirV only exists in this case +pub fn make_spirv(data: &[u8]) -> ShaderSource<'_> { + ShaderSource::SpirV(make_spirv_raw(data)) +} + +/// Check whether the byte slice has the SPIR-V magic number (in either byte order) and of an +/// appropriate size, and panic with a suitable message when it is not. +/// +/// Returns whether the endianness is opposite of native endianness (i.e. whether +/// [`u32::swap_bytes()`] should be called.) +/// +/// Note: this function’s checks are relied upon for the soundness of [`make_spirv_const()`]. +/// Undefined behavior will result if it does not panic when `bytes.len()` is not a multiple of 4. +#[track_caller] +const fn assert_has_spirv_magic_number_and_length(bytes: &[u8]) -> bool { + // First, check the magic number. + // This way we give the best error for wrong formats. + // (Plus a special case for the empty slice.) + let found_magic_number: Option = match *bytes { + [] => panic!("byte slice is empty, not SPIR-V"), + // This would be simpler as slice::starts_with(), but that isn't a const fn yet. + [b1, b2, b3, b4, ..] => { + let prefix = u32::from_ne_bytes([b1, b2, b3, b4]); + if prefix == SPIRV_MAGIC_NUMBER { + Some(false) + } else if prefix == const { SPIRV_MAGIC_NUMBER.swap_bytes() } { + // needs swapping + Some(true) + } else { + None + } + } + _ => None, // fallthrough case = between 1 and 3 bytes + }; + + match found_magic_number { + Some(needs_byte_swap) => { + // Note: this assertion is relied upon for the soundness of `make_spirv_const()`. + assert!( + bytes.len().is_multiple_of(mem::size_of::()), + "SPIR-V data must be a multiple of 4 bytes long" + ); + + needs_byte_swap + } + None => { + panic!( + "byte slice does not start with SPIR-V magic number. \ + Make sure you are using a binary SPIR-V file." + ); + } + } +} + +#[cfg_attr(not(feature = "spirv"), expect(rustdoc::broken_intra_doc_links))] +/// Version of [`make_spirv()`] intended for use with +/// [`Device::create_shader_module_passthrough()`]. +/// +/// Returns a raw slice instead of [`ShaderSource`]. +/// +/// # Panics +/// +/// This function panics if: +/// +/// - `data.len()` is not a multiple of 4 +/// - `data` does not begin with the SPIR-V magic number +/// +/// It does not check that the data is a valid SPIR-V module in any other way. +pub fn make_spirv_raw(bytes: &[u8]) -> Cow<'_, [u32]> { + let needs_byte_swap = assert_has_spirv_magic_number_and_length(bytes); + + // If the data happens to be aligned, directly use the byte array, + // otherwise copy the byte array in an owned vector and use that instead. + let mut words: Cow<'_, [u32]> = match bytemuck::try_cast_slice(bytes) { + Ok(words) => Cow::Borrowed(words), + // We already checked the length, so if this fails, it fails due to lack of alignment only. + Err(_) => Cow::Owned(bytemuck::pod_collect_to_vec(bytes)), + }; + + // If necessary, swap bytes to native endianness. + if needs_byte_swap { + for word in Cow::to_mut(&mut words) { + *word = word.swap_bytes(); + } + } + + assert!( + words[0] == SPIRV_MAGIC_NUMBER, + "can't happen: wrong magic number after swap_bytes" + ); + words +} + +/// Version of `make_spirv_raw` used for implementing [`include_spirv!`] and [`include_spirv_raw!`] macros. +/// +/// Not public API. Also, don't even try calling at runtime; you'll get a stack overflow. +/// +/// [`include_spirv!`]: crate::include_spirv +#[doc(hidden)] +pub const fn make_spirv_const(bytes: [u8; IN]) -> [u32; OUT] { + let needs_byte_swap = assert_has_spirv_magic_number_and_length(&bytes); + + // NOTE: to get around lack of generic const expressions, the input and output lengths must + // be specified separately. + // Check that they are consistent with each other. + assert!(mem::size_of_val(&bytes) == mem::size_of::<[u32; OUT]>()); + + // Can't use `bytemuck` in `const fn` (yet), so do it unsafely. + // SAFETY: + // * The previous assertion checked that the byte sizes of `bytes` and `words` are equal. + // * `transmute_copy` doesn't care that the alignment might be wrong. + let mut words: [u32; OUT] = unsafe { mem::transmute_copy(&bytes) }; + + // If necessary, swap bytes to native endianness. + if needs_byte_swap { + let mut idx = 0; + while idx < words.len() { + words[idx] = words[idx].swap_bytes(); + idx += 1; + } + } + + assert!( + words[0] == SPIRV_MAGIC_NUMBER, + "can't happen: wrong magic number after swap_bytes" + ); + + words +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + + fn test_success_with_misalignments( + input: &[u8; IN], + expected: [u32; OUT], + ) { + // We don't know which 3 out of 4 offsets will produce an actually misaligned slice, + // but they always will. (Note that it is necessary to reuse the same allocation for all 4 + // tests, or we could (in theory) get unlucky and not test any misalignments.) + let mut buffer = vec![0; input.len() + 4]; + for offset in 0..4 { + let misaligned_slice: &mut [u8; IN] = + (&mut buffer[offset..][..input.len()]).try_into().unwrap(); + + misaligned_slice.copy_from_slice(input); + assert_eq!(*make_spirv_raw(misaligned_slice), expected); + assert_eq!(make_spirv_const(*misaligned_slice), expected); + } + } + + #[test] + fn success_be() { + // magic number followed by dummy data to see the endianness handling + let input = b"\x07\x23\x02\x03\xF1\xF2\xF3\xF4"; + let expected: [u32; 2] = [SPIRV_MAGIC_NUMBER, 0xF1F2F3F4]; + test_success_with_misalignments(input, expected); + } + + #[test] + fn success_le() { + let input = b"\x03\x02\x23\x07\xF1\xF2\xF3\xF4"; + let expected: [u32; 2] = [SPIRV_MAGIC_NUMBER, 0xF4F3F2F1]; + test_success_with_misalignments(input, expected); + } + + #[should_panic = "multiple of 4"] + #[test] + fn nonconst_le_fail() { + let _: Cow<'_, [u32]> = make_spirv_raw(&[0x03, 0x02, 0x23, 0x07, 0x44, 0x33]); + } + + #[should_panic = "multiple of 4"] + #[test] + fn nonconst_be_fail() { + let _: Cow<'_, [u32]> = make_spirv_raw(&[0x07, 0x23, 0x02, 0x03, 0x11, 0x22]); + } + + #[should_panic = "multiple of 4"] + #[test] + fn const_le_fail() { + let _: [u32; 1] = make_spirv_const([0x03, 0x02, 0x23, 0x07, 0x44, 0x33]); + } + + #[should_panic = "multiple of 4"] + #[test] + fn const_be_fail() { + let _: [u32; 1] = make_spirv_const([0x07, 0x23, 0x02, 0x03, 0x11, 0x22]); + } + + #[should_panic = "byte slice is empty, not SPIR-V"] + #[test] + fn make_spirv_empty() { + let _: [u32; 0] = make_spirv_const([]); + } +} diff --git a/third_party/wgpu-29.0.4-patched/src/util/texture_blitter.rs b/third_party/wgpu-29.0.4-patched/src/util/texture_blitter.rs new file mode 100644 index 00000000..a57376dc --- /dev/null +++ b/third_party/wgpu-29.0.4-patched/src/util/texture_blitter.rs @@ -0,0 +1,218 @@ +#![cfg(feature = "wgsl")] + +use wgt::BlendState; + +use crate::{ + include_wgsl, AddressMode, BindGroupDescriptor, BindGroupEntry, BindGroupLayout, + BindGroupLayoutDescriptor, BindGroupLayoutEntry, BindingType, ColorTargetState, ColorWrites, + CommandEncoder, Device, FilterMode, FragmentState, FrontFace, LoadOp, MultisampleState, + PipelineCompilationOptions, PipelineLayoutDescriptor, PrimitiveState, PrimitiveTopology, + RenderPassDescriptor, RenderPipeline, RenderPipelineDescriptor, Sampler, SamplerBindingType, + SamplerDescriptor, ShaderStages, StoreOp, TextureFormat, TextureSampleType, TextureView, + TextureViewDimension, VertexState, +}; + +/// A builder for the [`TextureBlitter`] utility. +/// If you want the default [`TextureBlitter`] use [`TextureBlitter::new`] instead. +pub struct TextureBlitterBuilder<'a> { + device: &'a Device, + format: TextureFormat, + sample_type: FilterMode, + blend_state: Option, +} + +impl<'a> TextureBlitterBuilder<'a> { + /// Returns a new [`TextureBlitterBuilder`] + /// + /// # Arguments + /// - `device` - A [`Device`] + /// - `format` - The [`TextureFormat`] of the texture that will be copied to. This has to have the `RENDER_TARGET` usage. + pub fn new(device: &'a Device, format: TextureFormat) -> Self { + Self { + device, + format, + sample_type: FilterMode::Nearest, + blend_state: None, + } + } + + /// Sets the [`Sampler`] Filtering Mode + pub fn sample_type(mut self, sample_type: FilterMode) -> Self { + self.sample_type = sample_type; + self + } + + /// Sets the [`BlendState`] that is used. + pub fn blend_state(mut self, blend_state: BlendState) -> Self { + self.blend_state = Some(blend_state); + self + } + + /// Returns a new [`TextureBlitter`] with given settings. + pub fn build(self) -> TextureBlitter { + let sampler = self.device.create_sampler(&SamplerDescriptor { + label: Some("wgpu::util::TextureBlitter::sampler"), + address_mode_u: AddressMode::ClampToEdge, + address_mode_v: AddressMode::ClampToEdge, + address_mode_w: AddressMode::ClampToEdge, + mag_filter: self.sample_type, + ..Default::default() + }); + + let bind_group_layout = self + .device + .create_bind_group_layout(&BindGroupLayoutDescriptor { + label: Some("wgpu::util::TextureBlitter::bind_group_layout"), + entries: &[ + BindGroupLayoutEntry { + binding: 0, + visibility: ShaderStages::FRAGMENT, + ty: BindingType::Texture { + sample_type: TextureSampleType::Float { + filterable: self.sample_type == FilterMode::Linear, + }, + view_dimension: TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + BindGroupLayoutEntry { + binding: 1, + visibility: ShaderStages::FRAGMENT, + ty: BindingType::Sampler(if self.sample_type == FilterMode::Linear { + SamplerBindingType::Filtering + } else { + SamplerBindingType::NonFiltering + }), + count: None, + }, + ], + }); + + let pipeline_layout = self + .device + .create_pipeline_layout(&PipelineLayoutDescriptor { + label: Some("wgpu::util::TextureBlitter::pipeline_layout"), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + + let shader = self.device.create_shader_module(include_wgsl!("blit.wgsl")); + let pipeline = self + .device + .create_render_pipeline(&RenderPipelineDescriptor { + label: Some("wgpu::util::TextureBlitter::pipeline"), + layout: Some(&pipeline_layout), + vertex: VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: PipelineCompilationOptions::default(), + buffers: &[], + }, + primitive: PrimitiveState { + topology: PrimitiveTopology::TriangleList, + strip_index_format: None, + front_face: FrontFace::Ccw, + cull_mode: None, + unclipped_depth: false, + polygon_mode: wgt::PolygonMode::Fill, + conservative: false, + }, + depth_stencil: None, + multisample: MultisampleState::default(), + fragment: Some(FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: PipelineCompilationOptions::default(), + targets: &[Some(ColorTargetState { + format: self.format, + blend: self.blend_state, + write_mask: ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + + TextureBlitter { + pipeline, + bind_group_layout, + sampler, + } + } +} + +/// Texture Blitting (Copying) Utility +/// +/// Use this if you want to just render/copy texture A to texture B where [`CommandEncoder::copy_texture_to_texture`] would not work because: +/// - Textures are in incompatible formats. +/// - Textures are of different sizes. +/// - Your copy destination is the surface texture and does not have the `COPY_DST` usage. +pub struct TextureBlitter { + pipeline: RenderPipeline, + bind_group_layout: BindGroupLayout, + sampler: Sampler, +} + +impl TextureBlitter { + /// Returns a [`TextureBlitter`] with default settings. + /// + /// # Arguments + /// - `device` - A [`Device`] + /// - `format` - The [`TextureFormat`] of the texture that will be copied to. This has to have the `RENDER_TARGET` usage. + /// + /// Properties of the blitting (such as the [`BlendState`]) can be customised by using [`TextureBlitterBuilder`] instead. + pub fn new(device: &Device, format: TextureFormat) -> Self { + TextureBlitterBuilder::new(device, format).build() + } + + /// Copies the data from the source [`TextureView`] to the target [`TextureView`] + /// + /// # Arguments + /// - `device` - A [`Device`] + /// - `encoder` - A [`CommandEncoder`] + /// - `source` - A [`TextureView`] that gets copied. The format does not matter. + /// - `target` - A [`TextureView`] that gets the data copied from the `source`. It has to be the same format as the format specified in [`TextureBlitter::new`] + pub fn copy( + &self, + device: &Device, + encoder: &mut CommandEncoder, + source: &TextureView, + target: &TextureView, + ) { + let bind_group = device.create_bind_group(&BindGroupDescriptor { + label: Some("wgpu::util::TextureBlitter::bind_group"), + layout: &self.bind_group_layout, + entries: &[ + BindGroupEntry { + binding: 0, + resource: crate::BindingResource::TextureView(source), + }, + BindGroupEntry { + binding: 1, + resource: crate::BindingResource::Sampler(&self.sampler), + }, + ], + }); + + let mut pass = encoder.begin_render_pass(&RenderPassDescriptor { + label: Some("wgpu::util::TextureBlitter::pass"), + color_attachments: &[Some(crate::RenderPassColorAttachment { + view: target, + depth_slice: None, + resolve_target: None, + ops: wgt::Operations { + load: LoadOp::Load, + store: StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, &bind_group, &[]); + pass.draw(0..3, 0..1); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/.cargo-ok b/third_party/wgpu-core-29.0.4-patched/.cargo-ok new file mode 100644 index 00000000..5f8b7958 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/third_party/wgpu-core-29.0.4-patched/.cargo_vcs_info.json b/third_party/wgpu-core-29.0.4-patched/.cargo_vcs_info.json new file mode 100644 index 00000000..34ff129d --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "e99f5305ded96ff7006f0714d043a7f735bd45c2" + }, + "path_in_vcs": "wgpu-core" +} \ No newline at end of file diff --git a/third_party/wgpu-core-29.0.4-patched/Cargo.lock b/third_party/wgpu-core-29.0.4-patched/Cargo.lock new file mode 100644 index 00000000..1724d642 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/Cargo.lock @@ -0,0 +1,1165 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +dependencies = [ + "serde", +] + +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bit-set" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ddef2995421ab6a5c779542c81ee77c115206f4ad9d5a8e05f4ff49716a3dd" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +dependencies = [ + "serde_core", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "objc2", +] + +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + +[[package]] +name = "glow" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29038e1c483364cc6bb3cf78feee1816002e127c331a1eec55a4d202b9e1adb5" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "gpu-allocator" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795" +dependencies = [ + "ash", + "hashbrown 0.16.1", + "log", + "presser", + "thiserror", + "windows", +] + +[[package]] +name = "gpu-descriptor" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" +dependencies = [ + "bitflags", + "gpu-descriptor-types", + "hashbrown 0.15.5", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91" +dependencies = [ + "bitflags", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "serde", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "mach-dxcompiler-rs" +version = "0.1.4+2024.11.22-df583a3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e3cd67e8ea2ba061339150970542cf1c60ba44c6d17e31279cbc133a4b018f8" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + +[[package]] +name = "naga" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2bf919621e7975acb27d881bae2fb993e0d45c8e0446e85e6272971e00dc8df" +dependencies = [ + "arrayvec", + "bit-set", + "bitflags", + "cfg-if", + "cfg_aliases", + "codespan-reporting", + "half", + "hashbrown 0.16.1", + "hexf-parse", + "indexmap", + "libm", + "log", + "num-traits", + "once_cell", + "petgraph", + "pp-rs", + "rustc-hash", + "serde", + "spirv", + "thiserror", + "unicode-ident", +] + +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags", + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" +dependencies = [ + "bitflags", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-metal", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ordered-float" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "pp-rs" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb458bb7f6e250e6eb79d5026badc10a3ebb8f9a15d1fff0f13d17c71f4d6dee" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "profiling" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "range-alloc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" + +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + +[[package]] +name = "raw-window-metal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d213455a5f1dc59214213c7330e074ddf8114c9a42411eb890c767357ce135" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-quartz-core", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + +[[package]] +name = "ron" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd490c5b18261893f14449cbd28cb9c0b637aebf161cd77900bfdedaff21ec32" +dependencies = [ + "bitflags", + "once_cell", + "serde", + "serde_derive", + "typeid", + "unicode-ident", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "spirv" +version = "0.4.0+sdk-1.4.341.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9571ea910ebd84c86af4b3ed27f9dbdc6ad06f17c5f96146b2b671e2976744f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wayland-sys" +version = "0.31.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374f6b70e8e0d6bf9461a32988fd553b59ff630964924dad6e4a4eb6bd538d17" +dependencies = [ + "dlib", + "log", + "once_cell", + "pkg-config", +] + +[[package]] +name = "web-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wgpu-core" +version = "29.0.4" +dependencies = [ + "arrayvec", + "bit-set", + "bit-vec", + "bitflags", + "bytemuck", + "cfg_aliases", + "document-features", + "hashbrown 0.16.1", + "indexmap", + "log", + "macro_rules_attribute", + "naga", + "once_cell", + "parking_lot", + "portable-atomic", + "profiling", + "raw-window-handle", + "ron", + "rustc-hash", + "serde", + "smallvec", + "thiserror", + "wgpu-core-deps-apple", + "wgpu-core-deps-emscripten", + "wgpu-core-deps-wasm", + "wgpu-core-deps-windows-linux-android", + "wgpu-hal", + "wgpu-naga-bridge", + "wgpu-types", +] + +[[package]] +name = "wgpu-core-deps-apple" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e39e26c4c0e07589e67d18546cf79ff45383659fc72fca4dd293358a0347f3" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-emscripten" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e09be551dc939498bdd5f6b2c66e55ab275dad25825267a08605a80fc9f0af" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-wasm" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1fb1798be2a912497d4c224f72d39bb0cb34af50e8bcc29865bc339c943059" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-windows-linux-android" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e592c1bbef6ad047647ae6e666ebd8cee7a32bb4544d9700ec96cbf73230257" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-hal" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d" +dependencies = [ + "android_system_properties", + "arrayvec", + "ash", + "bit-set", + "bitflags", + "block2", + "bytemuck", + "cfg-if", + "cfg_aliases", + "glow", + "glutin_wgl_sys", + "gpu-allocator", + "gpu-descriptor", + "hashbrown 0.16.1", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "mach-dxcompiler-rs", + "naga", + "ndk-sys", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-metal", + "objc2-quartz-core", + "once_cell", + "ordered-float", + "parking_lot", + "portable-atomic", + "portable-atomic-util", + "profiling", + "range-alloc", + "raw-window-handle", + "raw-window-metal", + "renderdoc-sys", + "smallvec", + "thiserror", + "wasm-bindgen", + "wayland-sys", + "web-sys", + "wgpu-naga-bridge", + "wgpu-types", + "windows", + "windows-core", + "windows-result", +] + +[[package]] +name = "wgpu-naga-bridge" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95226013f547544b223281cd16a4fb549aa9dcb562adbda0faae4c73ffbbc161" +dependencies = [ + "naga", + "wgpu-types", +] + +[[package]] +name = "wgpu-types" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bf84cd9ca8ca45e2b223a3868f1adf9bfc0c66aeac212e76ee7e40fdadf8f5" +dependencies = [ + "bitflags", + "bytemuck", + "js-sys", + "log", + "raw-window-handle", + "serde", + "web-sys", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "zerocopy" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/third_party/wgpu-core-29.0.4-patched/Cargo.toml b/third_party/wgpu-core-29.0.4-patched/Cargo.toml new file mode 100644 index 00000000..94284eb6 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/Cargo.toml @@ -0,0 +1,230 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.87" +name = "wgpu-core" +version = "29.0.4" +authors = ["gfx-rs developers"] +build = "build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Core implementation logic of wgpu, the cross-platform, safe, pure-rust graphics API" +homepage = "https://wgpu.rs/" +readme = false +keywords = ["graphics"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/gfx-rs/wgpu" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs", +] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "wasm32-unknown-unknown", +] + +[package.metadata.cargo-machete] +ignored = ["cfg_aliases"] + +[features] +angle = ["wgpu-core-deps-apple/angle"] +api_log_info = [] +counters = ["wgpu-types/counters"] +default = ["std"] +dx12 = ["wgpu-core-deps-windows-linux-android/dx12"] +fragile-send-sync-non-atomic-wasm = ["wgpu-hal/fragile-send-sync-non-atomic-wasm"] +gles = [ + "wgpu-core-deps-windows-linux-android/gles", + "wgpu-core-deps-emscripten/gles", +] +glsl = ["naga/glsl-in"] +metal = ["wgpu-core-deps-apple/metal"] +noop = [] +observe_locks = [ + "std", + "dep:ron", + "serde/serde_derive", +] +portable-atomic = [ + "dep:portable-atomic", + "wgpu-hal/portable-atomic", +] +renderdoc = ["wgpu-core-deps-windows-linux-android/renderdoc"] +replay = [ + "serde", + "naga/deserialize", +] +resource_log_info = [] +serde = [ + "dep:serde", + "wgpu-types/serde", + "arrayvec/serde", + "hashbrown/serde", + "smallvec/serde", + "macro_rules_attribute", +] +spirv = ["naga/spv-in"] +static-dxc = ["wgpu-hal/static-dxc"] +std = [] +strict_asserts = ["wgpu-types/strict_asserts"] +trace = [ + "serde", + "std", + "dep:ron", + "naga/serialize", + "wgpu-types/trace", +] +vulkan = ["wgpu-core-deps-windows-linux-android/vulkan"] +vulkan-portability = ["wgpu-core-deps-apple/vulkan-portability"] +webgl = [ + "wgpu-core-deps-wasm/webgl", + "wgpu-types/web", +] +wgsl = ["naga/wgsl-in"] + +[lib] +name = "wgpu_core" +path = "src/lib.rs" + +[dependencies.arrayvec] +version = "0.7.1" +default-features = false + +[dependencies.bit-set] +version = "0.9" +default-features = false + +[dependencies.bit-vec] +version = "0.9" +default-features = false + +[dependencies.bitflags] +version = "2.9" + +[dependencies.bytemuck] +version = "1.22" +features = [ + "extern_crate_alloc", + "min_const_generics", +] + +[dependencies.document-features] +version = "0.2.10" + +[dependencies.hashbrown] +version = "0.16" +features = [ + "default-hasher", + "inline-more", +] +default-features = false + +[dependencies.indexmap] +version = "2.11.4" +default-features = false + +[dependencies.log] +version = "0.4.29" + +[dependencies.macro_rules_attribute] +version = "0.2" +optional = true + +[dependencies.naga] +version = "29.0.1" + +[dependencies.once_cell] +version = "1.21" +features = ["std"] +default-features = false + +[dependencies.parking_lot] +version = "0.12.3" + +[dependencies.profiling] +version = "1.0.1" +default-features = false + +[dependencies.raw-window-handle] +version = "0.6.2" +default-features = false + +[dependencies.ron] +version = "0.12" +optional = true + +[dependencies.rustc-hash] +version = "1.1" +default-features = false + +[dependencies.serde] +version = "1.0.225" +features = [ + "default", + "derive", +] +optional = true +default-features = false + +[dependencies.smallvec] +version = "1.14" + +[dependencies.thiserror] +version = "2.0.12" +default-features = false + +[dependencies.wgpu-hal] +version = "29.0.1" + +[dependencies.wgpu-naga-bridge] +version = "29.0.1" + +[dependencies.wgpu-types] +version = "29.0.1" +default-features = false + +[build-dependencies.cfg_aliases] +version = "0.2.1" + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.wgpu-core-deps-wasm] +version = "29.0.0" +optional = true + +[target.'cfg(any(windows, target_os = "linux", target_os = "android", target_os = "freebsd"))'.dependencies.wgpu-core-deps-windows-linux-android] +version = "29.0.0" +optional = true + +[target.'cfg(not(target_has_atomic = "64"))'.dependencies.portable-atomic] +version = "1.10" +optional = true + +[target.'cfg(target_os = "emscripten")'.dependencies.wgpu-core-deps-emscripten] +version = "29.0.0" +optional = true + +[target.'cfg(target_vendor = "apple")'.dependencies.wgpu-core-deps-apple] +version = "29.0.0" +optional = true + +[lints.rust.unexpected_cfgs] +level = "warn" +priority = 0 +check-cfg = ["cfg(wgpu_validate_locks)"] diff --git a/third_party/wgpu-core-29.0.4-patched/Cargo.toml.orig b/third_party/wgpu-core-29.0.4-patched/Cargo.toml.orig new file mode 100644 index 00000000..9fa3daee --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/Cargo.toml.orig @@ -0,0 +1,198 @@ +[package] +name = "wgpu-core" +version.workspace = true +authors.workspace = true +edition.workspace = true +description = "Core implementation logic of wgpu, the cross-platform, safe, pure-rust graphics API" +homepage.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true + +# Override the workspace's `rust-version` key. `wgpu-core` and its dependencies +# have a less strict MSRV, to allow firefox more leeway in updating their Rust toolchain. +# +# See the repo README for more information on MSRV policy. +rust-version = "1.87" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "wasm32-unknown-unknown", +] + +[package.metadata.cargo-machete] +# Cargo machete can't check build.rs dependencies. See https://github.com/bnjbvr/cargo-machete/issues/100 +ignored = ["cfg_aliases"] + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(wgpu_validate_locks)'] } + +[lib] + +[features] +#! See documentation for the `wgpu` crate for more in-depth information on these features. + +# TODO(https://github.com/gfx-rs/wgpu/issues/6826): "std" is a default feature for +# compatibility with prior behavior only, and should be removed once we know how +# wgpu-core’s dependents want to handle no_std. +default = ["std"] + +#! ### Logging Configuration +# -------------------------------------------------------------------- + +## Log all API entry points at info instead of trace level. +## Also, promotes certain debug log calls to info. +api_log_info = [] + +## Log resource lifecycle management at info instead of trace level. +resource_log_info = [] + +#! ### Runtime Checks +# -------------------------------------------------------------------- + +## Apply run-time checks, even in release builds. These are in addition +## to the validation carried out at public APIs in all builds. +strict_asserts = ["wgpu-types/strict_asserts"] + +#! ### Debugging +# -------------------------------------------------------------------- + +## Enable lock order observation. +observe_locks = ["std", "dep:ron", "serde/serde_derive"] + +#! ### Serialization +# -------------------------------------------------------------------- + +## Enables serialization via `serde` on common wgpu types. +serde = [ + "dep:serde", + "wgpu-types/serde", + "arrayvec/serde", + "hashbrown/serde", + "smallvec/serde", + "macro_rules_attribute", +] + +## Enable API tracing. +trace = ["serde", "std", "dep:ron", "naga/serialize", "wgpu-types/trace"] + +## Enable API replaying +replay = ["serde", "naga/deserialize"] + +#! ### Shading Language Support +# -------------------------------------------------------------------- + +## Enable `ShaderModuleSource::Wgsl` +wgsl = ["naga/wgsl-in"] + +## Enable `ShaderModuleSource::Glsl` +glsl = ["naga/glsl-in"] + +## Enable `ShaderModuleSource::SpirV` +spirv = ["naga/spv-in"] + +#! ### Other +# -------------------------------------------------------------------- + +## Internally count resources and events for debugging purposes. If the counters +## feature is disabled, the counting infrastructure is removed from the build and +## the exposed counters always return 0. +counters = ["wgpu-types/counters"] + +## Implement `Send` and `Sync` on Wasm, but only if atomics are not enabled. +fragile-send-sync-non-atomic-wasm = [ + "wgpu-hal/fragile-send-sync-non-atomic-wasm", +] + +## Enable certain items to be `Send` and `Sync` when they would not otherwise be. +## Also enables backtraces in some error cases when also under cfg(debug_assertions). +std = [] + +#! ### External libraries +# -------------------------------------------------------------------- +#! The following features facilitate integration with third-party supporting libraries. + +## Enable using the `mach-dxcompiler-rs` crate to compile DX12 shaders. +static-dxc = ["wgpu-hal/static-dxc"] + +## Enable portable atomics on platforms that do not support 64bit atomics. +portable-atomic = ["dep:portable-atomic", "wgpu-hal/portable-atomic"] + +#! ### Target Conditional Features +# -------------------------------------------------------------------- +# Look to wgpu-hal's Cargo.toml for explaination how these features and the wgpu-core +# platform crates collude to provide platform-specific behavior. + +## DX12 backend +dx12 = ["wgpu-core-deps-windows-linux-android/dx12"] +## Metal backend +metal = ["wgpu-core-deps-apple/metal"] +## Vulkan backend, only available on Windows, Linux, Android +vulkan = ["wgpu-core-deps-windows-linux-android/vulkan"] +## OpenGL backend, only available on Windows, Linux, Android, and Emscripten +gles = [ + "wgpu-core-deps-windows-linux-android/gles", + "wgpu-core-deps-emscripten/gles", +] + +## WebGL backend, only available on Emscripten +webgl = ["wgpu-core-deps-wasm/webgl", "wgpu-types/web"] +## OpenGL backend, on macOS only +angle = ["wgpu-core-deps-apple/angle"] +## Vulkan portability backend, only available on macOS +vulkan-portability = ["wgpu-core-deps-apple/vulkan-portability"] +## Renderdoc integration, only available on Windows, Linux, and Android +renderdoc = ["wgpu-core-deps-windows-linux-android/renderdoc"] + +## Enable the `noop` backend. +# TODO(https://github.com/gfx-rs/wgpu/issues/7120): there should be a hal feature +noop = [] + +# The target limitation here isn't needed, but prevents more than one of these +# platform crates from being included in the build at a time, preventing users +# from getting confused by seeing them in the list of crates. +[target.'cfg(target_vendor = "apple")'.dependencies] +wgpu-core-deps-apple = { workspace = true, optional = true } +[target.'cfg(target_os = "emscripten")'.dependencies] +wgpu-core-deps-emscripten = { workspace = true, optional = true } +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies] +wgpu-core-deps-wasm = { workspace = true, optional = true } +[target.'cfg(any(windows, target_os = "linux", target_os = "android", target_os = "freebsd"))'.dependencies] +wgpu-core-deps-windows-linux-android = { workspace = true, optional = true } + +[dependencies] +naga.workspace = true +wgpu-naga-bridge.workspace = true +wgpu-hal.workspace = true +wgpu-types.workspace = true + +arrayvec.workspace = true +bit-vec.workspace = true +bit-set.workspace = true +bitflags.workspace = true +bytemuck.workspace = true +document-features.workspace = true +hashbrown.workspace = true +indexmap.workspace = true +log.workspace = true +macro_rules_attribute = { workspace = true, optional = true } +once_cell = { workspace = true, features = ["std"] } +parking_lot.workspace = true +profiling = { workspace = true, default-features = false } +raw-window-handle.workspace = true +ron = { workspace = true, optional = true } +rustc-hash.workspace = true +serde = { workspace = true, features = ["default", "derive"], optional = true } +smallvec.workspace = true +thiserror.workspace = true + +[target.'cfg(not(target_has_atomic = "64"))'.dependencies] +portable-atomic = { workspace = true, optional = true } + +[build-dependencies] +cfg_aliases.workspace = true diff --git a/third_party/wgpu-core-29.0.4-patched/LICENSE.APACHE b/third_party/wgpu-core-29.0.4-patched/LICENSE.APACHE new file mode 100644 index 00000000..d9a10c0d --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/LICENSE.APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/third_party/wgpu-core-29.0.4-patched/LICENSE.MIT b/third_party/wgpu-core-29.0.4-patched/LICENSE.MIT new file mode 100644 index 00000000..8d02e4db --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/LICENSE.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 The gfx-rs developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md b/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md new file mode 100644 index 00000000..2773794b --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md @@ -0,0 +1,15 @@ +# Patch provenance + +Source: `wgpu-core` 29.0.4 from crates.io, checksum +`2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7`. + +Local change: add the hidden core half of the audited external-write handoff. +It validates that the requested byte range belongs to a live buffer and drains +only that range from wgpu's lazy buffer-initialization tracker. It does not +submit GPU work, change resource state, or establish synchronization; those +obligations remain in the unsafe public wrapper and the sole `j2k-ml` caller. + +This patch exists only to prevent wgpu from clearing a uniquely owned Burn +allocation after J2K's Metal producer has initialized it. It must be removed +when the targeted wgpu/Burn/CubeCL versions provide an upstream safe +external-write initialization and dependency-registration mechanism. diff --git a/third_party/wgpu-core-29.0.4-patched/build.rs b/third_party/wgpu-core-29.0.4-patched/build.rs new file mode 100644 index 00000000..15982414 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/build.rs @@ -0,0 +1,27 @@ +fn main() { + cfg_aliases::cfg_aliases! { + windows_linux_android: { any(windows, target_os = "linux", target_os = "android", target_os = "freebsd") }, + send_sync: { all( + feature = "std", + any( + not(target_arch = "wasm32"), + all(feature = "fragile-send-sync-non-atomic-wasm", not(target_feature = "atomics")) + ) + ) }, + dx12: { all(target_os = "windows", feature = "dx12") }, + webgl: { all(target_arch = "wasm32", not(target_os = "emscripten"), feature = "webgl") }, + gles: { any( + all(windows_linux_android, feature = "gles"), // Regular GLES + all(webgl), // WebGL + all(target_os = "emscripten", feature = "gles"), // Emscripten GLES + all(target_vendor = "apple", feature = "angle") // ANGLE on Apple + ) }, + vulkan: { any( + all(windows_linux_android, feature = "vulkan"), // Regular Vulkan + all(target_vendor = "apple", feature = "vulkan-portability") // Vulkan Portability on Apple + ) }, + metal: { all(target_vendor = "apple", feature = "metal") }, + + supports_64bit_atomics: { target_has_atomic = "64" } + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/as_hal.rs b/third_party/wgpu-core-29.0.4-patched/src/as_hal.rs new file mode 100644 index 00000000..cb09f03a --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/as_hal.rs @@ -0,0 +1,413 @@ +use core::{mem::ManuallyDrop, ops::Deref}; + +use alloc::sync::Arc; +use hal::DynResource; + +use crate::{ + device::Device, + global::Global, + id::{ + AdapterId, BlasId, BufferId, CommandEncoderId, DeviceId, QueueId, SurfaceId, TextureId, + TextureViewId, TlasId, + }, + lock::{RankData, RwLockReadGuard}, + resource::RawResourceAccess, + snatch::SnatchGuard, +}; + +/// A guard which holds alive a wgpu-core resource and dereferences to the Hal type. +struct SimpleResourceGuard { + _guard: Resource, + ptr: *const HalType, +} + +impl SimpleResourceGuard { + /// Creates a new guard from a resource, using a callback to derive the Hal type. + pub fn new(guard: Resource, callback: C) -> Option + where + C: Fn(&Resource) -> Option<&HalType>, + { + // Derive the hal type from the resource and coerce it to a pointer. + let ptr: *const HalType = callback(&guard)?; + + Some(Self { _guard: guard, ptr }) + } +} + +impl Deref for SimpleResourceGuard { + type Target = HalType; + + fn deref(&self) -> &Self::Target { + // SAFETY: The pointer is guaranteed to be valid as the original resource is + // still alive and this guard cannot be used with snatchable resources. + unsafe { &*self.ptr } + } +} + +unsafe impl Send for SimpleResourceGuard +where + Resource: Send, + HalType: Send, +{ +} +unsafe impl Sync for SimpleResourceGuard +where + Resource: Sync, + HalType: Sync, +{ +} + +/// A guard which holds alive a snatchable wgpu-core resource and dereferences to the Hal type. +struct SnatchableResourceGuard +where + Resource: RawResourceAccess, +{ + resource: Arc, + snatch_lock_rank_data: ManuallyDrop, + ptr: *const HalType, +} + +impl SnatchableResourceGuard +where + Resource: RawResourceAccess, + HalType: 'static, +{ + /// Creates a new guard from a snatchable resource. + /// + /// Returns `None` if: + /// - The resource is not of the expected Hal type. + /// - The resource has been destroyed. + pub fn new(resource: Arc) -> Option { + // Grab the snatchable lock. + let snatch_guard = resource.device().snatchable_lock.read(); + + // Get the raw resource and downcast it to the expected Hal type. + let underlying = resource + .raw(&snatch_guard)? + .as_any() + .downcast_ref::()?; + + // Cast the raw resource to a pointer to get rid of the lifetime + // connecting us to the snatch guard. + let ptr: *const HalType = underlying; + + // SAFETY: At this point all panicking or divergance has already happened, + // so we can safely forget the snatch guard without causing the lock to be left open. + let snatch_lock_rank_data = SnatchGuard::forget(snatch_guard); + + // SAFETY: We only construct this guard while the snatchable lock is held, + // as the `drop` implementation of this guard will unsafely release the lock. + Some(Self { + resource, + snatch_lock_rank_data: ManuallyDrop::new(snatch_lock_rank_data), + ptr, + }) + } +} + +impl Deref for SnatchableResourceGuard +where + Resource: RawResourceAccess, +{ + type Target = HalType; + + fn deref(&self) -> &Self::Target { + // SAFETY: The pointer is guaranteed to be valid as the original resource is + // still alive and the snatchable lock is still being held due to the forgotten + // snatch guard. + unsafe { &*self.ptr } + } +} + +impl Drop for SnatchableResourceGuard +where + Resource: RawResourceAccess, +{ + fn drop(&mut self) { + // SAFETY: + // - We are not going to access the rank data anymore. + let data = unsafe { ManuallyDrop::take(&mut self.snatch_lock_rank_data) }; + + // SAFETY: + // - The pointer is no longer going to be accessed. + // - The snatchable lock is being held because this type was not created + // until after the snatchable lock was forgotten. + unsafe { + self.resource + .device() + .snatchable_lock + .force_unlock_read(data) + }; + } +} + +unsafe impl Send for SnatchableResourceGuard +where + Resource: RawResourceAccess + Send, + HalType: Send, +{ +} +unsafe impl Sync for SnatchableResourceGuard +where + Resource: RawResourceAccess + Sync, + HalType: Sync, +{ +} + +/// A guard which holds alive a device and the device's fence lock, dereferencing to the Hal type. +struct FenceGuard { + device: Arc, + fence_lock_rank_data: ManuallyDrop, + ptr: *const Fence, +} + +impl FenceGuard +where + Fence: 'static, +{ + /// Creates a new guard over a device's fence. + /// + /// Returns `None` if: + /// - The device's fence is not of the expected Hal type. + pub fn new(device: Arc) -> Option { + // Grab the fence lock. + let fence_guard = device.fence.read(); + + // Get the raw fence and downcast it to the expected Hal type, coercing it to a pointer + // to get rid of the lifetime connecting us to the fence guard. + let ptr: *const Fence = fence_guard.as_any().downcast_ref::()?; + + // SAFETY: At this point all panicking or divergance has already happened, + // so we can safely forget the fence guard without causing the lock to be left open. + let fence_lock_rank_data = RwLockReadGuard::forget(fence_guard); + + // SAFETY: We only construct this guard while the fence lock is held, + // as the `drop` implementation of this guard will unsafely release the lock. + Some(Self { + device, + fence_lock_rank_data: ManuallyDrop::new(fence_lock_rank_data), + ptr, + }) + } +} + +impl Deref for FenceGuard { + type Target = Fence; + + fn deref(&self) -> &Self::Target { + // SAFETY: The pointer is guaranteed to be valid as the original device's fence + // is still alive and the fence lock is still being held due to the forgotten + // fence guard. + unsafe { &*self.ptr } + } +} + +impl Drop for FenceGuard { + fn drop(&mut self) { + // SAFETY: + // - We are not going to access the rank data anymore. + let data = unsafe { ManuallyDrop::take(&mut self.fence_lock_rank_data) }; + + // SAFETY: + // - The pointer is no longer going to be accessed. + // - The fence lock is being held because this type was not created + // until after the fence lock was forgotten. + unsafe { + self.device.fence.force_unlock_read(data); + }; + } +} + +unsafe impl Send for FenceGuard where Fence: Send {} +unsafe impl Sync for FenceGuard where Fence: Sync {} + +impl Global { + /// # Safety + /// + /// - The raw buffer handle must not be manually destroyed + pub unsafe fn buffer_as_hal( + &self, + id: BufferId, + ) -> Option> { + profiling::scope!("Buffer::as_hal"); + + let hub = &self.hub; + + let buffer = hub.buffers.get(id).get().ok()?; + + SnatchableResourceGuard::new(buffer) + } + + /// # Safety + /// + /// - The raw texture handle must not be manually destroyed + pub unsafe fn texture_as_hal( + &self, + id: TextureId, + ) -> Option> { + profiling::scope!("Texture::as_hal"); + + let hub = &self.hub; + + let texture = hub.textures.get(id).get().ok()?; + + SnatchableResourceGuard::new(texture) + } + + /// # Safety + /// + /// - The raw texture view handle must not be manually destroyed + pub unsafe fn texture_view_as_hal( + &self, + id: TextureViewId, + ) -> Option> { + profiling::scope!("TextureView::as_hal"); + + let hub = &self.hub; + + let view = hub.texture_views.get(id).get().ok()?; + + SnatchableResourceGuard::new(view) + } + + /// # Safety + /// + /// - The raw adapter handle must not be manually destroyed + pub unsafe fn adapter_as_hal( + &self, + id: AdapterId, + ) -> Option> { + profiling::scope!("Adapter::as_hal"); + + let hub = &self.hub; + let adapter = hub.adapters.get(id); + + SimpleResourceGuard::new(adapter, move |adapter| { + adapter.raw.adapter.as_any().downcast_ref() + }) + } + + /// # Safety + /// + /// - The raw device handle must not be manually destroyed + pub unsafe fn device_as_hal( + &self, + id: DeviceId, + ) -> Option> { + profiling::scope!("Device::as_hal"); + + let device = self.hub.devices.get(id); + + SimpleResourceGuard::new(device, move |device| device.raw().as_any().downcast_ref()) + } + + /// # Safety + /// + /// - The raw fence handle must not be manually destroyed + pub unsafe fn device_fence_as_hal( + &self, + id: DeviceId, + ) -> Option> { + profiling::scope!("Device::fence_as_hal"); + + let device = self.hub.devices.get(id); + + FenceGuard::new(device) + } + + /// # Safety + /// - The raw surface handle must not be manually destroyed + pub unsafe fn surface_as_hal( + &self, + id: SurfaceId, + ) -> Option> { + profiling::scope!("Surface::as_hal"); + + let surface = self.surfaces.get(id); + + SimpleResourceGuard::new(surface, move |surface| { + surface.raw(A::VARIANT)?.as_any().downcast_ref() + }) + } + + /// Encode commands using the raw HAL command encoder. + /// + /// # Panics + /// + /// If the command encoder has already been used with the wgpu encoding API. + /// + /// # Safety + /// + /// - The raw command encoder handle must not be manually destroyed + pub unsafe fn command_encoder_as_hal_mut< + A: hal::Api, + F: FnOnce(Option<&mut A::CommandEncoder>) -> R, + R, + >( + &self, + id: CommandEncoderId, + hal_command_encoder_callback: F, + ) -> R { + profiling::scope!("CommandEncoder::as_hal"); + + let hub = &self.hub; + + let cmd_enc = hub.command_encoders.get(id); + let mut cmd_buf_data = cmd_enc.data.lock(); + cmd_buf_data.record_as_hal_mut(|opt_cmd_buf| -> R { + hal_command_encoder_callback(opt_cmd_buf.and_then(|cmd_buf| { + cmd_buf + .encoder + .open() + .ok() + .and_then(|encoder| encoder.as_any_mut().downcast_mut()) + })) + }) + } + + /// # Safety + /// + /// - The raw queue handle must not be manually destroyed + pub unsafe fn queue_as_hal( + &self, + id: QueueId, + ) -> Option> { + profiling::scope!("Queue::as_hal"); + + let queue = self.hub.queues.get(id); + + SimpleResourceGuard::new(queue, move |queue| queue.raw().as_any().downcast_ref()) + } + + /// # Safety + /// + /// - The raw blas handle must not be manually destroyed + pub unsafe fn blas_as_hal( + &self, + id: BlasId, + ) -> Option> { + profiling::scope!("Blas::as_hal"); + + let hub = &self.hub; + + let blas = hub.blas_s.get(id).get().ok()?; + + SnatchableResourceGuard::new(blas) + } + + /// # Safety + /// + /// - The raw tlas handle must not be manually destroyed + pub unsafe fn tlas_as_hal( + &self, + id: TlasId, + ) -> Option> { + profiling::scope!("Tlas::as_hal"); + + let hub = &self.hub; + + let tlas = hub.tlas_s.get(id).get().ok()?; + + SnatchableResourceGuard::new(tlas) + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/binding_model.rs b/third_party/wgpu-core-29.0.4-patched/src/binding_model.rs new file mode 100644 index 00000000..a49bc73b --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/binding_model.rs @@ -0,0 +1,1362 @@ +use alloc::{ + borrow::{Cow, ToOwned}, + boxed::Box, + string::String, + sync::{Arc, Weak}, + vec::Vec, +}; +use core::{fmt, mem::ManuallyDrop, ops::Range}; + +use arrayvec::ArrayVec; +use thiserror::Error; + +#[cfg(feature = "serde")] +use serde::Deserialize; +#[cfg(feature = "serde")] +use serde::Serialize; + +use wgt::error::{ErrorType, WebGpuError}; + +use crate::{ + device::{bgl, Device, DeviceError, MissingDownlevelFlags, MissingFeatures}, + id::{BindGroupLayoutId, BufferId, ExternalTextureId, SamplerId, TextureViewId, TlasId}, + init_tracker::{BufferInitTrackerAction, TextureInitTrackerAction}, + pipeline::{ComputePipeline, RenderPipeline}, + resource::{ + Buffer, DestroyedResourceError, ExternalTexture, InvalidResourceError, Labeled, + MissingBufferUsageError, MissingTextureUsageError, RawResourceAccess, ResourceErrorIdent, + Sampler, TextureView, Tlas, TrackingData, + }, + resource_log, + snatch::{SnatchGuard, Snatchable}, + track::{BindGroupStates, ResourceUsageCompatibilityError}, + Label, +}; + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum BindGroupLayoutEntryError { + #[error("Cube dimension is not expected for texture storage")] + StorageTextureCube, + #[error("Atomic storage textures are not allowed by baseline webgpu, they require the native only feature TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES")] + StorageTextureAtomic, + #[error("Arrays of bindings unsupported for this type of binding")] + ArrayUnsupported, + #[error("Multisampled binding with sample type `TextureSampleType::Float` must have filterable set to false.")] + SampleTypeFloatFilterableBindingMultisampled, + #[error("Multisampled texture binding view dimension must be 2d, got {0:?}")] + Non2DMultisampled(wgt::TextureViewDimension), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateBindGroupLayoutError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Conflicting binding at index {0}")] + ConflictBinding(u32), + #[error("Binding {binding} entry is invalid")] + Entry { + binding: u32, + #[source] + error: BindGroupLayoutEntryError, + }, + #[error(transparent)] + TooManyBindings(BindingTypeMaxCountError), + #[error("Bind groups may not contain both a binding array and a dynamically offset buffer")] + ContainsBothBindingArrayAndDynamicOffsetArray, + #[error("Bind groups may not contain both a binding array and a uniform buffer")] + ContainsBothBindingArrayAndUniformBuffer, + #[error("Binding index {binding} is greater than the maximum number {maximum}")] + InvalidBindingIndex { binding: u32, maximum: u32 }, + #[error("Invalid visibility {0:?}")] + InvalidVisibility(wgt::ShaderStages), + #[error("Binding index {binding}: {access:?} access to storage textures with format {format:?} is not supported")] + UnsupportedStorageTextureAccess { + binding: u32, + access: wgt::StorageTextureAccess, + format: wgt::TextureFormat, + }, +} + +impl WebGpuError for CreateBindGroupLayoutError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + + Self::ConflictBinding(_) + | Self::Entry { .. } + | Self::TooManyBindings(_) + | Self::InvalidBindingIndex { .. } + | Self::InvalidVisibility(_) + | Self::ContainsBothBindingArrayAndDynamicOffsetArray + | Self::ContainsBothBindingArrayAndUniformBuffer + | Self::UnsupportedStorageTextureAccess { .. } => ErrorType::Validation, + } + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum BindingError { + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + #[error("Buffer {buffer}: Binding with size {binding_size} at offset {offset} would overflow buffer size of {buffer_size}")] + BindingRangeTooLarge { + buffer: ResourceErrorIdent, + offset: wgt::BufferAddress, + binding_size: u64, + buffer_size: u64, + }, + #[error("Buffer {buffer}: Binding offset {offset} is greater than buffer size {buffer_size}")] + BindingOffsetTooLarge { + buffer: ResourceErrorIdent, + offset: wgt::BufferAddress, + buffer_size: u64, + }, +} + +impl WebGpuError for BindingError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::DestroyedResource(e) => e.webgpu_error_type(), + Self::BindingRangeTooLarge { .. } | Self::BindingOffsetTooLarge { .. } => { + ErrorType::Validation + } + } + } +} + +// TODO: there may be additional variants here that can be extracted into +// `BindingError`. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateBindGroupError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + #[error(transparent)] + BindingError(#[from] BindingError), + #[error( + "Binding count declared with at most {expected} items, but {actual} items were provided" + )] + BindingArrayPartialLengthMismatch { actual: usize, expected: usize }, + #[error( + "Binding count declared with exactly {expected} items, but {actual} items were provided" + )] + BindingArrayLengthMismatch { actual: usize, expected: usize }, + #[error("Array binding provided zero elements")] + BindingArrayZeroLength, + #[error("Binding size {actual} of {buffer} is less than minimum {min}")] + BindingSizeTooSmall { + buffer: ResourceErrorIdent, + actual: u64, + min: u64, + }, + #[error("{0} binding size is zero")] + BindingZeroSize(ResourceErrorIdent), + #[error("Number of bindings in bind group descriptor ({actual}) does not match the number of bindings defined in the bind group layout ({expected})")] + BindingsNumMismatch { actual: usize, expected: usize }, + #[error("Binding {0} is used at least twice in the descriptor")] + DuplicateBinding(u32), + #[error("Unable to find a corresponding declaration for the given binding {0}")] + MissingBindingDeclaration(u32), + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + #[error(transparent)] + MissingTextureUsage(#[from] MissingTextureUsageError), + #[error("Binding declared as a single item, but bind group is using it as an array")] + SingleBindingExpected, + #[error("Effective buffer binding size {size} for storage buffers is expected to align to {alignment}, but size is {size}")] + UnalignedEffectiveBufferBindingSizeForStorage { alignment: u32, size: u64 }, + #[error("Buffer offset {0} does not respect device's requested `{1}` limit {2}")] + UnalignedBufferOffset(wgt::BufferAddress, &'static str, u32), + #[error( + "Buffer binding {binding} range {given} exceeds `max_*_buffer_binding_size` limit {limit}" + )] + BufferRangeTooLarge { + binding: u32, + given: u64, + limit: u64, + }, + #[error("Binding {binding} has a different type ({actual:?}) than the one in the layout ({expected:?})")] + WrongBindingType { + // Index of the binding + binding: u32, + // The type given to the function + actual: wgt::BindingType, + // Human-readable description of expected types + expected: &'static str, + }, + #[error("Texture binding {binding} expects multisampled = {layout_multisampled}, but given a view with samples = {view_samples}")] + InvalidTextureMultisample { + binding: u32, + layout_multisampled: bool, + view_samples: u32, + }, + #[error( + "Texture binding {} expects sample type {:?}, but was given a view with format {:?} (sample type {:?})", + binding, + layout_sample_type, + view_format, + view_sample_type + )] + InvalidTextureSampleType { + binding: u32, + layout_sample_type: wgt::TextureSampleType, + view_format: wgt::TextureFormat, + view_sample_type: wgt::TextureSampleType, + }, + #[error("Texture binding {binding} expects dimension = {layout_dimension:?}, but given a view with dimension = {view_dimension:?}")] + InvalidTextureDimension { + binding: u32, + layout_dimension: wgt::TextureViewDimension, + view_dimension: wgt::TextureViewDimension, + }, + #[error("Storage texture binding {binding} expects format = {layout_format:?}, but given a view with format = {view_format:?}")] + InvalidStorageTextureFormat { + binding: u32, + layout_format: wgt::TextureFormat, + view_format: wgt::TextureFormat, + }, + #[error("Storage texture bindings must have a single mip level, but given a view with mip_level_count = {mip_level_count:?} at binding {binding}")] + InvalidStorageTextureMipLevelCount { binding: u32, mip_level_count: u32 }, + #[error("External texture bindings must have a single mip level, but given a view with mip_level_count = {mip_level_count:?} at binding {binding}")] + InvalidExternalTextureMipLevelCount { binding: u32, mip_level_count: u32 }, + #[error("External texture bindings must have a format of `rgba8unorm`, `bgra8unorm`, or `rgba16float, but given a view with format = {format:?} at binding {binding}")] + InvalidExternalTextureFormat { + binding: u32, + format: wgt::TextureFormat, + }, + #[error("Sampler binding {binding} expects comparison = {layout_cmp}, but given a sampler with comparison = {sampler_cmp}")] + WrongSamplerComparison { + binding: u32, + layout_cmp: bool, + sampler_cmp: bool, + }, + #[error("Sampler binding {binding} expects filtering = {layout_flt}, but given a sampler with filtering = {sampler_flt}")] + WrongSamplerFiltering { + binding: u32, + layout_flt: bool, + sampler_flt: bool, + }, + #[error("TLAS binding {binding} is required to support vertex returns but is missing flag AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN")] + MissingTLASVertexReturn { binding: u32 }, + #[error("Bound texture views can not have both depth and stencil aspects enabled")] + DepthStencilAspect, + #[error(transparent)] + ResourceUsageCompatibility(#[from] ResourceUsageCompatibilityError), + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), +} + +impl WebGpuError for CreateBindGroupError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::DestroyedResource(e) => e.webgpu_error_type(), + Self::BindingError(e) => e.webgpu_error_type(), + Self::MissingBufferUsage(e) => e.webgpu_error_type(), + Self::MissingTextureUsage(e) => e.webgpu_error_type(), + Self::ResourceUsageCompatibility(e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::BindingArrayPartialLengthMismatch { .. } + | Self::BindingArrayLengthMismatch { .. } + | Self::BindingArrayZeroLength + | Self::BindingSizeTooSmall { .. } + | Self::BindingsNumMismatch { .. } + | Self::BindingZeroSize(_) + | Self::DuplicateBinding(_) + | Self::MissingBindingDeclaration(_) + | Self::SingleBindingExpected + | Self::UnalignedEffectiveBufferBindingSizeForStorage { .. } + | Self::UnalignedBufferOffset(_, _, _) + | Self::BufferRangeTooLarge { .. } + | Self::WrongBindingType { .. } + | Self::InvalidTextureMultisample { .. } + | Self::InvalidTextureSampleType { .. } + | Self::InvalidTextureDimension { .. } + | Self::InvalidStorageTextureFormat { .. } + | Self::InvalidStorageTextureMipLevelCount { .. } + | Self::WrongSamplerComparison { .. } + | Self::WrongSamplerFiltering { .. } + | Self::DepthStencilAspect + | Self::MissingTLASVertexReturn { .. } + | Self::InvalidExternalTextureMipLevelCount { .. } + | Self::InvalidExternalTextureFormat { .. } => ErrorType::Validation, + } + } +} + +#[derive(Clone, Debug, Error)] +pub enum BindingZone { + #[error("Stage {0:?}")] + Stage(wgt::ShaderStages), + #[error("Whole pipeline")] + Pipeline, +} + +#[derive(Clone, Debug, Error)] +#[error("Too many bindings of type {kind:?} in {zone}, limit is {limit}, count was {count}. Check the limit `{}` passed to `Adapter::request_device`", .kind.to_config_str())] +pub struct BindingTypeMaxCountError { + pub kind: BindingTypeMaxCountErrorKind, + pub zone: BindingZone, + pub limit: u32, + pub count: u32, +} + +impl WebGpuError for BindingTypeMaxCountError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +#[derive(Clone, Debug)] +pub enum BindingTypeMaxCountErrorKind { + DynamicUniformBuffers, + DynamicStorageBuffers, + SampledTextures, + Samplers, + StorageBuffers, + StorageTextures, + UniformBuffers, + BindingArrayElements, + BindingArraySamplerElements, + BindingArrayAccelerationStructureElements, + AccelerationStructures, +} + +impl BindingTypeMaxCountErrorKind { + fn to_config_str(&self) -> &'static str { + match self { + BindingTypeMaxCountErrorKind::DynamicUniformBuffers => { + "max_dynamic_uniform_buffers_per_pipeline_layout" + } + BindingTypeMaxCountErrorKind::DynamicStorageBuffers => { + "max_dynamic_storage_buffers_per_pipeline_layout" + } + BindingTypeMaxCountErrorKind::SampledTextures => { + "max_sampled_textures_per_shader_stage" + } + BindingTypeMaxCountErrorKind::Samplers => "max_samplers_per_shader_stage", + BindingTypeMaxCountErrorKind::StorageBuffers => "max_storage_buffers_per_shader_stage", + BindingTypeMaxCountErrorKind::StorageTextures => { + "max_storage_textures_per_shader_stage" + } + BindingTypeMaxCountErrorKind::UniformBuffers => "max_uniform_buffers_per_shader_stage", + BindingTypeMaxCountErrorKind::BindingArrayElements => { + "max_binding_array_elements_per_shader_stage" + } + BindingTypeMaxCountErrorKind::BindingArraySamplerElements => { + "max_binding_array_sampler_elements_per_shader_stage" + } + BindingTypeMaxCountErrorKind::BindingArrayAccelerationStructureElements => { + "max_binding_array_acceleration_structure_elements_per_shader_stage" + } + BindingTypeMaxCountErrorKind::AccelerationStructures => { + "max_acceleration_structures_per_shader_stage" + } + } + } +} + +#[derive(Debug, Default)] +pub(crate) struct PerStageBindingTypeCounter { + vertex: u32, + fragment: u32, + compute: u32, +} + +impl PerStageBindingTypeCounter { + pub(crate) fn add(&mut self, stage: wgt::ShaderStages, count: u32) { + if stage.contains(wgt::ShaderStages::VERTEX) { + self.vertex += count; + } + if stage.contains(wgt::ShaderStages::FRAGMENT) { + self.fragment += count; + } + if stage.contains(wgt::ShaderStages::COMPUTE) { + self.compute += count; + } + } + + pub(crate) fn max(&self) -> (BindingZone, u32) { + let max_value = self.vertex.max(self.fragment.max(self.compute)); + let mut stage = wgt::ShaderStages::NONE; + if max_value == self.vertex { + stage |= wgt::ShaderStages::VERTEX + } + if max_value == self.fragment { + stage |= wgt::ShaderStages::FRAGMENT + } + if max_value == self.compute { + stage |= wgt::ShaderStages::COMPUTE + } + (BindingZone::Stage(stage), max_value) + } + + pub(crate) fn merge(&mut self, other: &Self) { + self.vertex += other.vertex; + self.fragment += other.fragment; + self.compute += other.compute; + } + + pub(crate) fn validate( + &self, + limit: u32, + kind: BindingTypeMaxCountErrorKind, + ) -> Result<(), BindingTypeMaxCountError> { + let (zone, count) = self.max(); + if limit < count { + Err(BindingTypeMaxCountError { + kind, + zone, + limit, + count, + }) + } else { + Ok(()) + } + } +} + +#[derive(Debug, Default)] +pub(crate) struct BindingTypeMaxCountValidator { + dynamic_uniform_buffers: u32, + dynamic_storage_buffers: u32, + sampled_textures: PerStageBindingTypeCounter, + samplers: PerStageBindingTypeCounter, + storage_buffers: PerStageBindingTypeCounter, + storage_textures: PerStageBindingTypeCounter, + uniform_buffers: PerStageBindingTypeCounter, + acceleration_structures: PerStageBindingTypeCounter, + binding_array_elements: PerStageBindingTypeCounter, + binding_array_sampler_elements: PerStageBindingTypeCounter, + binding_array_acceleration_structure_elements: PerStageBindingTypeCounter, + has_bindless_array: bool, +} + +impl BindingTypeMaxCountValidator { + pub(crate) fn add_binding(&mut self, binding: &wgt::BindGroupLayoutEntry) { + let count = binding.count.map_or(1, |count| count.get()); + + if binding.count.is_some() { + self.binding_array_elements.add(binding.visibility, count); + self.has_bindless_array = true; + + match binding.ty { + wgt::BindingType::Sampler(_) => { + self.binding_array_sampler_elements + .add(binding.visibility, count); + } + wgt::BindingType::AccelerationStructure { .. } => { + self.binding_array_acceleration_structure_elements + .add(binding.visibility, count); + } + _ => {} + } + } else { + match binding.ty { + wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Uniform, + has_dynamic_offset, + .. + } => { + self.uniform_buffers.add(binding.visibility, count); + if has_dynamic_offset { + self.dynamic_uniform_buffers += count; + } + } + wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Storage { .. }, + has_dynamic_offset, + .. + } => { + self.storage_buffers.add(binding.visibility, count); + if has_dynamic_offset { + self.dynamic_storage_buffers += count; + } + } + wgt::BindingType::Sampler { .. } => { + self.samplers.add(binding.visibility, count); + } + wgt::BindingType::Texture { .. } => { + self.sampled_textures.add(binding.visibility, count); + } + wgt::BindingType::StorageTexture { .. } => { + self.storage_textures.add(binding.visibility, count); + } + wgt::BindingType::AccelerationStructure { .. } => { + self.acceleration_structures.add(binding.visibility, count); + } + wgt::BindingType::ExternalTexture => { + // https://www.w3.org/TR/webgpu/#gpuexternaltexture + // In order to account for many possible representations, + // the binding conservatively uses the following, for each + // external texture: + // * Three sampled textures for up to 3 planes + // * One additional sampled texture for a 3D LUT + // * One sampler to sample the LUT + // * One uniform buffer for metadata + self.sampled_textures.add(binding.visibility, count * 4); + self.samplers.add(binding.visibility, count); + self.uniform_buffers.add(binding.visibility, count); + } + } + } + } + + pub(crate) fn merge(&mut self, other: &Self) { + self.dynamic_uniform_buffers += other.dynamic_uniform_buffers; + self.dynamic_storage_buffers += other.dynamic_storage_buffers; + self.sampled_textures.merge(&other.sampled_textures); + self.samplers.merge(&other.samplers); + self.storage_buffers.merge(&other.storage_buffers); + self.storage_textures.merge(&other.storage_textures); + self.uniform_buffers.merge(&other.uniform_buffers); + self.acceleration_structures + .merge(&other.acceleration_structures); + self.binding_array_elements + .merge(&other.binding_array_elements); + self.binding_array_sampler_elements + .merge(&other.binding_array_sampler_elements); + self.binding_array_acceleration_structure_elements + .merge(&other.binding_array_acceleration_structure_elements); + } + + pub(crate) fn validate(&self, limits: &wgt::Limits) -> Result<(), BindingTypeMaxCountError> { + if limits.max_dynamic_uniform_buffers_per_pipeline_layout < self.dynamic_uniform_buffers { + return Err(BindingTypeMaxCountError { + kind: BindingTypeMaxCountErrorKind::DynamicUniformBuffers, + zone: BindingZone::Pipeline, + limit: limits.max_dynamic_uniform_buffers_per_pipeline_layout, + count: self.dynamic_uniform_buffers, + }); + } + if limits.max_dynamic_storage_buffers_per_pipeline_layout < self.dynamic_storage_buffers { + return Err(BindingTypeMaxCountError { + kind: BindingTypeMaxCountErrorKind::DynamicStorageBuffers, + zone: BindingZone::Pipeline, + limit: limits.max_dynamic_storage_buffers_per_pipeline_layout, + count: self.dynamic_storage_buffers, + }); + } + self.sampled_textures.validate( + limits.max_sampled_textures_per_shader_stage, + BindingTypeMaxCountErrorKind::SampledTextures, + )?; + self.samplers.validate( + limits.max_samplers_per_shader_stage, + BindingTypeMaxCountErrorKind::Samplers, + )?; + self.storage_buffers.validate( + limits.max_storage_buffers_per_shader_stage, + BindingTypeMaxCountErrorKind::StorageBuffers, + )?; + self.storage_textures.validate( + limits.max_storage_textures_per_shader_stage, + BindingTypeMaxCountErrorKind::StorageTextures, + )?; + self.uniform_buffers.validate( + limits.max_uniform_buffers_per_shader_stage, + BindingTypeMaxCountErrorKind::UniformBuffers, + )?; + self.binding_array_elements.validate( + limits.max_binding_array_elements_per_shader_stage, + BindingTypeMaxCountErrorKind::BindingArrayElements, + )?; + self.binding_array_sampler_elements.validate( + limits.max_binding_array_sampler_elements_per_shader_stage, + BindingTypeMaxCountErrorKind::BindingArraySamplerElements, + )?; + self.binding_array_acceleration_structure_elements + .validate( + limits.max_binding_array_acceleration_structure_elements_per_shader_stage, + BindingTypeMaxCountErrorKind::BindingArrayAccelerationStructureElements, + )?; + self.acceleration_structures.validate( + limits.max_acceleration_structures_per_shader_stage, + BindingTypeMaxCountErrorKind::AccelerationStructures, + )?; + Ok(()) + } + + /// Validate that the bind group layout does not contain both a binding array and a dynamic offset array. + /// + /// This allows us to use `UPDATE_AFTER_BIND` on vulkan for bindless arrays. Vulkan does not allow + /// `UPDATE_AFTER_BIND` on dynamic offset arrays. See + pub(crate) fn validate_binding_arrays(&self) -> Result<(), CreateBindGroupLayoutError> { + let has_dynamic_offset_array = + self.dynamic_uniform_buffers > 0 || self.dynamic_storage_buffers > 0; + let has_uniform_buffer = self.uniform_buffers.max().1 > 0; + if self.has_bindless_array && has_dynamic_offset_array { + return Err(CreateBindGroupLayoutError::ContainsBothBindingArrayAndDynamicOffsetArray); + } + if self.has_bindless_array && has_uniform_buffer { + return Err(CreateBindGroupLayoutError::ContainsBothBindingArrayAndUniformBuffer); + } + Ok(()) + } +} + +/// Bindable resource and the slot to bind it to. +/// cbindgen:ignore +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct BindGroupEntry< + 'a, + B = BufferId, + S = SamplerId, + TV = TextureViewId, + TLAS = TlasId, + ET = ExternalTextureId, +> where + [BufferBinding]: ToOwned, + [S]: ToOwned, + [TV]: ToOwned, + [TLAS]: ToOwned, + <[BufferBinding] as ToOwned>::Owned: fmt::Debug, + <[S] as ToOwned>::Owned: fmt::Debug, + <[TV] as ToOwned>::Owned: fmt::Debug, + <[TLAS] as ToOwned>::Owned: fmt::Debug, +{ + /// Slot for which binding provides resource. Corresponds to an entry of the same + /// binding index in the [`BindGroupLayoutDescriptor`]. + pub binding: u32, + #[cfg_attr( + feature = "serde", + serde(bound(deserialize = "BindingResource<'a, B, S, TV, TLAS, ET>: Deserialize<'de>")) + )] + /// Resource to attach to the binding + pub resource: BindingResource<'a, B, S, TV, TLAS, ET>, +} + +/// cbindgen:ignore +pub type ResolvedBindGroupEntry<'a> = BindGroupEntry< + 'a, + Arc, + Arc, + Arc, + Arc, + Arc, +>; + +/// Describes a group of bindings and the resources to be bound. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct BindGroupDescriptor< + 'a, + BGL = BindGroupLayoutId, + B = BufferId, + S = SamplerId, + TV = TextureViewId, + TLAS = TlasId, + ET = ExternalTextureId, +> where + [BufferBinding]: ToOwned, + [S]: ToOwned, + [TV]: ToOwned, + [TLAS]: ToOwned, + <[BufferBinding] as ToOwned>::Owned: fmt::Debug, + <[S] as ToOwned>::Owned: fmt::Debug, + <[TV] as ToOwned>::Owned: fmt::Debug, + <[TLAS] as ToOwned>::Owned: fmt::Debug, + [BindGroupEntry<'a, B, S, TV, TLAS, ET>]: ToOwned, + <[BindGroupEntry<'a, B, S, TV, TLAS, ET>] as ToOwned>::Owned: fmt::Debug, +{ + /// Debug label of the bind group. + /// + /// This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The [`BindGroupLayout`] that corresponds to this bind group. + pub layout: BGL, + #[cfg_attr( + feature = "serde", + serde(bound( + deserialize = "<[BindGroupEntry<'a, B, S, TV, TLAS, ET>] as ToOwned>::Owned: Deserialize<'de>" + )) + )] + /// The resources to bind to this bind group. + #[allow(clippy::type_complexity)] + pub entries: Cow<'a, [BindGroupEntry<'a, B, S, TV, TLAS, ET>]>, +} + +/// cbindgen:ignore +pub type ResolvedBindGroupDescriptor<'a> = BindGroupDescriptor< + 'a, + Arc, + Arc, + Arc, + Arc, + Arc, + Arc, +>; + +/// Describes a [`BindGroupLayout`]. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct BindGroupLayoutDescriptor<'a> { + /// Debug label of the bind group layout. + /// + /// This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// Array of entries in this BindGroupLayout + pub entries: Cow<'a, [wgt::BindGroupLayoutEntry]>, +} + +/// Used by [`BindGroupLayout`]. It indicates whether the BGL must be +/// used with a specific pipeline. This constraint only happens when +/// the BGLs have been derived from a pipeline without a layout. +#[derive(Clone, Debug)] +pub(crate) enum ExclusivePipeline { + None, + Render(Weak), + Compute(Weak), +} + +impl fmt::Display for ExclusivePipeline { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ExclusivePipeline::None => f.write_str("None"), + ExclusivePipeline::Render(p) => { + if let Some(p) = p.upgrade() { + p.error_ident().fmt(f) + } else { + f.write_str("RenderPipeline") + } + } + ExclusivePipeline::Compute(p) => { + if let Some(p) = p.upgrade() { + p.error_ident().fmt(f) + } else { + f.write_str("ComputePipeline") + } + } + } + } +} + +#[derive(Debug)] +pub enum RawBindGroupLayout { + Owning(ManuallyDrop>), + /// The empty BGL was created by the device and will be destroyed by the device. + RefDeviceEmptyBGL, +} + +/// Bind group layout. +#[derive(Debug)] +pub struct BindGroupLayout { + pub(crate) raw: RawBindGroupLayout, + pub(crate) device: Arc, + pub(crate) entries: bgl::EntryMap, + /// It is very important that we know if the bind group comes from the BGL pool. + /// + /// If it does, then we need to remove it from the pool when we drop it. + /// + /// We cannot unconditionally remove from the pool, as BGLs that don't come from the pool + /// (derived BGLs) must not be removed. + pub(crate) origin: bgl::Origin, + pub(crate) exclusive_pipeline: crate::OnceCellOrLock, + pub(crate) binding_count_validator: BindingTypeMaxCountValidator, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, +} + +impl Drop for BindGroupLayout { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + if matches!(self.origin, bgl::Origin::Pool) { + self.device.bgl_pool.remove(&self.entries); + } + match self.raw { + RawBindGroupLayout::Owning(ref mut raw) => { + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(raw) }; + unsafe { + self.device.raw().destroy_bind_group_layout(raw); + } + } + RawBindGroupLayout::RefDeviceEmptyBGL => {} + } + } +} + +crate::impl_resource_type!(BindGroupLayout); +crate::impl_labeled!(BindGroupLayout); +crate::impl_parent_device!(BindGroupLayout); +crate::impl_storage_item!(BindGroupLayout); + +impl BindGroupLayout { + pub(crate) fn raw(&self) -> &dyn hal::DynBindGroupLayout { + match &self.raw { + RawBindGroupLayout::Owning(raw) => raw.as_ref(), + RawBindGroupLayout::RefDeviceEmptyBGL => self.device.empty_bgl.as_ref(), + } + } + + fn empty(device: &Arc) -> Arc { + let exclusive_pipeline = crate::OnceCellOrLock::new(); + exclusive_pipeline.set(ExclusivePipeline::None).unwrap(); + Arc::new(Self { + raw: RawBindGroupLayout::RefDeviceEmptyBGL, + device: device.clone(), + entries: bgl::EntryMap::default(), + origin: bgl::Origin::Derived, + exclusive_pipeline, + binding_count_validator: BindingTypeMaxCountValidator::default(), + label: String::new(), + }) + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreatePipelineLayoutError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error( + "Immediate data has range bound {size} which is not aligned to IMMEDIATE_DATA_ALIGNMENT ({})", + wgt::IMMEDIATE_DATA_ALIGNMENT + )] + MisalignedImmediateSize { size: u32 }, + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error( + "Immediate data has size {size} which exceeds device immediate data size limit 0..{max}" + )] + ImmediateRangeTooLarge { size: u32, max: u32 }, + #[error(transparent)] + TooManyBindings(BindingTypeMaxCountError), + #[error("Bind group layout count {actual} exceeds device bind group limit {max}")] + TooManyGroups { actual: usize, max: usize }, + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), + #[error("Bind group layout at index {index} has an exclusive pipeline: {pipeline}")] + BglHasExclusivePipeline { index: usize, pipeline: String }, +} + +impl WebGpuError for CreatePipelineLayoutError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::TooManyBindings(e) => e.webgpu_error_type(), + Self::MisalignedImmediateSize { .. } + | Self::ImmediateRangeTooLarge { .. } + | Self::TooManyGroups { .. } + | Self::BglHasExclusivePipeline { .. } => ErrorType::Validation, + } + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ImmediateUploadError { + #[error( + "Start offset {start_offset} overruns the immediate data range with a size of {immediate_size}" + )] + StartOffsetOverrun { + start_offset: u32, + immediate_size: u32, + }, + #[error( + "Provided immediate data start offset {0} does not respect \ + `IMMEDIATE_DATA_ALIGNMENT` ({ida})", + ida = wgt::IMMEDIATE_DATA_ALIGNMENT + )] + StartOffsetUnaligned(u32), + #[error( + "Provided immediate data byte size {0} does not respect \ + `IMMEDIATE_DATA_ALIGNMENT` ({ida})", + ida = wgt::IMMEDIATE_DATA_ALIGNMENT + )] + SizeUnaligned(u32), + #[error( + "Provided immediate data start offset {} + size {} overruns the immediate data range \ + with a size of {}", + start_offset, + size, + immediate_size + )] + EndOffsetOverrun { + start_offset: u32, + size: u32, + immediate_size: u32, + }, + #[error("Start index {start_index} overruns the value data range with {data_size} element(s)")] + ValueStartIndexOverrun { start_index: u32, data_size: usize }, + #[error( + "Start index {} + count of {} overruns the value data range \ + with {} element(s)", + start_index, + count, + data_size + )] + ValueEndIndexOverrun { + start_index: u32, + count: u32, + data_size: usize, + }, +} + +impl WebGpuError for ImmediateUploadError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +/// Describes a pipeline layout. +/// +/// A `PipelineLayoutDescriptor` can be used to create a pipeline layout. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(bound = "BGL: Serialize"))] +pub struct PipelineLayoutDescriptor<'a, BGL = BindGroupLayoutId> +where + [Option]: ToOwned, + <[Option] as ToOwned>::Owned: fmt::Debug, +{ + /// Debug label of the pipeline layout. + /// + /// This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// Bind groups that this pipeline uses. The first entry will provide all the bindings for + /// "set = 0", second entry will provide all the bindings for "set = 1" etc. + #[cfg_attr( + feature = "serde", + serde(bound(deserialize = "<[Option] as ToOwned>::Owned: Deserialize<'de>")) + )] + pub bind_group_layouts: Cow<'a, [Option]>, + /// The number of bytes of immediate data that are allocated for use + /// in the shader. The `var`s in the shader attached to + /// this pipeline must be equal or smaller than this size. + /// + /// If this value is non-zero, [`wgt::Features::IMMEDIATES`] must be enabled. + pub immediate_size: u32, +} + +/// cbindgen:ignore +pub type ResolvedPipelineLayoutDescriptor<'a, BGL = Arc> = + PipelineLayoutDescriptor<'a, BGL>; + +#[derive(Debug)] +pub struct PipelineLayout { + pub(crate) raw: ManuallyDrop>, + pub(crate) device: Arc, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) bind_group_layouts: ArrayVec>, { hal::MAX_BIND_GROUPS }>, + pub(crate) immediate_size: u32, +} + +impl Drop for PipelineLayout { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { + self.device.raw().destroy_pipeline_layout(raw); + } + } +} + +impl PipelineLayout { + pub(crate) fn raw(&self) -> &dyn hal::DynPipelineLayout { + self.raw.as_ref() + } + + pub fn get_bind_group_layout( + self: &Arc, + index: u32, + ) -> Result, GetBindGroupLayoutError> { + let max_bind_groups = self.device.limits.max_bind_groups; + if index >= max_bind_groups { + return Err(GetBindGroupLayoutError::IndexOutOfRange { + index, + max: max_bind_groups, + }); + } + Ok(self + .bind_group_layouts + .get(index as usize) + .cloned() + .flatten() + .unwrap_or_else(|| BindGroupLayout::empty(&self.device))) + } + + pub(crate) fn get_bgl_entry( + &self, + group: u32, + binding: u32, + ) -> Option<&wgt::BindGroupLayoutEntry> { + let bgl = self.bind_group_layouts.get(group as usize)?; + let bgl = bgl.as_ref()?; + bgl.entries.get(binding) + } + + /// Validate immediates match up with expected ranges. + pub(crate) fn validate_immediates_ranges( + &self, + offset: u32, + size_bytes: u32, + ) -> Result<(), ImmediateUploadError> { + // Don't need to validate size against the immediate data size limit here, + // as immediate data ranges are already validated to be within bounds, + // and we validate that they are within the ranges. + + if !offset.is_multiple_of(wgt::IMMEDIATE_DATA_ALIGNMENT) { + return Err(ImmediateUploadError::StartOffsetUnaligned(offset)); + } + + if !size_bytes.is_multiple_of(wgt::IMMEDIATE_DATA_ALIGNMENT) { + return Err(ImmediateUploadError::SizeUnaligned(offset)); + } + + if offset > self.immediate_size { + return Err(ImmediateUploadError::StartOffsetOverrun { + start_offset: offset, + immediate_size: self.immediate_size, + }); + } + + if size_bytes > self.immediate_size - offset { + return Err(ImmediateUploadError::EndOffsetOverrun { + start_offset: offset, + size: size_bytes, + immediate_size: self.immediate_size, + }); + } + + Ok(()) + } +} + +crate::impl_resource_type!(PipelineLayout); +crate::impl_labeled!(PipelineLayout); +crate::impl_parent_device!(PipelineLayout); +crate::impl_storage_item!(PipelineLayout); + +#[repr(C)] +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct BufferBinding { + pub buffer: B, + pub offset: wgt::BufferAddress, + + /// Size of the binding. If `None`, the binding spans from `offset` to the + /// end of the buffer. + /// + /// We use `BufferAddress` to allow a size of zero on this `wgpu_core` type, + /// because JavaScript bindings cannot readily express `Option`. + /// The `wgpu` API uses `Option` (i.e. `NonZeroU64`) for this + /// field. + pub size: Option, +} + +pub type ResolvedBufferBinding = BufferBinding>; + +// Note: Duplicated in `wgpu-rs` as `BindingResource` +// They're different enough that it doesn't make sense to share a common type +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum BindingResource< + 'a, + B = BufferId, + S = SamplerId, + TV = TextureViewId, + TLAS = TlasId, + ET = ExternalTextureId, +> where + [BufferBinding]: ToOwned, + [S]: ToOwned, + [TV]: ToOwned, + [TLAS]: ToOwned, + <[BufferBinding] as ToOwned>::Owned: fmt::Debug, + <[S] as ToOwned>::Owned: fmt::Debug, + <[TV] as ToOwned>::Owned: fmt::Debug, + <[TLAS] as ToOwned>::Owned: fmt::Debug, +{ + Buffer(BufferBinding), + #[cfg_attr( + feature = "serde", + serde(bound(deserialize = "<[BufferBinding] as ToOwned>::Owned: Deserialize<'de>")) + )] + BufferArray(Cow<'a, [BufferBinding]>), + Sampler(S), + #[cfg_attr( + feature = "serde", + serde(bound(deserialize = "<[S] as ToOwned>::Owned: Deserialize<'de>")) + )] + SamplerArray(Cow<'a, [S]>), + TextureView(TV), + #[cfg_attr( + feature = "serde", + serde(bound(deserialize = "<[TV] as ToOwned>::Owned: Deserialize<'de>")) + )] + TextureViewArray(Cow<'a, [TV]>), + AccelerationStructure(TLAS), + #[cfg_attr( + feature = "serde", + serde(bound(deserialize = "<[TLAS] as ToOwned>::Owned: Deserialize<'de>")) + )] + AccelerationStructureArray(Cow<'a, [TLAS]>), + ExternalTexture(ET), +} + +pub type ResolvedBindingResource<'a> = BindingResource< + 'a, + Arc, + Arc, + Arc, + Arc, + Arc, +>; + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum BindError { + #[error( + "Dynamic offsets not expected with null bind group at index {group}. However {actual} dynamic offset{s1} were provided.", + s1 = if *.actual >= 2 { "s" } else { "" }, + )] + DynamicOffsetCountNotZero { group: u32, actual: usize }, + #[error( + "{bind_group} {group} expects {expected} dynamic offset{s0}. However {actual} dynamic offset{s1} were provided.", + s0 = if *.expected >= 2 { "s" } else { "" }, + s1 = if *.actual >= 2 { "s" } else { "" }, + )] + MismatchedDynamicOffsetCount { + bind_group: ResourceErrorIdent, + group: u32, + actual: usize, + expected: usize, + }, + #[error( + "Dynamic binding index {idx} (targeting {bind_group} {group}, binding {binding}) with value {offset}, does not respect device's requested `{limit_name}` limit: {alignment}" + )] + UnalignedDynamicBinding { + bind_group: ResourceErrorIdent, + idx: usize, + group: u32, + binding: u32, + offset: u32, + alignment: u32, + limit_name: &'static str, + }, + #[error( + "Dynamic binding offset index {idx} with offset {offset} would overrun the buffer bound to {bind_group} {group} -> binding {binding}. \ + Buffer size is {buffer_size} bytes, the binding binds bytes {binding_range:?}, meaning the maximum the binding can be offset is {maximum_dynamic_offset} bytes", + )] + DynamicBindingOutOfBounds { + bind_group: ResourceErrorIdent, + idx: usize, + group: u32, + binding: u32, + offset: u32, + buffer_size: wgt::BufferAddress, + binding_range: Range, + maximum_dynamic_offset: wgt::BufferAddress, + }, +} + +impl WebGpuError for BindError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +#[derive(Debug)] +pub struct BindGroupDynamicBindingData { + /// The index of the binding. + /// + /// Used for more descriptive errors. + pub(crate) binding_idx: u32, + /// The size of the buffer. + /// + /// Used for more descriptive errors. + pub(crate) buffer_size: wgt::BufferAddress, + /// The range that the binding covers. + /// + /// Used for more descriptive errors. + pub(crate) binding_range: Range, + /// The maximum value the dynamic offset can have before running off the end of the buffer. + pub(crate) maximum_dynamic_offset: wgt::BufferAddress, + /// The binding type. + pub(crate) binding_type: wgt::BufferBindingType, +} + +pub(crate) fn buffer_binding_type_alignment( + limits: &wgt::Limits, + binding_type: wgt::BufferBindingType, +) -> (u32, &'static str) { + match binding_type { + wgt::BufferBindingType::Uniform => ( + limits.min_uniform_buffer_offset_alignment, + "min_uniform_buffer_offset_alignment", + ), + wgt::BufferBindingType::Storage { .. } => ( + limits.min_storage_buffer_offset_alignment, + "min_storage_buffer_offset_alignment", + ), + } +} + +pub(crate) fn buffer_binding_type_bounds_check_alignment( + alignments: &hal::Alignments, + binding_type: wgt::BufferBindingType, +) -> wgt::BufferAddress { + match binding_type { + wgt::BufferBindingType::Uniform => alignments.uniform_bounds_check_alignment.get(), + wgt::BufferBindingType::Storage { .. } => wgt::COPY_BUFFER_ALIGNMENT, + } +} + +#[derive(Debug)] +pub(crate) struct BindGroupLateBufferBindingInfo { + /// The normal binding index in the bind group. + pub binding_index: u32, + /// The size that exists at bind time. + pub size: wgt::BufferSize, +} + +#[derive(Debug)] +pub struct BindGroup { + pub(crate) raw: Snatchable>, + pub(crate) device: Arc, + pub(crate) layout: Arc, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, + pub(crate) used: BindGroupStates, + pub(crate) used_buffer_ranges: Vec, + pub(crate) used_texture_ranges: Vec, + pub(crate) dynamic_binding_info: Vec, + /// Actual binding sizes for buffers that don't have `min_binding_size` + /// specified in BGL. Listed in the order of iteration of `BGL.entries`. + pub(crate) late_buffer_binding_infos: Vec, +} + +impl Drop for BindGroup { + fn drop(&mut self) { + if let Some(raw) = self.raw.take() { + resource_log!("Destroy raw {}", self.error_ident()); + unsafe { + self.device.raw().destroy_bind_group(raw); + } + } + } +} + +impl BindGroup { + pub(crate) fn try_raw<'a>( + &'a self, + guard: &'a SnatchGuard, + ) -> Result<&'a dyn hal::DynBindGroup, DestroyedResourceError> { + // Clippy insist on writing it this way. The idea is to return None + // if any of the raw buffer is not valid anymore. + for buffer in &self.used_buffer_ranges { + buffer.buffer.try_raw(guard)?; + } + for texture in &self.used_texture_ranges { + texture.texture.try_raw(guard)?; + } + + self.raw + .get(guard) + .map(|raw| raw.as_ref()) + .ok_or_else(|| DestroyedResourceError(self.error_ident())) + } + + pub(crate) fn validate_dynamic_bindings( + &self, + bind_group_index: u32, + offsets: &[wgt::DynamicOffset], + ) -> Result<(), BindError> { + if self.dynamic_binding_info.len() != offsets.len() { + return Err(BindError::MismatchedDynamicOffsetCount { + bind_group: self.error_ident(), + group: bind_group_index, + expected: self.dynamic_binding_info.len(), + actual: offsets.len(), + }); + } + + for (idx, (info, &offset)) in self + .dynamic_binding_info + .iter() + .zip(offsets.iter()) + .enumerate() + { + let (alignment, limit_name) = + buffer_binding_type_alignment(&self.device.limits, info.binding_type); + if !(offset as wgt::BufferAddress).is_multiple_of(alignment as u64) { + return Err(BindError::UnalignedDynamicBinding { + bind_group: self.error_ident(), + group: bind_group_index, + binding: info.binding_idx, + idx, + offset, + alignment, + limit_name, + }); + } + + if offset as wgt::BufferAddress > info.maximum_dynamic_offset { + return Err(BindError::DynamicBindingOutOfBounds { + bind_group: self.error_ident(), + group: bind_group_index, + binding: info.binding_idx, + idx, + offset, + buffer_size: info.buffer_size, + binding_range: info.binding_range.clone(), + maximum_dynamic_offset: info.maximum_dynamic_offset, + }); + } + } + + Ok(()) + } +} + +crate::impl_resource_type!(BindGroup); +crate::impl_labeled!(BindGroup); +crate::impl_parent_device!(BindGroup); +crate::impl_storage_item!(BindGroup); +crate::impl_trackable!(BindGroup); + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum GetBindGroupLayoutError { + #[error("Bind group layout index {index} is greater than the device's configured `max_bind_groups` limit {max}")] + IndexOutOfRange { index: u32, max: u32 }, + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), +} + +impl WebGpuError for GetBindGroupLayoutError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::IndexOutOfRange { .. } => ErrorType::Validation, + Self::InvalidResource(e) => e.webgpu_error_type(), + } + } +} + +#[derive(Clone, Debug, Error, Eq, PartialEq)] +#[error( + "In bind group index {group_index}, the buffer bound at binding index {binding_index} \ + is bound with size {bound_size} where the shader expects {shader_size}." +)] +pub struct LateMinBufferBindingSizeMismatch { + pub group_index: u32, + pub binding_index: u32, + pub shader_size: wgt::BufferAddress, + pub bound_size: wgt::BufferAddress, +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/allocator.rs b/third_party/wgpu-core-29.0.4-patched/src/command/allocator.rs new file mode 100644 index 00000000..21b81c97 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/allocator.rs @@ -0,0 +1,52 @@ +use alloc::{boxed::Box, vec::Vec}; + +use crate::lock::{rank, Mutex}; + +/// A pool of free [`wgpu_hal::CommandEncoder`]s, owned by a `Device`. +/// +/// Each encoder in this list is in the "closed" state. +/// +/// Since a raw [`CommandEncoder`][ce] is itself a pool for allocating +/// raw [`CommandBuffer`][cb]s, this is a pool of pools. +/// +/// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder +/// [ce]: hal::CommandEncoder +/// [cb]: hal::Api::CommandBuffer +pub(crate) struct CommandAllocator { + free_encoders: Mutex>>, +} + +impl CommandAllocator { + pub(crate) fn new() -> Self { + Self { + free_encoders: Mutex::new(rank::COMMAND_ALLOCATOR_FREE_ENCODERS, Vec::new()), + } + } + + /// Return a fresh [`wgpu_hal::CommandEncoder`] in the "closed" state. + /// + /// If we have free encoders in the pool, take one of those. Otherwise, + /// create a new one on `device`. + /// + /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder + pub(crate) fn acquire_encoder( + &self, + device: &dyn hal::DynDevice, + queue: &dyn hal::DynQueue, + ) -> Result, hal::DeviceError> { + let mut free_encoders = self.free_encoders.lock(); + match free_encoders.pop() { + Some(encoder) => Ok(encoder), + None => unsafe { + let hal_desc = hal::CommandEncoderDescriptor { label: None, queue }; + device.create_command_encoder(&hal_desc) + }, + } + } + + /// Add `encoder` back to the free pool. + pub(crate) fn release_encoder(&self, encoder: Box) { + let mut free_encoders = self.free_encoders.lock(); + free_encoders.push(encoder); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/bind.rs b/third_party/wgpu-core-29.0.4-patched/src/command/bind.rs new file mode 100644 index 00000000..c0733934 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/bind.rs @@ -0,0 +1,534 @@ +use alloc::{boxed::Box, sync::Arc, vec::Vec}; + +use thiserror::Error; + +use crate::{ + binding_model::{BindGroup, LateMinBufferBindingSizeMismatch, PipelineLayout}, + pipeline::LateSizedBufferGroup, + resource::{Labeled, ParentDevice, ResourceErrorIdent}, +}; + +mod compat { + use alloc::{ + string::{String, ToString as _}, + sync::{Arc, Weak}, + vec::Vec, + }; + use core::num::NonZeroU32; + + use thiserror::Error; + use wgt::{BindingType, ShaderStages}; + + use crate::{ + binding_model::BindGroupLayout, + error::MultiError, + resource::{Labeled, ParentDevice, ResourceErrorIdent}, + }; + + pub(crate) enum Error { + Incompatible { + expected_bgl: ResourceErrorIdent, + assigned_bgl: ResourceErrorIdent, + inner: MultiError, + }, + Missing, + } + + #[derive(Debug, Clone)] + struct Entry { + assigned: Option>, + expected: Option>, + } + + impl Entry { + const fn empty() -> Self { + Self { + assigned: None, + expected: None, + } + } + + fn is_active(&self) -> bool { + self.assigned.is_some() && self.expected.is_some() + } + + fn is_valid(&self) -> bool { + if let Some(expected_bgl) = self.expected.as_ref() { + if let Some(assigned_bgl) = self.assigned.as_ref() { + expected_bgl.is_equal(assigned_bgl) + } else { + false + } + } else { + false + } + } + + fn check(&self) -> Result<(), Error> { + if let Some(expected_bgl) = self.expected.as_ref() { + if let Some(assigned_bgl) = self.assigned.as_ref() { + if expected_bgl.is_equal(assigned_bgl) { + Ok(()) + } else { + #[derive(Clone, Debug, Error)] + #[error( + "Exclusive pipelines don't match: expected {expected}, got {assigned}" + )] + struct IncompatibleExclusivePipelines { + expected: String, + assigned: String, + } + + use crate::binding_model::ExclusivePipeline; + match ( + expected_bgl.exclusive_pipeline.get().unwrap(), + assigned_bgl.exclusive_pipeline.get().unwrap(), + ) { + (ExclusivePipeline::None, ExclusivePipeline::None) => {} + ( + ExclusivePipeline::Render(e_pipeline), + ExclusivePipeline::Render(a_pipeline), + ) if Weak::ptr_eq(e_pipeline, a_pipeline) => {} + ( + ExclusivePipeline::Compute(e_pipeline), + ExclusivePipeline::Compute(a_pipeline), + ) if Weak::ptr_eq(e_pipeline, a_pipeline) => {} + (expected, assigned) => { + return Err(Error::Incompatible { + expected_bgl: expected_bgl.error_ident(), + assigned_bgl: assigned_bgl.error_ident(), + inner: MultiError::new(core::iter::once( + IncompatibleExclusivePipelines { + expected: expected.to_string(), + assigned: assigned.to_string(), + }, + )) + .unwrap(), + }); + } + } + + #[derive(Clone, Debug, Error)] + enum EntryError { + #[error("Entries with binding {binding} differ in visibility: expected {expected:?}, got {assigned:?}")] + Visibility { + binding: u32, + expected: ShaderStages, + assigned: ShaderStages, + }, + #[error("Entries with binding {binding} differ in type: expected {expected:?}, got {assigned:?}")] + Type { + binding: u32, + expected: BindingType, + assigned: BindingType, + }, + #[error("Entries with binding {binding} differ in count: expected {expected:?}, got {assigned:?}")] + Count { + binding: u32, + expected: Option, + assigned: Option, + }, + #[error("Expected entry with binding {binding} not found in assigned bind group layout")] + ExtraExpected { binding: u32 }, + #[error("Assigned entry with binding {binding} not found in expected bind group layout")] + ExtraAssigned { binding: u32 }, + } + + let mut errors = Vec::new(); + + for (&binding, expected_entry) in expected_bgl.entries.iter() { + if let Some(assigned_entry) = assigned_bgl.entries.get(binding) { + if assigned_entry.visibility != expected_entry.visibility { + errors.push(EntryError::Visibility { + binding, + expected: expected_entry.visibility, + assigned: assigned_entry.visibility, + }); + } + if assigned_entry.ty != expected_entry.ty { + errors.push(EntryError::Type { + binding, + expected: expected_entry.ty, + assigned: assigned_entry.ty, + }); + } + if assigned_entry.count != expected_entry.count { + errors.push(EntryError::Count { + binding, + expected: expected_entry.count, + assigned: assigned_entry.count, + }); + } + } else { + errors.push(EntryError::ExtraExpected { binding }); + } + } + + for (&binding, _) in assigned_bgl.entries.iter() { + if !expected_bgl.entries.contains_key(binding) { + errors.push(EntryError::ExtraAssigned { binding }); + } + } + + Err(Error::Incompatible { + expected_bgl: expected_bgl.error_ident(), + assigned_bgl: assigned_bgl.error_ident(), + inner: MultiError::new(errors.drain(..)).unwrap(), + }) + } + } else { + Err(Error::Missing) + } + } else { + Ok(()) + } + } + } + + #[derive(Debug)] + pub(super) struct BoundBindGroupLayouts { + entries: [Entry; hal::MAX_BIND_GROUPS], + rebind_start: usize, + } + + impl BoundBindGroupLayouts { + pub fn new() -> Self { + Self { + entries: [const { Entry::empty() }; hal::MAX_BIND_GROUPS], + rebind_start: 0, + } + } + + /// Takes the start index of the bind group range to be rebound, and clears it. + pub fn take_rebind_start_index(&mut self) -> usize { + let start = self.rebind_start; + self.rebind_start = self.entries.len(); + start + } + + pub fn update_rebind_start_index(&mut self, start_index: usize) { + self.rebind_start = self.rebind_start.min(start_index); + } + + pub fn update_expectations(&mut self, expectations: &[Option>]) { + let mut rebind_start_index = None; + + for (i, (e, new_expected_bgl)) in self + .entries + .iter_mut() + .zip(expectations.iter().chain(core::iter::repeat(&None))) + .enumerate() + { + let (must_set, must_rebind) = match (&mut e.expected, new_expected_bgl) { + (None, None) => (false, false), + (None, Some(_)) => (true, true), + (Some(_), None) => (true, false), + (Some(old_expected_bgl), Some(new_expected_bgl)) => { + let is_different = !old_expected_bgl.is_equal(new_expected_bgl); + (is_different, is_different) + } + }; + if must_set { + e.expected = new_expected_bgl.clone(); + } + if must_rebind && rebind_start_index.is_none() { + rebind_start_index = Some(i); + } + } + + if let Some(rebind_start_index) = rebind_start_index { + self.update_rebind_start_index(rebind_start_index); + } + } + + pub fn assign(&mut self, index: usize, value: Arc) { + self.entries[index].assigned = Some(value); + self.update_rebind_start_index(index); + } + + pub fn clear(&mut self, index: usize) { + self.entries[index].assigned = None; + } + + pub fn list_active(&self) -> impl Iterator + '_ { + self.entries + .iter() + .enumerate() + .filter_map(|(i, e)| if e.is_active() { Some(i) } else { None }) + } + + pub fn list_valid(&self) -> impl Iterator + '_ { + self.entries + .iter() + .enumerate() + .filter_map(|(i, e)| if e.is_valid() { Some(i) } else { None }) + } + + #[allow(clippy::result_large_err)] + pub fn get_invalid(&self) -> Result<(), (usize, Error)> { + for (index, entry) in self.entries.iter().enumerate() { + entry.check().map_err(|e| (index, e))?; + } + Ok(()) + } + } +} + +#[derive(Clone, Debug, Error)] +pub enum BinderError { + #[error("The current set {pipeline} expects a BindGroup to be set at index {index}")] + MissingBindGroup { + index: usize, + pipeline: ResourceErrorIdent, + }, + #[error("The {assigned_bgl} of current set {assigned_bg} at index {index} is not compatible with the corresponding {expected_bgl} of {pipeline}")] + IncompatibleBindGroup { + expected_bgl: ResourceErrorIdent, + assigned_bgl: ResourceErrorIdent, + assigned_bg: ResourceErrorIdent, + index: usize, + pipeline: ResourceErrorIdent, + #[source] + inner: crate::error::MultiError, + }, +} + +#[derive(Debug)] +struct LateBufferBinding { + binding_index: u32, + shader_expect_size: wgt::BufferAddress, + bound_size: wgt::BufferAddress, +} + +#[derive(Debug, Default)] +struct EntryPayload { + group: Option>, + dynamic_offsets: Vec, + late_buffer_bindings: Vec, + /// Since `LateBufferBinding` may contain information about the bindings + /// not used by the pipeline, we need to know when to stop validating. + late_bindings_effective_count: usize, +} + +impl EntryPayload { + fn reset(&mut self) { + self.group = None; + self.dynamic_offsets.clear(); + self.late_buffer_bindings.clear(); + self.late_bindings_effective_count = 0; + } +} + +#[derive(Debug)] +pub(super) struct Binder { + pub(super) pipeline_layout: Option>, + manager: compat::BoundBindGroupLayouts, + payloads: [EntryPayload; hal::MAX_BIND_GROUPS], +} + +impl Binder { + pub(super) fn new() -> Self { + Self { + pipeline_layout: None, + manager: compat::BoundBindGroupLayouts::new(), + payloads: Default::default(), + } + } + pub(super) fn reset(&mut self) { + self.pipeline_layout = None; + self.manager = compat::BoundBindGroupLayouts::new(); + for payload in self.payloads.iter_mut() { + payload.reset(); + } + } + + /// Returns `true` if the pipeline layout has been changed, i.e. if the + /// new PL was not the same as the old PL. + pub(super) fn change_pipeline_layout<'a>( + &'a mut self, + new: &Arc, + late_sized_buffer_groups: &[LateSizedBufferGroup], + ) -> bool { + self.update_late_buffer_bindings(late_sized_buffer_groups); + + if let Some(old) = self.pipeline_layout.as_ref() { + if old.is_equal(new) { + return false; + } + } + + let old = self.pipeline_layout.replace(new.clone()); + + self.manager.update_expectations(&new.bind_group_layouts); + + if let Some(old) = old { + // root constants are the base compatibility property + if old.immediate_size != new.immediate_size { + self.manager.update_rebind_start_index(0); + } + } + + true + } + + pub(super) fn assign_group<'a>( + &'a mut self, + index: usize, + bind_group: &Arc, + offsets: &[wgt::DynamicOffset], + ) { + let payload = &mut self.payloads[index]; + payload.group = Some(bind_group.clone()); + payload.dynamic_offsets.clear(); + payload.dynamic_offsets.extend_from_slice(offsets); + + // Fill out the actual binding sizes for buffers, + // whose layout doesn't specify `min_binding_size`. + + // Update entries that already exist as the pipeline was bound before the group + // was bound. + for (late_binding, late_info) in payload + .late_buffer_bindings + .iter_mut() + .zip(bind_group.late_buffer_binding_infos.iter()) + { + late_binding.binding_index = late_info.binding_index; + late_binding.bound_size = late_info.size.get(); + } + + // Add new entries for the bindings that were not known when the pipeline was bound. + if bind_group.late_buffer_binding_infos.len() > payload.late_buffer_bindings.len() { + for late_info in + bind_group.late_buffer_binding_infos[payload.late_buffer_bindings.len()..].iter() + { + payload.late_buffer_bindings.push(LateBufferBinding { + binding_index: late_info.binding_index, + shader_expect_size: 0, + bound_size: late_info.size.get(), + }); + } + } + + self.manager.assign(index, bind_group.layout.clone()); + } + + pub(super) fn clear_group(&mut self, index: usize) { + self.payloads[index].reset(); + self.manager.clear(index); + } + + /// Takes the start index of the bind group range to be rebound, and clears it. + pub(super) fn take_rebind_start_index(&mut self) -> usize { + self.manager.take_rebind_start_index() + } + + pub(super) fn list_valid_with_start( + &self, + start: usize, + ) -> impl Iterator, &[wgt::DynamicOffset])> + '_ { + let payloads = &self.payloads; + self.manager + .list_valid() + .filter(move |i| *i >= start) + .map(move |index| { + ( + index, + payloads[index].group.as_ref().unwrap(), + payloads[index].dynamic_offsets.as_slice(), + ) + }) + } + + pub(super) fn list_active(&self) -> impl Iterator> + '_ { + let payloads = &self.payloads; + self.manager + .list_active() + .map(move |index| payloads[index].group.as_ref().unwrap()) + } + + pub(super) fn list_valid( + &self, + ) -> impl Iterator, &[wgt::DynamicOffset])> + '_ { + self.list_valid_with_start(0) + } + + pub(super) fn check_compatibility( + &self, + pipeline: &T, + ) -> Result<(), Box> { + self.manager.get_invalid().map_err(|(index, error)| { + Box::new(match error { + compat::Error::Incompatible { + expected_bgl, + assigned_bgl, + inner, + } => BinderError::IncompatibleBindGroup { + expected_bgl, + assigned_bgl, + assigned_bg: self.payloads[index].group.as_ref().unwrap().error_ident(), + index, + pipeline: pipeline.error_ident(), + inner, + }, + compat::Error::Missing => BinderError::MissingBindGroup { + index, + pipeline: pipeline.error_ident(), + }, + }) + }) + } + + /// Scan active buffer bindings corresponding to layouts without `min_binding_size` specified. + pub(super) fn check_late_buffer_bindings( + &self, + ) -> Result<(), LateMinBufferBindingSizeMismatch> { + for group_index in self.manager.list_active() { + let payload = &self.payloads[group_index]; + for late_binding in + &payload.late_buffer_bindings[..payload.late_bindings_effective_count] + { + if late_binding.bound_size < late_binding.shader_expect_size { + return Err(LateMinBufferBindingSizeMismatch { + group_index: group_index as u32, + binding_index: late_binding.binding_index, + shader_size: late_binding.shader_expect_size, + bound_size: late_binding.bound_size, + }); + } + } + } + Ok(()) + } + + /// This must be called even when a new pipeline has the same layout + /// as the previous one, because different pipelines can have different + /// shader-expected buffer sizes even with identical layouts. + fn update_late_buffer_bindings(&mut self, late_sized_buffer_groups: &[LateSizedBufferGroup]) { + for (payload, late_group) in self.payloads.iter_mut().zip(late_sized_buffer_groups) { + payload.late_bindings_effective_count = late_group.shader_sizes.len(); + + // Update entries that already exist as the bind group was bound before the pipeline + // was bound. + for (late_binding, &shader_expect_size) in payload + .late_buffer_bindings + .iter_mut() + .zip(late_group.shader_sizes.iter()) + { + late_binding.shader_expect_size = shader_expect_size; + } + + // Add new entries for the bindings that were not known when the bind group was bound. + if late_group.shader_sizes.len() > payload.late_buffer_bindings.len() { + for &shader_expect_size in + late_group.shader_sizes[payload.late_buffer_bindings.len()..].iter() + { + payload.late_buffer_bindings.push(LateBufferBinding { + binding_index: 0, + shader_expect_size, + bound_size: 0, + }); + } + } + } + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/bundle.rs b/third_party/wgpu-core-29.0.4-patched/src/command/bundle.rs new file mode 100644 index 00000000..d26dae86 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/bundle.rs @@ -0,0 +1,1735 @@ +/*! Render Bundles + +A render bundle is a prerecorded sequence of commands that can be replayed on a +command encoder with a single call. A single bundle can replayed any number of +times, on different encoders. Constructing a render bundle lets `wgpu` validate +and analyze its commands up front, so that replaying a bundle can be more +efficient than simply re-recording its commands each time. + +Not all commands are available in bundles; for example, a render bundle may not +contain a [`RenderCommand::SetViewport`] command. + +Most of `wgpu`'s backend graphics APIs have something like bundles. For example, +Vulkan calls them "secondary command buffers", and Metal calls them "indirect +command buffers". Although we plan to take advantage of these platform features +at some point in the future, for now `wgpu`'s implementation of render bundles +does not use them: at the hal level, `wgpu` render bundles just replay the +commands. + +## Render Bundle Isolation + +One important property of render bundles is that the draw calls in a render +bundle depend solely on the pipeline and state established within the render +bundle itself. A draw call in a bundle will never use a vertex buffer, say, that +was set in the `RenderPass` before executing the bundle. We call this property +'isolation', in that a render bundle is somewhat isolated from the passes that +use it. + +Render passes are also isolated from the effects of bundles. After executing a +render bundle, a render pass's pipeline, bind groups, and vertex and index +buffers are are unset, so the bundle cannot affect later draw calls in the pass. + +A render pass is not fully isolated from a bundle's effects on immediate data +values. Draw calls following a bundle's execution will see whatever values the +bundle writes to immediate data storage. Setting a pipeline initializes any push +constant storage it could access to zero, and this initialization may also be +visible after bundle execution. + +## Render Bundle Lifecycle + +To create a render bundle: + +1) Create a [`RenderBundleEncoder`] by calling + [`Global::device_create_render_bundle_encoder`][Gdcrbe]. + +2) Record commands in the `RenderBundleEncoder` using functions from the + [`bundle_ffi`] module. + +3) Call [`Global::render_bundle_encoder_finish`][Grbef], which analyzes and cleans up + the command stream and returns a `RenderBundleId`. + +4) Then, any number of times, call [`render_pass_execute_bundles`][wrpeb] to + execute the bundle as part of some render pass. + +## Implementation + +The most complex part of render bundles is the "finish" step, mostly implemented +in [`RenderBundleEncoder::finish`]. This consumes the commands stored in the +encoder's [`BasePass`], while validating everything, tracking the state, +dropping redundant or unnecessary commands, and presenting the results as a new +[`RenderBundle`]. It doesn't actually execute any commands. + +This step also enforces the 'isolation' property mentioned above: every draw +call is checked to ensure that the resources it uses on were established since +the last time the pipeline was set. This means the bundle can be executed +verbatim without any state tracking. + +### Execution + +When the bundle is used in an actual render pass, `RenderBundle::execute` is +called. It goes through the commands and issues them into the native command +buffer. Thanks to isolation, it doesn't track any bind group invalidations or +index format changes. + +[Gdcrbe]: crate::global::Global::device_create_render_bundle_encoder +[Grbef]: crate::global::Global::render_bundle_encoder_finish +[wrpeb]: crate::global::Global::render_pass_execute_bundles +!*/ + +#![allow(clippy::reversed_empty_ranges)] + +use alloc::{ + borrow::{Cow, ToOwned as _}, + string::String, + sync::Arc, + vec::Vec, +}; +use core::{ + convert::Infallible, + num::{NonZeroU32, NonZeroU64}, + ops::Range, +}; + +use arrayvec::ArrayVec; +use thiserror::Error; + +use wgpu_hal::ShouldBeNonZeroExt; +use wgt::error::{ErrorType, WebGpuError}; + +#[cfg(feature = "trace")] +use crate::command::ArcReferences; +use crate::{ + binding_model::{BindError, BindGroup, PipelineLayout}, + command::{ + bind::Binder, BasePass, BindGroupStateChange, ColorAttachmentError, DrawError, + IdReferences, MapPassErr, PassErrorScope, RenderCommand, RenderCommandError, StateChange, + }, + device::{AttachmentData, Device, DeviceError, MissingDownlevelFlags, RenderPassContext}, + hub::Hub, + id, + init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction}, + pipeline::{PipelineFlags, RenderPipeline, VertexStep}, + resource::{ + Buffer, DestroyedResourceError, Fallible, InvalidResourceError, Labeled, ParentDevice, + RawResourceAccess, TrackingData, + }, + resource_log, + snatch::SnatchGuard, + track::RenderBundleScope, + Label, LabelHelpers, +}; + +use super::{pass, render_command::ArcRenderCommand, DrawCommandFamily, DrawKind}; + +/// Describes a [`RenderBundleEncoder`]. +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct RenderBundleEncoderDescriptor<'a> { + /// Debug label of the render bundle encoder. + /// + /// This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// The formats of the color attachments that this render bundle is capable + /// to rendering to. + /// + /// This must match the formats of the color attachments in the + /// renderpass this render bundle is executed in. + pub color_formats: Cow<'a, [Option]>, + /// Information about the depth attachment that this render bundle is + /// capable to rendering to. + /// + /// The format must match the format of the depth attachments in the + /// renderpass this render bundle is executed in. + pub depth_stencil: Option, + /// Sample count this render bundle is capable of rendering to. + /// + /// This must match the pipelines and the renderpasses it is used in. + pub sample_count: u32, + /// If this render bundle will rendering to multiple array layers in the + /// attachments at the same time. + pub multiview: Option, +} + +#[derive(Debug)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct RenderBundleEncoder { + base: BasePass, Infallible>, + parent_id: id::DeviceId, + pub(crate) context: RenderPassContext, + pub(crate) is_depth_read_only: bool, + pub(crate) is_stencil_read_only: bool, + + // Resource binding dedupe state. + #[cfg_attr(feature = "serde", serde(skip))] + current_bind_groups: BindGroupStateChange, + #[cfg_attr(feature = "serde", serde(skip))] + current_pipeline: StateChange, +} + +impl RenderBundleEncoder { + pub fn new( + desc: &RenderBundleEncoderDescriptor, + parent_id: id::DeviceId, + ) -> Result { + let (is_depth_read_only, is_stencil_read_only) = match desc.depth_stencil { + Some(ds) => { + let aspects = hal::FormatAspects::from(ds.format); + ( + !aspects.contains(hal::FormatAspects::DEPTH) || ds.depth_read_only, + !aspects.contains(hal::FormatAspects::STENCIL) || ds.stencil_read_only, + ) + } + // There's no depth/stencil attachment, so these values just don't + // matter. Choose the most accommodating value, to simplify + // validation. + None => (true, true), + }; + + // TODO: should be device.limits.max_color_attachments + let max_color_attachments = hal::MAX_COLOR_ATTACHMENTS; + + //TODO: validate that attachment formats are renderable, + // have expected aspects, support multisampling. + Ok(Self { + base: BasePass::new(&desc.label), + parent_id, + context: RenderPassContext { + attachments: AttachmentData { + colors: if desc.color_formats.len() > max_color_attachments { + return Err(CreateRenderBundleError::ColorAttachment( + ColorAttachmentError::TooMany { + given: desc.color_formats.len(), + limit: max_color_attachments, + }, + )); + } else { + desc.color_formats.iter().cloned().collect() + }, + resolves: ArrayVec::new(), + depth_stencil: desc.depth_stencil.map(|ds| ds.format), + }, + sample_count: { + let sc = desc.sample_count; + if sc == 0 || sc > 32 || !sc.is_power_of_two() { + return Err(CreateRenderBundleError::InvalidSampleCount(sc)); + } + sc + }, + multiview_mask: desc.multiview, + }, + + is_depth_read_only, + is_stencil_read_only, + current_bind_groups: BindGroupStateChange::new(), + current_pipeline: StateChange::new(), + }) + } + + pub fn dummy(parent_id: id::DeviceId) -> Self { + Self { + base: BasePass::new(&None), + parent_id, + context: RenderPassContext { + attachments: AttachmentData { + colors: ArrayVec::new(), + resolves: ArrayVec::new(), + depth_stencil: None, + }, + sample_count: 0, + multiview_mask: None, + }, + is_depth_read_only: false, + is_stencil_read_only: false, + + current_bind_groups: BindGroupStateChange::new(), + current_pipeline: StateChange::new(), + } + } + + pub fn parent(&self) -> id::DeviceId { + self.parent_id + } + + /// Convert this encoder's commands into a [`RenderBundle`]. + /// + /// We want executing a [`RenderBundle`] to be quick, so we take + /// this opportunity to clean up the [`RenderBundleEncoder`]'s + /// command stream and gather metadata about it that will help + /// keep [`ExecuteBundle`] simple and fast. We remove redundant + /// commands (along with their side data), note resource usage, + /// and accumulate buffer and texture initialization actions. + /// + /// [`ExecuteBundle`]: RenderCommand::ExecuteBundle + pub(crate) fn finish( + self, + desc: &RenderBundleDescriptor, + device: &Arc, + hub: &Hub, + ) -> Result, RenderBundleError> { + let scope = PassErrorScope::Bundle; + + device.check_is_valid().map_pass_err(scope)?; + + let bind_group_guard = hub.bind_groups.read(); + let pipeline_guard = hub.render_pipelines.read(); + let buffer_guard = hub.buffers.read(); + + let mut state = State { + trackers: RenderBundleScope::new(), + pipeline: None, + vertex: Default::default(), + index: None, + flat_dynamic_offsets: Vec::new(), + device: device.clone(), + commands: Vec::new(), + buffer_memory_init_actions: Vec::new(), + texture_memory_init_actions: Vec::new(), + next_dynamic_offset: 0, + binder: Binder::new(), + }; + + let indices = &state.device.tracker_indices; + state.trackers.buffers.set_size(indices.buffers.size()); + state.trackers.textures.set_size(indices.textures.size()); + + let base = &self.base; + + for command in &base.commands { + match command { + &RenderCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group, + } => { + let scope = PassErrorScope::SetBindGroup; + set_bind_group( + &mut state, + &bind_group_guard, + &base.dynamic_offsets, + index, + num_dynamic_offsets, + bind_group, + ) + .map_pass_err(scope)?; + } + &RenderCommand::SetPipeline(pipeline) => { + let scope = PassErrorScope::SetPipelineRender; + set_pipeline( + &mut state, + &pipeline_guard, + &self.context, + self.is_depth_read_only, + self.is_stencil_read_only, + pipeline, + ) + .map_pass_err(scope)?; + } + &RenderCommand::SetIndexBuffer { + buffer, + index_format, + offset, + size, + } => { + let scope = PassErrorScope::SetIndexBuffer; + set_index_buffer( + &mut state, + &buffer_guard, + buffer, + index_format, + offset, + size, + ) + .map_pass_err(scope)?; + } + &RenderCommand::SetVertexBuffer { + slot, + buffer, + offset, + size, + } => { + let scope = PassErrorScope::SetVertexBuffer; + set_vertex_buffer(&mut state, &buffer_guard, slot, buffer, offset, size) + .map_pass_err(scope)?; + } + &RenderCommand::SetImmediate { + offset, + size_bytes, + values_offset, + } => { + let scope = PassErrorScope::SetImmediate; + set_immediates(&mut state, offset, size_bytes, values_offset) + .map_pass_err(scope)?; + } + &RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + } => { + let scope = PassErrorScope::Draw { + kind: DrawKind::Draw, + family: DrawCommandFamily::Draw, + }; + draw( + &mut state, + vertex_count, + instance_count, + first_vertex, + first_instance, + ) + .map_pass_err(scope)?; + } + &RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + } => { + let scope = PassErrorScope::Draw { + kind: DrawKind::Draw, + family: DrawCommandFamily::DrawIndexed, + }; + draw_indexed( + &mut state, + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + ) + .map_pass_err(scope)?; + } + &RenderCommand::DrawMeshTasks { + group_count_x, + group_count_y, + group_count_z, + } => { + let scope = PassErrorScope::Draw { + kind: DrawKind::Draw, + family: DrawCommandFamily::DrawMeshTasks, + }; + draw_mesh_tasks(&mut state, group_count_x, group_count_y, group_count_z) + .map_pass_err(scope)?; + } + &RenderCommand::DrawIndirect { + buffer, + offset, + count: 1, + family, + vertex_or_index_limit: None, + instance_limit: None, + } => { + let scope = PassErrorScope::Draw { + kind: DrawKind::DrawIndirect, + family, + }; + multi_draw_indirect(&mut state, &buffer_guard, buffer, offset, family) + .map_pass_err(scope)?; + } + &RenderCommand::DrawIndirect { + count, + vertex_or_index_limit, + instance_limit, + .. + } => { + unreachable!("unexpected (multi-)draw indirect with count {count}, vertex_or_index_limits {vertex_or_index_limit:?}, instance_limit {instance_limit:?} found in a render bundle"); + } + &RenderCommand::MultiDrawIndirectCount { .. } + | &RenderCommand::PushDebugGroup { color: _, len: _ } + | &RenderCommand::InsertDebugMarker { color: _, len: _ } + | &RenderCommand::PopDebugGroup => { + unimplemented!("not supported by a render bundle") + } + // Must check the TIMESTAMP_QUERY_INSIDE_PASSES feature + &RenderCommand::WriteTimestamp { .. } + | &RenderCommand::BeginOcclusionQuery { .. } + | &RenderCommand::EndOcclusionQuery + | &RenderCommand::BeginPipelineStatisticsQuery { .. } + | &RenderCommand::EndPipelineStatisticsQuery => { + unimplemented!("not supported by a render bundle") + } + &RenderCommand::ExecuteBundle(_) + | &RenderCommand::SetBlendConstant(_) + | &RenderCommand::SetStencilReference(_) + | &RenderCommand::SetViewport { .. } + | &RenderCommand::SetScissor(_) => unreachable!("not supported by a render bundle"), + } + } + + let State { + trackers, + flat_dynamic_offsets, + device, + commands, + buffer_memory_init_actions, + texture_memory_init_actions, + .. + } = state; + + let tracker_indices = device.tracker_indices.bundles.clone(); + let discard_hal_labels = device + .instance_flags + .contains(wgt::InstanceFlags::DISCARD_HAL_LABELS); + + let render_bundle = RenderBundle { + base: BasePass { + label: desc.label.as_deref().map(str::to_owned), + error: None, + commands, + dynamic_offsets: flat_dynamic_offsets, + string_data: self.base.string_data, + immediates_data: self.base.immediates_data, + }, + is_depth_read_only: self.is_depth_read_only, + is_stencil_read_only: self.is_stencil_read_only, + device: device.clone(), + used: trackers, + buffer_memory_init_actions, + texture_memory_init_actions, + context: self.context, + label: desc.label.to_string(), + tracking_data: TrackingData::new(tracker_indices), + discard_hal_labels, + }; + + let render_bundle = Arc::new(render_bundle); + + Ok(render_bundle) + } + + pub fn set_index_buffer( + &mut self, + buffer: id::BufferId, + index_format: wgt::IndexFormat, + offset: wgt::BufferAddress, + size: Option, + ) { + self.base.commands.push(RenderCommand::SetIndexBuffer { + buffer, + index_format, + offset, + size, + }); + } +} + +fn set_bind_group( + state: &mut State, + bind_group_guard: &crate::storage::Storage>, + dynamic_offsets: &[u32], + index: u32, + num_dynamic_offsets: usize, + bind_group_id: Option>, +) -> Result<(), RenderBundleErrorInner> { + let max_bind_groups = state.device.limits.max_bind_groups; + if index >= max_bind_groups { + return Err( + RenderCommandError::BindGroupIndexOutOfRange(pass::BindGroupIndexOutOfRange { + index, + max: max_bind_groups, + }) + .into(), + ); + } + + // Identify the next `num_dynamic_offsets` entries from `dynamic_offsets`. + let offsets_range = state.next_dynamic_offset..state.next_dynamic_offset + num_dynamic_offsets; + state.next_dynamic_offset = offsets_range.end; + let offsets = &dynamic_offsets[offsets_range.clone()]; + + let bind_group = bind_group_id.map(|id| bind_group_guard.get(id)); + + if let Some(bind_group) = bind_group { + let bind_group = bind_group.get()?; + bind_group.same_device(&state.device)?; + bind_group.validate_dynamic_bindings(index, offsets)?; + + unsafe { state.trackers.merge_bind_group(&bind_group.used)? }; + let bind_group = state.trackers.bind_groups.insert_single(bind_group); + + state + .binder + .assign_group(index as usize, bind_group, offsets); + } else { + if !offsets.is_empty() { + return Err(RenderBundleErrorInner::Bind( + BindError::DynamicOffsetCountNotZero { + group: index, + actual: offsets.len(), + }, + )); + } + + state.binder.clear_group(index as usize); + } + + Ok(()) +} + +fn set_pipeline( + state: &mut State, + pipeline_guard: &crate::storage::Storage>, + context: &RenderPassContext, + is_depth_read_only: bool, + is_stencil_read_only: bool, + pipeline_id: id::Id, +) -> Result<(), RenderBundleErrorInner> { + let pipeline = pipeline_guard.get(pipeline_id).get()?; + + pipeline.same_device(&state.device)?; + + context + .check_compatible(&pipeline.pass_context, pipeline.as_ref()) + .map_err(RenderCommandError::IncompatiblePipelineTargets)?; + + if pipeline.flags.contains(PipelineFlags::WRITES_DEPTH) && is_depth_read_only { + return Err(RenderCommandError::IncompatibleDepthAccess(pipeline.error_ident()).into()); + } + if pipeline.flags.contains(PipelineFlags::WRITES_STENCIL) && is_stencil_read_only { + return Err(RenderCommandError::IncompatibleStencilAccess(pipeline.error_ident()).into()); + } + + let pipeline_state = PipelineState::new(&pipeline); + + state + .commands + .push(ArcRenderCommand::SetPipeline(pipeline.clone())); + + // If this pipeline uses immediates, zero out their values. + if let Some(cmd) = pipeline_state.zero_immediates() { + state.commands.push(cmd); + } + + state.pipeline = Some(pipeline_state); + + state + .binder + .change_pipeline_layout(&pipeline.layout, &pipeline.late_sized_buffer_groups); + + state.trackers.render_pipelines.insert_single(pipeline); + Ok(()) +} + +// This function is duplicative of `render::set_index_buffer`. +fn set_index_buffer( + state: &mut State, + buffer_guard: &crate::storage::Storage>, + buffer_id: id::Id, + index_format: wgt::IndexFormat, + offset: u64, + size: Option, +) -> Result<(), RenderBundleErrorInner> { + let buffer = buffer_guard.get(buffer_id).get()?; + + state + .trackers + .buffers + .merge_single(&buffer, wgt::BufferUses::INDEX)?; + + buffer.same_device(&state.device)?; + buffer.check_usage(wgt::BufferUsages::INDEX)?; + + if !offset.is_multiple_of(u64::try_from(index_format.byte_size()).unwrap()) { + return Err(RenderCommandError::UnalignedIndexBuffer { + offset, + alignment: index_format.byte_size(), + } + .into()); + } + let end = offset + buffer.resolve_binding_size(offset, size)?; + + state + .buffer_memory_init_actions + .extend(buffer.initialization_status.read().create_action( + &buffer, + offset..end.get(), + MemoryInitKind::NeedsInitializedMemory, + )); + state.set_index_buffer(buffer, index_format, offset..end.get()); + Ok(()) +} + +// This function is duplicative of `render::set_vertex_buffer`. +fn set_vertex_buffer( + state: &mut State, + buffer_guard: &crate::storage::Storage>, + slot: u32, + buffer_id: id::Id, + offset: u64, + size: Option, +) -> Result<(), RenderBundleErrorInner> { + let max_vertex_buffers = state.device.limits.max_vertex_buffers; + if slot >= max_vertex_buffers { + return Err(RenderCommandError::VertexBufferIndexOutOfRange { + index: slot, + max: max_vertex_buffers, + } + .into()); + } + + let buffer = buffer_guard.get(buffer_id).get()?; + + state + .trackers + .buffers + .merge_single(&buffer, wgt::BufferUses::VERTEX)?; + + buffer.same_device(&state.device)?; + buffer.check_usage(wgt::BufferUsages::VERTEX)?; + + if !offset.is_multiple_of(wgt::VERTEX_ALIGNMENT) { + return Err(RenderCommandError::UnalignedVertexBuffer { slot, offset }.into()); + } + let end = offset + buffer.resolve_binding_size(offset, size)?; + + state + .buffer_memory_init_actions + .extend(buffer.initialization_status.read().create_action( + &buffer, + offset..end.get(), + MemoryInitKind::NeedsInitializedMemory, + )); + state.vertex[slot as usize] = Some(VertexState::new(buffer, offset..end.get())); + Ok(()) +} + +fn set_immediates( + state: &mut State, + offset: u32, + size_bytes: u32, + values_offset: Option, +) -> Result<(), RenderBundleErrorInner> { + let pipeline_state = state.pipeline()?; + + pipeline_state + .pipeline + .layout + .validate_immediates_ranges(offset, size_bytes)?; + + state.commands.push(ArcRenderCommand::SetImmediate { + offset, + size_bytes, + values_offset, + }); + Ok(()) +} + +fn draw( + state: &mut State, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, +) -> Result<(), RenderBundleErrorInner> { + state.is_ready(DrawCommandFamily::Draw)?; + let pipeline = state.pipeline()?; + + let vertex_limits = super::VertexLimits::new(state.vertex_buffer_sizes(), &pipeline.steps); + vertex_limits.validate_vertex_limit(first_vertex, vertex_count)?; + vertex_limits.validate_instance_limit(first_instance, instance_count)?; + + if instance_count > 0 && vertex_count > 0 { + state.flush_vertices(); + state.flush_bindings(); + state.commands.push(ArcRenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + }); + } + Ok(()) +} + +fn draw_indexed( + state: &mut State, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, +) -> Result<(), RenderBundleErrorInner> { + state.is_ready(DrawCommandFamily::DrawIndexed)?; + let pipeline = state.pipeline()?; + + let index = state.index.as_ref().unwrap(); + + let vertex_limits = super::VertexLimits::new(state.vertex_buffer_sizes(), &pipeline.steps); + + let last_index = first_index as u64 + index_count as u64; + let index_limit = index.limit(); + if last_index > index_limit { + return Err(DrawError::IndexBeyondLimit { + last_index, + index_limit, + } + .into()); + } + vertex_limits.validate_instance_limit(first_instance, instance_count)?; + + if instance_count > 0 && index_count > 0 { + state.flush_index(); + state.flush_vertices(); + state.flush_bindings(); + state.commands.push(ArcRenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + }); + } + Ok(()) +} + +fn draw_mesh_tasks( + state: &mut State, + group_count_x: u32, + group_count_y: u32, + group_count_z: u32, +) -> Result<(), RenderBundleErrorInner> { + state.is_ready(DrawCommandFamily::DrawMeshTasks)?; + + let groups_size_limit = state.device.limits.max_task_mesh_workgroups_per_dimension; + let max_groups = state.device.limits.max_task_mesh_workgroup_total_count; + if group_count_x > groups_size_limit + || group_count_y > groups_size_limit + || group_count_z > groups_size_limit + || group_count_x * group_count_y * group_count_z > max_groups + { + return Err(RenderBundleErrorInner::Draw(DrawError::InvalidGroupSize { + current: [group_count_x, group_count_y, group_count_z], + limit: groups_size_limit, + max_total: max_groups, + })); + } + + if group_count_x > 0 && group_count_y > 0 && group_count_z > 0 { + state.flush_bindings(); + state.commands.push(ArcRenderCommand::DrawMeshTasks { + group_count_x, + group_count_y, + group_count_z, + }); + } + Ok(()) +} + +fn multi_draw_indirect( + state: &mut State, + buffer_guard: &crate::storage::Storage>, + buffer_id: id::Id, + offset: u64, + family: DrawCommandFamily, +) -> Result<(), RenderBundleErrorInner> { + state.is_ready(family)?; + state + .device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?; + + let pipeline = state.pipeline()?; + + let buffer = buffer_guard.get(buffer_id).get()?; + + buffer.same_device(&state.device)?; + buffer.check_usage(wgt::BufferUsages::INDIRECT)?; + + let vertex_limits = super::VertexLimits::new(state.vertex_buffer_sizes(), &pipeline.steps); + + let stride = super::get_src_stride_of_indirect_args(family); + state + .buffer_memory_init_actions + .extend(buffer.initialization_status.read().create_action( + &buffer, + offset..(offset + stride), + MemoryInitKind::NeedsInitializedMemory, + )); + + let vertex_or_index_limit = if family == DrawCommandFamily::DrawIndexed { + let index = state.index.as_mut().unwrap(); + state.commands.extend(index.flush()); + index.limit() + } else { + vertex_limits.vertex_limit + }; + let instance_limit = vertex_limits.instance_limit; + + let buffer_uses = if state.device.indirect_validation.is_some() { + wgt::BufferUses::STORAGE_READ_ONLY + } else { + wgt::BufferUses::INDIRECT + }; + + state.trackers.buffers.merge_single(&buffer, buffer_uses)?; + + state.flush_vertices(); + state.flush_bindings(); + state.commands.push(ArcRenderCommand::DrawIndirect { + buffer, + offset, + count: 1, + family, + + vertex_or_index_limit: Some(vertex_or_index_limit), + instance_limit: Some(instance_limit), + }); + Ok(()) +} + +/// Error type returned from `RenderBundleEncoder::new` if the sample count is invalid. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateRenderBundleError { + #[error(transparent)] + ColorAttachment(#[from] ColorAttachmentError), + #[error("Invalid number of samples {0}")] + InvalidSampleCount(u32), +} + +impl WebGpuError for CreateRenderBundleError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::ColorAttachment(e) => e.webgpu_error_type(), + Self::InvalidSampleCount(_) => ErrorType::Validation, + } + } +} + +/// Error type returned from `RenderBundleEncoder::new` if the sample count is invalid. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ExecutionError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + #[error("Using {0} in a render bundle is not implemented")] + Unimplemented(&'static str), +} + +pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor>; + +//Note: here, `RenderBundle` is just wrapping a raw stream of render commands. +// The plan is to back it by an actual Vulkan secondary buffer, D3D12 Bundle, +// or Metal indirect command buffer. +/// cbindgen:ignore +#[derive(Debug)] +pub struct RenderBundle { + // Normalized command stream. It can be executed verbatim, + // without re-binding anything on the pipeline change. + base: BasePass, + pub(super) is_depth_read_only: bool, + pub(super) is_stencil_read_only: bool, + pub(crate) device: Arc, + pub(crate) used: RenderBundleScope, + pub(super) buffer_memory_init_actions: Vec, + pub(super) texture_memory_init_actions: Vec, + pub(super) context: RenderPassContext, + /// The `label` from the descriptor used to create the resource. + label: String, + pub(crate) tracking_data: TrackingData, + discard_hal_labels: bool, +} + +impl Drop for RenderBundle { + fn drop(&mut self) { + resource_log!("Drop {}", self.error_ident()); + } +} + +#[cfg(send_sync)] +unsafe impl Send for RenderBundle {} +#[cfg(send_sync)] +unsafe impl Sync for RenderBundle {} + +impl RenderBundle { + #[cfg(feature = "trace")] + pub(crate) fn to_base_pass(&self) -> BasePass, Infallible> { + self.base.clone() + } + + /// Actually encode the contents into a native command buffer. + /// + /// This is partially duplicating the logic of `render_pass_end`. + /// However the point of this function is to be lighter, since we already had + /// a chance to go through the commands in `render_bundle_encoder_finish`. + /// + /// Note that the function isn't expected to fail, generally. + /// All the validation has already been done by this point. + /// The only failure condition is if some of the used buffers are destroyed. + pub(super) unsafe fn execute( + &self, + raw: &mut dyn hal::DynCommandEncoder, + indirect_draw_validation_resources: &mut crate::indirect_validation::DrawResources, + indirect_draw_validation_batcher: &mut crate::indirect_validation::DrawBatcher, + snatch_guard: &SnatchGuard, + ) -> Result<(), ExecutionError> { + let mut offsets = self.base.dynamic_offsets.as_slice(); + let mut pipeline_layout = None::>; + if !self.discard_hal_labels { + if let Some(ref label) = self.base.label { + unsafe { raw.begin_debug_marker(label) }; + } + } + + use ArcRenderCommand as Cmd; + for command in self.base.commands.iter() { + match command { + Cmd::SetBindGroup { + index, + num_dynamic_offsets, + bind_group, + } => { + let raw_bg = bind_group.as_ref().unwrap().try_raw(snatch_guard)?; + unsafe { + raw.set_bind_group( + pipeline_layout.as_ref().unwrap().raw(), + *index, + raw_bg, + &offsets[..*num_dynamic_offsets], + ) + }; + offsets = &offsets[*num_dynamic_offsets..]; + } + Cmd::SetPipeline(pipeline) => { + unsafe { raw.set_render_pipeline(pipeline.raw()) }; + + pipeline_layout = Some(pipeline.layout.clone()); + } + Cmd::SetIndexBuffer { + buffer, + index_format, + offset, + size, + } => { + let buffer = buffer.try_raw(snatch_guard)?; + // SAFETY: The binding size was checked against the buffer size + // in `set_index_buffer` and again in `IndexState::flush`. + let bb = hal::BufferBinding::new_unchecked(buffer, *offset, *size); + unsafe { raw.set_index_buffer(bb, *index_format) }; + } + Cmd::SetVertexBuffer { + slot, + buffer, + offset, + size, + } => { + let buffer = buffer.try_raw(snatch_guard)?; + // SAFETY: The binding size was checked against the buffer size + // in `set_vertex_buffer` and again in `VertexState::flush`. + let bb = hal::BufferBinding::new_unchecked(buffer, *offset, *size); + unsafe { raw.set_vertex_buffer(*slot, bb) }; + } + Cmd::SetImmediate { + offset, + size_bytes, + values_offset, + } => { + let pipeline_layout = pipeline_layout.as_ref().unwrap(); + + if let Some(values_offset) = *values_offset { + let values_end_offset = + (values_offset + size_bytes / wgt::IMMEDIATE_DATA_ALIGNMENT) as usize; + let data_slice = + &self.base.immediates_data[(values_offset as usize)..values_end_offset]; + + unsafe { raw.set_immediates(pipeline_layout.raw(), *offset, data_slice) } + } else { + super::immediates_clear( + *offset, + *size_bytes, + |clear_offset, clear_data| { + unsafe { + raw.set_immediates( + pipeline_layout.raw(), + clear_offset, + clear_data, + ) + }; + }, + ); + } + } + Cmd::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + } => { + unsafe { + raw.draw( + *first_vertex, + *vertex_count, + *first_instance, + *instance_count, + ) + }; + } + Cmd::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + } => { + unsafe { + raw.draw_indexed( + *first_index, + *index_count, + *base_vertex, + *first_instance, + *instance_count, + ) + }; + } + Cmd::DrawMeshTasks { + group_count_x, + group_count_y, + group_count_z, + } => unsafe { + raw.draw_mesh_tasks(*group_count_x, *group_count_y, *group_count_z); + }, + Cmd::DrawIndirect { + buffer, + offset, + count: 1, + family, + + vertex_or_index_limit, + instance_limit, + } => { + let (buffer, offset) = if self.device.indirect_validation.is_some() { + let (dst_resource_index, offset) = indirect_draw_validation_batcher.add( + indirect_draw_validation_resources, + &self.device, + buffer, + *offset, + *family, + vertex_or_index_limit + .expect("finalized render bundle missing vertex_or_index_limit"), + instance_limit.expect("finalized render bundle missing instance_limit"), + )?; + + let dst_buffer = + indirect_draw_validation_resources.get_dst_buffer(dst_resource_index); + (dst_buffer, offset) + } else { + (buffer.try_raw(snatch_guard)?, *offset) + }; + match family { + DrawCommandFamily::Draw => unsafe { raw.draw_indirect(buffer, offset, 1) }, + DrawCommandFamily::DrawIndexed => unsafe { + raw.draw_indexed_indirect(buffer, offset, 1) + }, + DrawCommandFamily::DrawMeshTasks => unsafe { + raw.draw_mesh_tasks_indirect(buffer, offset, 1); + }, + } + } + Cmd::DrawIndirect { .. } | Cmd::MultiDrawIndirectCount { .. } => { + return Err(ExecutionError::Unimplemented("multi-draw-indirect")) + } + Cmd::PushDebugGroup { .. } | Cmd::InsertDebugMarker { .. } | Cmd::PopDebugGroup => { + return Err(ExecutionError::Unimplemented("debug-markers")) + } + Cmd::WriteTimestamp { .. } + | Cmd::BeginOcclusionQuery { .. } + | Cmd::EndOcclusionQuery + | Cmd::BeginPipelineStatisticsQuery { .. } + | Cmd::EndPipelineStatisticsQuery => { + return Err(ExecutionError::Unimplemented("queries")) + } + Cmd::ExecuteBundle(_) + | Cmd::SetBlendConstant(_) + | Cmd::SetStencilReference(_) + | Cmd::SetViewport { .. } + | Cmd::SetScissor(_) => unreachable!(), + } + } + + if !self.discard_hal_labels { + if let Some(_) = self.base.label { + unsafe { raw.end_debug_marker() }; + } + } + + Ok(()) + } +} + +crate::impl_resource_type!(RenderBundle); +crate::impl_labeled!(RenderBundle); +crate::impl_parent_device!(RenderBundle); +crate::impl_storage_item!(RenderBundle); +crate::impl_trackable!(RenderBundle); + +/// A render bundle's current index buffer state. +/// +/// [`RenderBundleEncoder::finish`] records the currently set index buffer here, +/// and calls [`State::flush_index`] before any indexed draw command to produce +/// a `SetIndexBuffer` command if one is necessary. +/// +/// Binding ranges must be validated against the size of the buffer before +/// being stored in `IndexState`. +#[derive(Debug)] +struct IndexState { + buffer: Arc, + format: wgt::IndexFormat, + range: Range, + is_dirty: bool, +} + +impl IndexState { + /// Return the number of entries in the current index buffer. + /// + /// Panic if no index buffer has been set. + fn limit(&self) -> u64 { + let bytes_per_index = self.format.byte_size() as u64; + + (self.range.end - self.range.start) / bytes_per_index + } + + /// Generate a `SetIndexBuffer` command to prepare for an indexed draw + /// command, if needed. + fn flush(&mut self) -> Option { + // This was all checked before, but let's check again just in case. + let binding_size = self + .range + .end + .checked_sub(self.range.start) + .filter(|_| self.range.end <= self.buffer.size) + .expect("index range must be contained in buffer"); + + if self.is_dirty { + self.is_dirty = false; + Some(ArcRenderCommand::SetIndexBuffer { + buffer: self.buffer.clone(), + index_format: self.format, + offset: self.range.start, + size: NonZeroU64::new(binding_size), + }) + } else { + None + } + } +} + +/// The state of a single vertex buffer slot during render bundle encoding. +/// +/// [`RenderBundleEncoder::finish`] uses this to drop redundant +/// `SetVertexBuffer` commands from the final [`RenderBundle`]. It +/// records one vertex buffer slot's state changes here, and then +/// calls this type's [`flush`] method just before any draw command to +/// produce a `SetVertexBuffer` commands if one is necessary. +/// +/// Binding ranges must be validated against the size of the buffer before +/// being stored in `VertexState`. +/// +/// [`flush`]: IndexState::flush +#[derive(Debug)] +struct VertexState { + buffer: Arc, + range: Range, + is_dirty: bool, +} + +impl VertexState { + /// Create a new `VertexState`. + /// + /// The `range` must be contained within `buffer`. + fn new(buffer: Arc, range: Range) -> Self { + Self { + buffer, + range, + is_dirty: true, + } + } + + /// Generate a `SetVertexBuffer` command for this slot, if necessary. + /// + /// `slot` is the index of the vertex buffer slot that `self` tracks. + fn flush(&mut self, slot: u32) -> Option { + let binding_size = self + .range + .end + .checked_sub(self.range.start) + .filter(|_| self.range.end <= self.buffer.size) + .expect("vertex range must be contained in buffer"); + + if self.is_dirty { + self.is_dirty = false; + Some(ArcRenderCommand::SetVertexBuffer { + slot, + buffer: self.buffer.clone(), + offset: self.range.start, + size: NonZeroU64::new(binding_size), + }) + } else { + None + } + } +} + +/// The bundle's current pipeline, and some cached information needed for validation. +struct PipelineState { + /// The pipeline + pipeline: Arc, + + /// How this pipeline's vertex shader traverses each vertex buffer, indexed + /// by vertex buffer slot number. + steps: Vec, + + /// Size of the immediate data ranges this pipeline uses. Copied from the pipeline layout. + immediate_size: u32, +} + +impl PipelineState { + fn new(pipeline: &Arc) -> Self { + Self { + pipeline: pipeline.clone(), + steps: pipeline.vertex_steps.to_vec(), + immediate_size: pipeline.layout.immediate_size, + } + } + + /// Return a sequence of commands to zero the immediate data ranges this + /// pipeline uses. If no initialization is necessary, return `None`. + fn zero_immediates(&self) -> Option { + if self.immediate_size == 0 { + return None; + } + + Some(ArcRenderCommand::SetImmediate { + offset: 0, + size_bytes: self.immediate_size, + values_offset: None, + }) + } +} + +/// State for analyzing and cleaning up bundle command streams. +/// +/// To minimize state updates, [`RenderBundleEncoder::finish`] +/// actually just applies commands like [`SetBindGroup`] and +/// [`SetIndexBuffer`] to the simulated state stored here, and then +/// calls the `flush_foo` methods before draw calls to produce the +/// update commands we actually need. +/// +/// [`SetBindGroup`]: RenderCommand::SetBindGroup +/// [`SetIndexBuffer`]: RenderCommand::SetIndexBuffer +struct State { + /// Resources used by this bundle. This will become [`RenderBundle::used`]. + trackers: RenderBundleScope, + + /// The currently set pipeline, if any. + pipeline: Option, + + /// The state of each vertex buffer slot. + vertex: [Option; hal::MAX_VERTEX_BUFFERS], + + /// The current index buffer, if one has been set. We flush this state + /// before indexed draw commands. + index: Option, + + /// Dynamic offset values used by the cleaned-up command sequence. + /// + /// This becomes the final [`RenderBundle`]'s [`BasePass`]'s + /// [`dynamic_offsets`] list. + /// + /// [`dynamic_offsets`]: BasePass::dynamic_offsets + flat_dynamic_offsets: Vec, + + device: Arc, + commands: Vec, + buffer_memory_init_actions: Vec, + texture_memory_init_actions: Vec, + next_dynamic_offset: usize, + binder: Binder, +} + +impl State { + /// Return the current pipeline state. Return an error if none is set. + fn pipeline(&self) -> Result<&PipelineState, RenderBundleErrorInner> { + self.pipeline + .as_ref() + .ok_or(DrawError::MissingPipeline(pass::MissingPipeline).into()) + } + + /// Set the bundle's current index buffer and its associated parameters. + fn set_index_buffer( + &mut self, + buffer: Arc, + format: wgt::IndexFormat, + range: Range, + ) { + match self.index { + Some(ref current) + if current.buffer.is_equal(&buffer) + && current.format == format + && current.range == range => + { + return + } + _ => (), + } + + self.index = Some(IndexState { + buffer, + format, + range, + is_dirty: true, + }); + } + + /// Generate a `SetIndexBuffer` command to prepare for an indexed draw + /// command, if needed. + fn flush_index(&mut self) { + let commands = self.index.as_mut().and_then(|index| index.flush()); + self.commands.extend(commands); + } + + fn flush_vertices(&mut self) { + let commands = self + .vertex + .iter_mut() + .enumerate() + .flat_map(|(i, vs)| vs.as_mut().and_then(|vs| vs.flush(i as u32))); + self.commands.extend(commands); + } + + /// Validation for a draw command. + /// + /// This should be further deduplicated with similar validation on render/compute passes. + fn is_ready(&mut self, family: DrawCommandFamily) -> Result<(), DrawError> { + if let Some(pipeline) = self.pipeline.as_ref() { + self.binder + .check_compatibility(pipeline.pipeline.as_ref())?; + self.binder.check_late_buffer_bindings()?; + + if family == DrawCommandFamily::DrawIndexed { + let pipeline = &pipeline.pipeline; + let index_format = match &self.index { + Some(index) => index.format, + None => return Err(DrawError::MissingIndexBuffer), + }; + + if pipeline.topology.is_strip() && pipeline.strip_index_format != Some(index_format) + { + return Err(DrawError::UnmatchedStripIndexFormat { + pipeline: pipeline.error_ident(), + strip_index_format: pipeline.strip_index_format, + buffer_format: index_format, + }); + } + } + + Ok(()) + } else { + Err(DrawError::MissingPipeline(pass::MissingPipeline)) + } + } + + /// Generate `SetBindGroup` commands for any bind groups that need to be updated. + /// + /// This should be further deduplicated with similar code on render/compute passes. + fn flush_bindings(&mut self) { + let start = self.binder.take_rebind_start_index(); + let entries = self.binder.list_valid_with_start(start); + + self.commands + .extend(entries.map(|(i, bind_group, dynamic_offsets)| { + self.buffer_memory_init_actions + .extend_from_slice(&bind_group.used_buffer_ranges); + self.texture_memory_init_actions + .extend_from_slice(&bind_group.used_texture_ranges); + + self.flat_dynamic_offsets.extend_from_slice(dynamic_offsets); + + ArcRenderCommand::SetBindGroup { + index: i.try_into().unwrap(), + bind_group: Some(bind_group.clone()), + num_dynamic_offsets: dynamic_offsets.len(), + } + })); + } + + fn vertex_buffer_sizes(&self) -> impl Iterator> + '_ { + self.vertex + .iter() + .map(|vbs| vbs.as_ref().map(|vbs| vbs.range.end - vbs.range.start)) + } +} + +/// Error encountered when finishing recording a render bundle. +#[derive(Clone, Debug, Error)] +pub enum RenderBundleErrorInner { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + RenderCommand(RenderCommandError), + #[error(transparent)] + Draw(#[from] DrawError), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), + #[error(transparent)] + Bind(#[from] BindError), + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), +} + +impl From for RenderBundleErrorInner +where + T: Into, +{ + fn from(t: T) -> Self { + Self::RenderCommand(t.into()) + } +} + +/// Error encountered when finishing recording a render bundle. +#[derive(Clone, Debug, Error)] +#[error("{scope}")] +pub struct RenderBundleError { + pub scope: PassErrorScope, + #[source] + inner: RenderBundleErrorInner, +} + +impl WebGpuError for RenderBundleError { + fn webgpu_error_type(&self) -> ErrorType { + let Self { scope: _, inner } = self; + match inner { + RenderBundleErrorInner::Device(e) => e.webgpu_error_type(), + RenderBundleErrorInner::RenderCommand(e) => e.webgpu_error_type(), + RenderBundleErrorInner::Draw(e) => e.webgpu_error_type(), + RenderBundleErrorInner::MissingDownlevelFlags(e) => e.webgpu_error_type(), + RenderBundleErrorInner::Bind(e) => e.webgpu_error_type(), + RenderBundleErrorInner::InvalidResource(e) => e.webgpu_error_type(), + } + } +} + +impl RenderBundleError { + pub fn from_device_error(e: DeviceError) -> Self { + Self { + scope: PassErrorScope::Bundle, + inner: e.into(), + } + } +} + +impl MapPassErr for E +where + E: Into, +{ + fn map_pass_err(self, scope: PassErrorScope) -> RenderBundleError { + RenderBundleError { + scope, + inner: self.into(), + } + } +} + +pub mod bundle_ffi { + use super::{RenderBundleEncoder, RenderCommand}; + use crate::{command::DrawCommandFamily, id, RawString}; + use core::{convert::TryInto, slice}; + use wgt::{BufferAddress, BufferSize, DynamicOffset, IndexFormat}; + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `offset_length` elements. + pub unsafe fn wgpu_render_bundle_set_bind_group( + bundle: &mut RenderBundleEncoder, + index: u32, + bind_group_id: Option, + offsets: *const DynamicOffset, + offset_length: usize, + ) { + let offsets = unsafe { slice::from_raw_parts(offsets, offset_length) }; + + let redundant = bundle.current_bind_groups.set_and_check_redundant( + bind_group_id, + index, + &mut bundle.base.dynamic_offsets, + offsets, + ); + + if redundant { + return; + } + + bundle.base.commands.push(RenderCommand::SetBindGroup { + index, + num_dynamic_offsets: offset_length, + bind_group: bind_group_id, + }); + } + + pub fn wgpu_render_bundle_set_pipeline( + bundle: &mut RenderBundleEncoder, + pipeline_id: id::RenderPipelineId, + ) { + if bundle.current_pipeline.set_and_check_redundant(pipeline_id) { + return; + } + + bundle + .base + .commands + .push(RenderCommand::SetPipeline(pipeline_id)); + } + + pub fn wgpu_render_bundle_set_vertex_buffer( + bundle: &mut RenderBundleEncoder, + slot: u32, + buffer_id: id::BufferId, + offset: BufferAddress, + size: Option, + ) { + bundle.base.commands.push(RenderCommand::SetVertexBuffer { + slot, + buffer: buffer_id, + offset, + size, + }); + } + + pub fn wgpu_render_bundle_set_index_buffer( + encoder: &mut RenderBundleEncoder, + buffer: id::BufferId, + index_format: IndexFormat, + offset: BufferAddress, + size: Option, + ) { + encoder.set_index_buffer(buffer, index_format, offset, size); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given pointer is + /// valid for `data` elements. + pub unsafe fn wgpu_render_bundle_set_immediates( + pass: &mut RenderBundleEncoder, + offset: u32, + size_bytes: u32, + data: *const u8, + ) { + assert_eq!( + offset & (wgt::IMMEDIATE_DATA_ALIGNMENT - 1), + 0, + "Immediate data offset must be aligned to 4 bytes." + ); + assert_eq!( + size_bytes & (wgt::IMMEDIATE_DATA_ALIGNMENT - 1), + 0, + "Immediate data size must be aligned to 4 bytes." + ); + let data_slice = unsafe { slice::from_raw_parts(data, size_bytes as usize) }; + let value_offset = pass.base.immediates_data.len().try_into().expect( + "Ran out of immediate data space. Don't set 4gb of immediates per RenderBundle.", + ); + + pass.base.immediates_data.extend( + data_slice + .chunks_exact(wgt::IMMEDIATE_DATA_ALIGNMENT as usize) + .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), + ); + + pass.base.commands.push(RenderCommand::SetImmediate { + offset, + size_bytes, + values_offset: Some(value_offset), + }); + } + + pub fn wgpu_render_bundle_draw( + bundle: &mut RenderBundleEncoder, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, + ) { + bundle.base.commands.push(RenderCommand::Draw { + vertex_count, + instance_count, + first_vertex, + first_instance, + }); + } + + pub fn wgpu_render_bundle_draw_indexed( + bundle: &mut RenderBundleEncoder, + index_count: u32, + instance_count: u32, + first_index: u32, + base_vertex: i32, + first_instance: u32, + ) { + bundle.base.commands.push(RenderCommand::DrawIndexed { + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + }); + } + + pub fn wgpu_render_bundle_draw_indirect( + bundle: &mut RenderBundleEncoder, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + bundle.base.commands.push(RenderCommand::DrawIndirect { + buffer: buffer_id, + offset, + count: 1, + family: DrawCommandFamily::Draw, + vertex_or_index_limit: None, + instance_limit: None, + }); + } + + pub fn wgpu_render_bundle_draw_indexed_indirect( + bundle: &mut RenderBundleEncoder, + buffer_id: id::BufferId, + offset: BufferAddress, + ) { + bundle.base.commands.push(RenderCommand::DrawIndirect { + buffer: buffer_id, + offset, + count: 1, + family: DrawCommandFamily::DrawIndexed, + vertex_or_index_limit: None, + instance_limit: None, + }); + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given `label` + /// is a valid null-terminated string. + pub unsafe fn wgpu_render_bundle_push_debug_group( + _bundle: &mut RenderBundleEncoder, + _label: RawString, + ) { + //TODO + } + + pub fn wgpu_render_bundle_pop_debug_group(_bundle: &mut RenderBundleEncoder) { + //TODO + } + + /// # Safety + /// + /// This function is unsafe as there is no guarantee that the given `label` + /// is a valid null-terminated string. + pub unsafe fn wgpu_render_bundle_insert_debug_marker( + _bundle: &mut RenderBundleEncoder, + _label: RawString, + ) { + //TODO + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/clear.rs b/third_party/wgpu-core-29.0.4-patched/src/command/clear.rs new file mode 100644 index 00000000..b8c1d9fb --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/clear.rs @@ -0,0 +1,547 @@ +use alloc::{sync::Arc, vec::Vec}; +use core::ops::Range; + +use crate::{ + api_log, + command::{encoder::EncodingState, ArcCommand, EncoderStateError}, + device::{DeviceError, MissingFeatures}, + get_lowest_common_denom, + global::Global, + hal_label, + id::{BufferId, CommandEncoderId, TextureId}, + init_tracker::{MemoryInitKind, TextureInitRange}, + resource::{ + Buffer, DestroyedResourceError, InvalidResourceError, Labeled, MissingBufferUsageError, + ParentDevice, RawResourceAccess, ResourceErrorIdent, Texture, TextureClearMode, + }, + snatch::SnatchGuard, + track::TextureTrackerSetSingle, +}; + +use thiserror::Error; +use wgt::{ + error::{ErrorType, WebGpuError}, + math::align_to, + BufferAddress, BufferUsages, ImageSubresourceRange, TextureAspect, TextureSelector, +}; + +/// Error encountered while attempting a clear. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ClearError { + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error("{0} can not be cleared")] + NoValidTextureClearMode(ResourceErrorIdent), + #[error("Buffer clear size {0:?} is not a multiple of `COPY_BUFFER_ALIGNMENT`")] + UnalignedFillSize(BufferAddress), + #[error("Buffer offset {0:?} is not a multiple of `COPY_BUFFER_ALIGNMENT`")] + UnalignedBufferOffset(BufferAddress), + #[error("Clear starts at offset {start_offset} with size of {requested_size}, but these added together exceed `u64::MAX`")] + OffsetPlusSizeExceeds64BitBounds { + start_offset: BufferAddress, + requested_size: BufferAddress, + }, + #[error("Clear of {start_offset}..{end_offset} would end up overrunning the bounds of the buffer of size {buffer_size}")] + BufferOverrun { + start_offset: BufferAddress, + end_offset: BufferAddress, + buffer_size: BufferAddress, + }, + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + #[error("Texture lacks the aspects that were specified in the image subresource range. Texture with format {texture_format:?}, specified was {subresource_range_aspects:?}")] + MissingTextureAspect { + texture_format: wgt::TextureFormat, + subresource_range_aspects: TextureAspect, + }, + #[error("Image subresource level range is outside of the texture's level range. texture range is {texture_level_range:?}, \ +whereas subesource range specified start {subresource_base_mip_level} and count {subresource_mip_level_count:?}")] + InvalidTextureLevelRange { + texture_level_range: Range, + subresource_base_mip_level: u32, + subresource_mip_level_count: Option, + }, + #[error("Image subresource layer range is outside of the texture's layer range. texture range is {texture_layer_range:?}, \ +whereas subesource range specified start {subresource_base_array_layer} and count {subresource_array_layer_count:?}")] + InvalidTextureLayerRange { + texture_layer_range: Range, + subresource_base_array_layer: u32, + subresource_array_layer_count: Option, + }, + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + EncoderState(#[from] EncoderStateError), + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), +} + +impl WebGpuError for ClearError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::DestroyedResource(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + Self::MissingBufferUsage(e) => e.webgpu_error_type(), + Self::Device(e) => e.webgpu_error_type(), + Self::EncoderState(e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::NoValidTextureClearMode(..) + | Self::UnalignedFillSize(..) + | Self::UnalignedBufferOffset(..) + | Self::OffsetPlusSizeExceeds64BitBounds { .. } + | Self::BufferOverrun { .. } + | Self::MissingTextureAspect { .. } + | Self::InvalidTextureLevelRange { .. } + | Self::InvalidTextureLayerRange { .. } => ErrorType::Validation, + } + } +} + +impl Global { + pub fn command_encoder_clear_buffer( + &self, + command_encoder_id: CommandEncoderId, + dst: BufferId, + offset: BufferAddress, + size: Option, + ) -> Result<(), EncoderStateError> { + profiling::scope!("CommandEncoder::clear_buffer"); + api_log!("CommandEncoder::clear_buffer {dst:?}"); + + let hub = &self.hub; + + let cmd_enc = hub.command_encoders.get(command_encoder_id); + let mut cmd_buf_data = cmd_enc.data.lock(); + + cmd_buf_data.push_with(|| -> Result<_, ClearError> { + Ok(ArcCommand::ClearBuffer { + dst: self.resolve_buffer_id(dst)?, + offset, + size, + }) + }) + } + + pub fn command_encoder_clear_texture( + &self, + command_encoder_id: CommandEncoderId, + dst: TextureId, + subresource_range: &ImageSubresourceRange, + ) -> Result<(), EncoderStateError> { + profiling::scope!("CommandEncoder::clear_texture"); + api_log!("CommandEncoder::clear_texture {dst:?}"); + + let hub = &self.hub; + + let cmd_enc = hub.command_encoders.get(command_encoder_id); + let mut cmd_buf_data = cmd_enc.data.lock(); + + cmd_buf_data.push_with(|| -> Result<_, ClearError> { + Ok(ArcCommand::ClearTexture { + dst: self.resolve_texture_id(dst)?, + subresource_range: *subresource_range, + }) + }) + } +} + +pub(super) fn clear_buffer( + state: &mut EncodingState, + dst_buffer: Arc, + offset: BufferAddress, + size: Option, +) -> Result<(), ClearError> { + dst_buffer.same_device(state.device)?; + + let dst_pending = state + .tracker + .buffers + .set_single(&dst_buffer, wgt::BufferUses::COPY_DST); + + let dst_raw = dst_buffer.try_raw(state.snatch_guard)?; + dst_buffer.check_usage(BufferUsages::COPY_DST)?; + + // Check if offset & size are valid. + if !offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) { + return Err(ClearError::UnalignedBufferOffset(offset)); + } + + let size = size.unwrap_or(dst_buffer.size.saturating_sub(offset)); + if !size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) { + return Err(ClearError::UnalignedFillSize(size)); + } + let end_offset = + offset + .checked_add(size) + .ok_or(ClearError::OffsetPlusSizeExceeds64BitBounds { + start_offset: offset, + requested_size: size, + })?; + if end_offset > dst_buffer.size { + return Err(ClearError::BufferOverrun { + start_offset: offset, + end_offset, + buffer_size: dst_buffer.size, + }); + } + + // This must happen after parameter validation (so that errors are reported + // as required by the spec), but before any side effects. + if offset == end_offset { + log::trace!("Ignoring fill_buffer of size 0"); + return Ok(()); + } + + // Mark dest as initialized. + state + .buffer_memory_init_actions + .extend(dst_buffer.initialization_status.read().create_action( + &dst_buffer, + offset..end_offset, + MemoryInitKind::ImplicitlyInitialized, + )); + + // actual hal barrier & operation + let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, state.snatch_guard)); + unsafe { + state.raw_encoder.transition_buffers(dst_barrier.as_slice()); + state.raw_encoder.clear_buffer(dst_raw, offset..end_offset); + } + + Ok(()) +} + +/// Validate and encode a "Clear Texture" command. +/// +/// This function implements `CommandEncoder::clear_texture` when invoked via +/// the command encoder APIs or trace playback. It has the suffix `_cmd` to +/// distinguish it from [`clear_texture`]. [`clear_texture`], used internally by +/// this function, is a lower-level function that encodes a texture clear +/// operation without validating it. +pub(super) fn clear_texture_cmd( + state: &mut EncodingState, + dst_texture: Arc, + subresource_range: &ImageSubresourceRange, +) -> Result<(), ClearError> { + dst_texture.same_device(state.device)?; + state + .device + .require_features(wgt::Features::CLEAR_TEXTURE)?; + + // Check if subresource aspects are valid. + let clear_aspects = hal::FormatAspects::new(dst_texture.desc.format, subresource_range.aspect); + if clear_aspects.is_empty() { + return Err(ClearError::MissingTextureAspect { + texture_format: dst_texture.desc.format, + subresource_range_aspects: subresource_range.aspect, + }); + }; + + // Check if subresource level range is valid + let subresource_mip_range = subresource_range.mip_range(dst_texture.full_range.mips.end); + if dst_texture.full_range.mips.start > subresource_mip_range.start + || dst_texture.full_range.mips.end < subresource_mip_range.end + { + return Err(ClearError::InvalidTextureLevelRange { + texture_level_range: dst_texture.full_range.mips.clone(), + subresource_base_mip_level: subresource_range.base_mip_level, + subresource_mip_level_count: subresource_range.mip_level_count, + }); + } + // Check if subresource layer range is valid + let subresource_layer_range = subresource_range.layer_range(dst_texture.full_range.layers.end); + if dst_texture.full_range.layers.start > subresource_layer_range.start + || dst_texture.full_range.layers.end < subresource_layer_range.end + { + return Err(ClearError::InvalidTextureLayerRange { + texture_layer_range: dst_texture.full_range.layers.clone(), + subresource_base_array_layer: subresource_range.base_array_layer, + subresource_array_layer_count: subresource_range.array_layer_count, + }); + } + + clear_texture( + &dst_texture, + TextureInitRange { + mip_range: subresource_mip_range, + layer_range: subresource_layer_range, + }, + state.raw_encoder, + &mut state.tracker.textures, + &state.device.alignments, + state.device.zero_buffer.as_ref(), + state.snatch_guard, + state.device.instance_flags, + )?; + + Ok(()) +} + +/// Encode a texture clear operation. +/// +/// This function encodes a texture clear operation without validating it. +/// Texture clears requested via the API call this function via +/// [`clear_texture_cmd`], which does the validation. This function is also +/// called directly from various places within wgpu that need to clear a +/// texture. +pub(crate) fn clear_texture( + dst_texture: &Arc, + range: TextureInitRange, + encoder: &mut dyn hal::DynCommandEncoder, + texture_tracker: &mut T, + alignments: &hal::Alignments, + zero_buffer: &dyn hal::DynBuffer, + snatch_guard: &SnatchGuard<'_>, + instance_flags: wgt::InstanceFlags, +) -> Result<(), ClearError> { + let dst_raw = dst_texture.try_raw(snatch_guard)?; + + // Issue the right barrier. + let clear_usage = match *dst_texture.clear_mode.read() { + TextureClearMode::BufferCopy => wgt::TextureUses::COPY_DST, + TextureClearMode::RenderPass { + is_color: false, .. + } => wgt::TextureUses::DEPTH_STENCIL_WRITE, + TextureClearMode::Surface { .. } | TextureClearMode::RenderPass { is_color: true, .. } => { + wgt::TextureUses::COLOR_TARGET + } + TextureClearMode::None => { + return Err(ClearError::NoValidTextureClearMode( + dst_texture.error_ident(), + )); + } + }; + + let selector = TextureSelector { + mips: range.mip_range.clone(), + layers: range.layer_range.clone(), + }; + + // If we're in a texture-init usecase, we know that the texture is already + // tracked since whatever caused the init requirement, will have caused the + // usage tracker to be aware of the texture. Meaning, that it is safe to + // call call change_replace_tracked if the life_guard is already gone (i.e. + // the user no longer holds on to this texture). + // + // On the other hand, when coming via command_encoder_clear_texture, the + // life_guard is still there since in order to call it a texture object is + // needed. + // + // We could in theory distinguish these two scenarios in the internal + // clear_texture api in order to remove this check and call the cheaper + // change_replace_tracked whenever possible. + let dst_barrier = texture_tracker + .set_single(dst_texture, selector, clear_usage) + .map(|pending| pending.into_hal(dst_raw)) + .collect::>(); + unsafe { + encoder.transition_textures(&dst_barrier); + } + + // Record actual clearing + let clear_mode = dst_texture.clear_mode.read(); + match *clear_mode { + TextureClearMode::BufferCopy => clear_texture_via_buffer_copies( + &dst_texture.desc, + alignments, + zero_buffer, + range, + encoder, + dst_raw, + ), + TextureClearMode::Surface { .. } => { + drop(clear_mode); + clear_texture_via_render_passes(dst_texture, range, true, encoder, instance_flags)? + } + TextureClearMode::RenderPass { is_color, .. } => { + drop(clear_mode); + clear_texture_via_render_passes(dst_texture, range, is_color, encoder, instance_flags)? + } + TextureClearMode::None => { + return Err(ClearError::NoValidTextureClearMode( + dst_texture.error_ident(), + )); + } + } + Ok(()) +} + +fn clear_texture_via_buffer_copies( + texture_desc: &wgt::TextureDescriptor<(), Vec>, + alignments: &hal::Alignments, + zero_buffer: &dyn hal::DynBuffer, // Buffer of size device::ZERO_BUFFER_SIZE + range: TextureInitRange, + encoder: &mut dyn hal::DynCommandEncoder, + dst_raw: &dyn hal::DynTexture, +) { + assert!(!texture_desc.format.is_depth_stencil_format()); + + if texture_desc.format == wgt::TextureFormat::NV12 + || texture_desc.format == wgt::TextureFormat::P010 + { + // TODO: Currently COPY_DST for NV12 and P010 textures is unsupported. + return; + } + + // Gather list of zero_buffer copies and issue a single command then to perform them + let mut zero_buffer_copy_regions = Vec::new(); + let buffer_copy_pitch = alignments.buffer_copy_pitch.get() as u32; + let (block_width, block_height) = texture_desc.format.block_dimensions(); + let block_size = texture_desc.format.block_copy_size(None).unwrap(); + + let bytes_per_row_alignment = get_lowest_common_denom(buffer_copy_pitch, block_size); + + for mip_level in range.mip_range { + let mut mip_size = texture_desc.mip_level_size(mip_level).unwrap(); + // Round to multiple of block size + mip_size.width = align_to(mip_size.width, block_width); + mip_size.height = align_to(mip_size.height, block_height); + + let bytes_per_row = align_to( + mip_size.width / block_width * block_size, + bytes_per_row_alignment, + ); + + let max_rows_per_copy = crate::device::ZERO_BUFFER_SIZE as u32 / bytes_per_row; + // round down to a multiple of rows needed by the texture format + let max_rows_per_copy = max_rows_per_copy / block_height * block_height; + assert!( + max_rows_per_copy > 0, + "Zero buffer size is too small to fill a single row \ + of a texture with format {:?} and desc {:?}", + texture_desc.format, + texture_desc.size + ); + + let z_range = 0..(if texture_desc.dimension == wgt::TextureDimension::D3 { + mip_size.depth_or_array_layers + } else { + 1 + }); + + for array_layer in range.layer_range.clone() { + // TODO: Only doing one layer at a time for volume textures right now. + for z in z_range.clone() { + // May need multiple copies for each subresource! However, we + // assume that we never need to split a row. + let mut num_rows_left = mip_size.height; + while num_rows_left > 0 { + let num_rows = num_rows_left.min(max_rows_per_copy); + + zero_buffer_copy_regions.push(hal::BufferTextureCopy { + buffer_layout: wgt::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(bytes_per_row), + rows_per_image: None, + }, + texture_base: hal::TextureCopyBase { + mip_level, + array_layer, + origin: wgt::Origin3d { + x: 0, // Always full rows + y: mip_size.height - num_rows_left, + z, + }, + aspect: hal::FormatAspects::COLOR, + }, + size: hal::CopyExtent { + width: mip_size.width, // full row + height: num_rows, + depth: 1, // Only single slice of volume texture at a time right now + }, + }); + + num_rows_left -= num_rows; + } + } + } + } + + unsafe { + encoder.copy_buffer_to_texture(zero_buffer, dst_raw, &zero_buffer_copy_regions); + } +} + +fn clear_texture_via_render_passes( + dst_texture: &Texture, + range: TextureInitRange, + is_color: bool, + encoder: &mut dyn hal::DynCommandEncoder, + instance_flags: wgt::InstanceFlags, +) -> Result<(), ClearError> { + assert_eq!(dst_texture.desc.dimension, wgt::TextureDimension::D2); + + let extent_base = wgt::Extent3d { + width: dst_texture.desc.size.width, + height: dst_texture.desc.size.height, + depth_or_array_layers: 1, // Only one layer is cleared at a time. + }; + + let clear_mode = dst_texture.clear_mode.read(); + + for mip_level in range.mip_range { + let extent = extent_base.mip_level_size(mip_level, dst_texture.desc.dimension); + for depth_or_layer in range.layer_range.clone() { + let color_attachments_tmp; + let (color_attachments, depth_stencil_attachment) = if is_color { + color_attachments_tmp = [Some(hal::ColorAttachment { + target: hal::Attachment { + view: Texture::get_clear_view( + &clear_mode, + &dst_texture.desc, + mip_level, + depth_or_layer, + ), + usage: wgt::TextureUses::COLOR_TARGET, + }, + depth_slice: None, + resolve_target: None, + ops: hal::AttachmentOps::STORE | hal::AttachmentOps::LOAD_CLEAR, + clear_value: wgt::Color::TRANSPARENT, + })]; + (&color_attachments_tmp[..], None) + } else { + ( + &[][..], + Some(hal::DepthStencilAttachment { + target: hal::Attachment { + view: Texture::get_clear_view( + &clear_mode, + &dst_texture.desc, + mip_level, + depth_or_layer, + ), + usage: wgt::TextureUses::DEPTH_STENCIL_WRITE, + }, + depth_ops: hal::AttachmentOps::STORE | hal::AttachmentOps::LOAD_CLEAR, + stencil_ops: hal::AttachmentOps::STORE | hal::AttachmentOps::LOAD_CLEAR, + clear_value: (0.0, 0), + }), + ) + }; + unsafe { + encoder + .begin_render_pass(&hal::RenderPassDescriptor { + label: hal_label( + Some("(wgpu internal) clear_texture clear pass"), + instance_flags, + ), + extent, + sample_count: dst_texture.desc.sample_count, + color_attachments, + depth_stencil_attachment, + multiview_mask: None, + timestamp_writes: None, + occlusion_query_set: None, + }) + .map_err(|e| dst_texture.device.handle_hal_error(e))?; + encoder.end_render_pass(); + } + } + } + + Ok(()) +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/compute.rs b/third_party/wgpu-core-29.0.4-patched/src/command/compute.rs new file mode 100644 index 00000000..7ae15dd5 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/compute.rs @@ -0,0 +1,1319 @@ +use thiserror::Error; +use wgt::{ + error::{ErrorType, WebGpuError}, + BufferAddress, DynamicOffset, +}; + +use alloc::{borrow::Cow, boxed::Box, sync::Arc, vec::Vec}; +use core::{convert::Infallible, fmt, str}; + +use crate::{ + api_log, + binding_model::{BindError, ImmediateUploadError, LateMinBufferBindingSizeMismatch}, + command::{ + bind::{Binder, BinderError}, + compute_command::ArcComputeCommand, + encoder::EncodingState, + memory_init::{fixup_discarded_surfaces, SurfacesInDiscardState}, + pass::{self, flush_bindings_helper}, + pass_base, pass_try, + query::{end_pipeline_statistics_query, validate_and_begin_pipeline_statistics_query}, + ArcCommand, ArcPassTimestampWrites, BasePass, BindGroupStateChange, CommandEncoder, + CommandEncoderError, DebugGroupError, EncoderStateError, InnerCommandEncoder, MapPassErr, + PassErrorScope, PassStateError, PassTimestampWrites, QueryUseError, StateChange, + TimestampWritesError, + }, + device::{Device, DeviceError, MissingDownlevelFlags, MissingFeatures}, + global::Global, + hal_label, id, + init_tracker::MemoryInitKind, + pipeline::ComputePipeline, + resource::{ + self, Buffer, DestroyedResourceError, InvalidResourceError, Labeled, + MissingBufferUsageError, ParentDevice, RawResourceAccess, Trackable, + }, + track::{ResourceUsageCompatibilityError, Tracker}, + Label, +}; + +pub type ComputeBasePass = BasePass; + +/// A pass's [encoder state](https://www.w3.org/TR/webgpu/#encoder-state) and +/// its validity are two distinct conditions, i.e., the full matrix of +/// (open, ended) x (valid, invalid) is possible. +/// +/// The presence or absence of the `parent` `Option` indicates the pass's state. +/// The presence or absence of an error in `base.error` indicates the pass's +/// validity. +pub struct ComputePass { + /// All pass data & records is stored here. + base: ComputeBasePass, + + /// Parent command encoder that this pass records commands into. + /// + /// If this is `Some`, then the pass is in WebGPU's "open" state. If it is + /// `None`, then the pass is in the "ended" state. + /// See + parent: Option>, + + timestamp_writes: Option, + + // Resource binding dedupe state. + current_bind_groups: BindGroupStateChange, + current_pipeline: StateChange, +} + +impl ComputePass { + /// If the parent command encoder is invalid, the returned pass will be invalid. + fn new(parent: Arc, desc: ArcComputePassDescriptor) -> Self { + let ArcComputePassDescriptor { + label, + timestamp_writes, + } = desc; + + Self { + base: BasePass::new(&label), + parent: Some(parent), + timestamp_writes, + + current_bind_groups: BindGroupStateChange::new(), + current_pipeline: StateChange::new(), + } + } + + fn new_invalid(parent: Arc, label: &Label, err: ComputePassError) -> Self { + Self { + base: BasePass::new_invalid(label, err), + parent: Some(parent), + timestamp_writes: None, + current_bind_groups: BindGroupStateChange::new(), + current_pipeline: StateChange::new(), + } + } + + #[inline] + pub fn label(&self) -> Option<&str> { + self.base.label.as_deref() + } +} + +impl fmt::Debug for ComputePass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.parent { + Some(ref cmd_enc) => write!(f, "ComputePass {{ parent: {} }}", cmd_enc.error_ident()), + None => write!(f, "ComputePass {{ parent: None }}"), + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct ComputePassDescriptor<'a, PTW = PassTimestampWrites> { + pub label: Label<'a>, + /// Defines where and when timestamp values will be written for this pass. + pub timestamp_writes: Option, +} + +/// cbindgen:ignore +type ArcComputePassDescriptor<'a> = ComputePassDescriptor<'a, ArcPassTimestampWrites>; + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum DispatchError { + #[error("Compute pipeline must be set")] + MissingPipeline(pass::MissingPipeline), + #[error(transparent)] + IncompatibleBindGroup(#[from] Box), + #[error( + "Each current dispatch group size dimension ({current:?}) must be less or equal to {limit}" + )] + InvalidGroupSize { current: [u32; 3], limit: u32 }, + #[error(transparent)] + BindingSizeTooSmall(#[from] LateMinBufferBindingSizeMismatch), +} + +impl WebGpuError for DispatchError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +/// Error encountered when performing a compute pass. +#[derive(Clone, Debug, Error)] +pub enum ComputePassErrorInner { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + EncoderState(#[from] EncoderStateError), + #[error("Parent encoder is invalid")] + InvalidParentEncoder, + #[error(transparent)] + DebugGroupError(#[from] DebugGroupError), + #[error(transparent)] + BindGroupIndexOutOfRange(#[from] pass::BindGroupIndexOutOfRange), + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + #[error("Indirect buffer offset {0:?} is not a multiple of 4")] + UnalignedIndirectBufferOffset(BufferAddress), + #[error("Indirect buffer uses bytes {offset}..{end_offset} which overruns indirect buffer of size {buffer_size}")] + IndirectBufferOverrun { + offset: u64, + end_offset: u64, + buffer_size: u64, + }, + #[error(transparent)] + ResourceUsageCompatibility(#[from] ResourceUsageCompatibilityError), + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + #[error(transparent)] + Dispatch(#[from] DispatchError), + #[error(transparent)] + Bind(#[from] BindError), + #[error(transparent)] + ImmediateData(#[from] ImmediateUploadError), + #[error("Immediate data offset must be aligned to 4 bytes")] + ImmediateOffsetAlignment, + #[error("Immediate data size must be aligned to 4 bytes")] + ImmediateDataizeAlignment, + #[error("Ran out of immediate data space. Don't set 4gb of immediates per ComputePass.")] + ImmediateOutOfMemory, + #[error(transparent)] + QueryUse(#[from] QueryUseError), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), + #[error("The compute pass has already been ended and no further commands can be recorded")] + PassEnded, + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), + #[error(transparent)] + TimestampWrites(#[from] TimestampWritesError), + // This one is unreachable, but required for generic pass support + #[error(transparent)] + InvalidValuesOffset(#[from] pass::InvalidValuesOffset), +} + +/// Error encountered when performing a compute pass, stored for later reporting +/// when encoding ends. +#[derive(Clone, Debug, Error)] +#[error("{scope}")] +pub struct ComputePassError { + pub scope: PassErrorScope, + #[source] + pub(super) inner: ComputePassErrorInner, +} + +impl From for ComputePassErrorInner { + fn from(value: pass::MissingPipeline) -> Self { + Self::Dispatch(DispatchError::MissingPipeline(value)) + } +} + +impl MapPassErr for E +where + E: Into, +{ + fn map_pass_err(self, scope: PassErrorScope) -> ComputePassError { + ComputePassError { + scope, + inner: self.into(), + } + } +} + +impl WebGpuError for ComputePassError { + fn webgpu_error_type(&self) -> ErrorType { + let Self { scope: _, inner } = self; + match inner { + ComputePassErrorInner::Device(e) => e.webgpu_error_type(), + ComputePassErrorInner::EncoderState(e) => e.webgpu_error_type(), + ComputePassErrorInner::DebugGroupError(e) => e.webgpu_error_type(), + ComputePassErrorInner::DestroyedResource(e) => e.webgpu_error_type(), + ComputePassErrorInner::ResourceUsageCompatibility(e) => e.webgpu_error_type(), + ComputePassErrorInner::MissingBufferUsage(e) => e.webgpu_error_type(), + ComputePassErrorInner::Dispatch(e) => e.webgpu_error_type(), + ComputePassErrorInner::Bind(e) => e.webgpu_error_type(), + ComputePassErrorInner::ImmediateData(e) => e.webgpu_error_type(), + ComputePassErrorInner::QueryUse(e) => e.webgpu_error_type(), + ComputePassErrorInner::MissingFeatures(e) => e.webgpu_error_type(), + ComputePassErrorInner::MissingDownlevelFlags(e) => e.webgpu_error_type(), + ComputePassErrorInner::InvalidResource(e) => e.webgpu_error_type(), + ComputePassErrorInner::TimestampWrites(e) => e.webgpu_error_type(), + ComputePassErrorInner::InvalidValuesOffset(e) => e.webgpu_error_type(), + + ComputePassErrorInner::InvalidParentEncoder + | ComputePassErrorInner::BindGroupIndexOutOfRange { .. } + | ComputePassErrorInner::UnalignedIndirectBufferOffset(_) + | ComputePassErrorInner::IndirectBufferOverrun { .. } + | ComputePassErrorInner::ImmediateOffsetAlignment + | ComputePassErrorInner::ImmediateDataizeAlignment + | ComputePassErrorInner::ImmediateOutOfMemory + | ComputePassErrorInner::PassEnded => ErrorType::Validation, + } + } +} + +struct State<'scope, 'snatch_guard, 'cmd_enc> { + pipeline: Option>, + + pass: pass::PassState<'scope, 'snatch_guard, 'cmd_enc>, + + active_query: Option<(Arc, u32)>, + + immediates: Vec, + + intermediate_trackers: Tracker, +} + +impl<'scope, 'snatch_guard, 'cmd_enc> State<'scope, 'snatch_guard, 'cmd_enc> { + fn is_ready(&self) -> Result<(), DispatchError> { + if let Some(pipeline) = self.pipeline.as_ref() { + self.pass.binder.check_compatibility(pipeline.as_ref())?; + self.pass.binder.check_late_buffer_bindings()?; + Ok(()) + } else { + Err(DispatchError::MissingPipeline(pass::MissingPipeline)) + } + } + + /// Flush binding state in preparation for a dispatch. + /// + /// # Differences between render and compute passes + /// + /// There are differences between the `flush_bindings` implementations for + /// render and compute passes, because render passes have a single usage + /// scope for the entire pass, and compute passes have a separate usage + /// scope for each dispatch. + /// + /// For compute passes, bind groups are merged into a fresh usage scope + /// here, not into the pass usage scope within calls to `set_bind_group`. As + /// specified by WebGPU, for compute passes, we merge only the bind groups + /// that are actually used by the pipeline, unlike render passes, which + /// merge every bind group that is ever set, even if it is not ultimately + /// used by the pipeline. + /// + /// For compute passes, we call `drain_barriers` here, because barriers may + /// be needed before each dispatch if a previous dispatch had a conflicting + /// usage. For render passes, barriers are emitted once at the start of the + /// render pass. + /// + /// # Indirect buffer handling + /// + /// The `indirect_buffer` argument should be passed for any indirect + /// dispatch (with or without validation). It will be checked for + /// conflicting usages according to WebGPU rules. For the purpose of + /// these rules, the fact that we have actually processed the buffer in + /// the validation pass is an implementation detail. + /// + /// The `track_indirect_buffer` argument should be set when doing indirect + /// dispatch *without* validation. In this case, the indirect buffer will + /// be added to the tracker in order to generate any necessary transitions + /// for that usage. + /// + /// When doing indirect dispatch *with* validation, the indirect buffer is + /// processed by the validation pass and is not used by the actual dispatch. + /// The indirect validation code handles transitions for the validation + /// pass. + fn flush_bindings( + &mut self, + indirect_buffer: Option<&Arc>, + track_indirect_buffer: bool, + ) -> Result<(), ComputePassErrorInner> { + for bind_group in self.pass.binder.list_active() { + unsafe { self.pass.scope.merge_bind_group(&bind_group.used)? }; + } + + // Add the indirect buffer. Because usage scopes are per-dispatch, this + // is the only place where INDIRECT usage could be added, and it is safe + // for us to remove it below. + if let Some(buffer) = indirect_buffer { + self.pass + .scope + .buffers + .merge_single(buffer, wgt::BufferUses::INDIRECT)?; + } + + // For compute, usage scopes are associated with each dispatch and not + // with the pass as a whole. However, because the cost of creating and + // dropping `UsageScope`s is significant (even with the pool), we + // add and then remove usage from a single usage scope. + + for bind_group in self.pass.binder.list_active() { + self.intermediate_trackers + .set_and_remove_from_usage_scope_sparse(&mut self.pass.scope, &bind_group.used); + } + + if track_indirect_buffer { + self.intermediate_trackers + .buffers + .set_and_remove_from_usage_scope_sparse( + &mut self.pass.scope.buffers, + indirect_buffer.map(|buf| buf.tracker_index()), + ); + } else if let Some(buffer) = indirect_buffer { + self.pass + .scope + .buffers + .remove_usage(buffer, wgt::BufferUses::INDIRECT); + } + + flush_bindings_helper(&mut self.pass)?; + + CommandEncoder::drain_barriers( + self.pass.base.raw_encoder, + &mut self.intermediate_trackers, + self.pass.base.snatch_guard, + ); + Ok(()) + } +} + +// Running the compute pass. + +impl Global { + /// Creates a compute pass. + /// + /// If creation fails, an invalid pass is returned. Attempting to record + /// commands into an invalid pass is permitted, but a validation error will + /// ultimately be generated when the parent encoder is finished, and it is + /// not possible to run any commands from the invalid pass. + /// + /// If successful, puts the encoder into the [`Locked`] state. + /// + /// [`Locked`]: crate::command::CommandEncoderStatus::Locked + pub fn command_encoder_begin_compute_pass( + &self, + encoder_id: id::CommandEncoderId, + desc: &ComputePassDescriptor<'_>, + ) -> (ComputePass, Option) { + use EncoderStateError as SErr; + + let scope = PassErrorScope::Pass; + let hub = &self.hub; + + let label = desc.label.as_deref().map(Cow::Borrowed); + + let cmd_enc = hub.command_encoders.get(encoder_id); + let mut cmd_buf_data = cmd_enc.data.lock(); + + match cmd_buf_data.lock_encoder() { + Ok(()) => { + drop(cmd_buf_data); + if let Err(err) = cmd_enc.device.check_is_valid() { + return ( + ComputePass::new_invalid(cmd_enc, &label, err.map_pass_err(scope)), + None, + ); + } + + match desc + .timestamp_writes + .as_ref() + .map(|tw| { + Self::validate_pass_timestamp_writes::( + &cmd_enc.device, + &hub.query_sets.read(), + tw, + ) + }) + .transpose() + { + Ok(timestamp_writes) => { + let arc_desc = ArcComputePassDescriptor { + label, + timestamp_writes, + }; + (ComputePass::new(cmd_enc, arc_desc), None) + } + Err(err) => ( + ComputePass::new_invalid(cmd_enc, &label, err.map_pass_err(scope)), + None, + ), + } + } + Err(err @ SErr::Locked) => { + // Attempting to open a new pass while the encoder is locked + // invalidates the encoder, but does not generate a validation + // error. + cmd_buf_data.invalidate(err.clone()); + drop(cmd_buf_data); + ( + ComputePass::new_invalid(cmd_enc, &label, err.map_pass_err(scope)), + None, + ) + } + Err(err @ (SErr::Ended | SErr::Submitted)) => { + // Attempting to open a new pass after the encode has ended + // generates an immediate validation error. + drop(cmd_buf_data); + ( + ComputePass::new_invalid(cmd_enc, &label, err.clone().map_pass_err(scope)), + Some(err.into()), + ) + } + Err(err @ SErr::Invalid) => { + // Passes can be opened even on an invalid encoder. Such passes + // are even valid, but since there's no visible side-effect of + // the pass being valid and there's no point in storing recorded + // commands that will ultimately be discarded, we open an + // invalid pass to save that work. + drop(cmd_buf_data); + ( + ComputePass::new_invalid(cmd_enc, &label, err.map_pass_err(scope)), + None, + ) + } + Err(SErr::Unlocked) => { + unreachable!("lock_encoder cannot fail due to the encoder being unlocked") + } + } + } + + pub fn compute_pass_end(&self, pass: &mut ComputePass) -> Result<(), EncoderStateError> { + profiling::scope!( + "CommandEncoder::run_compute_pass {}", + pass.base.label.as_deref().unwrap_or("") + ); + + let cmd_enc = pass.parent.take().ok_or(EncoderStateError::Ended)?; + let mut cmd_buf_data = cmd_enc.data.lock(); + + cmd_buf_data.unlock_encoder()?; + + let base = pass.base.take(); + + if let Err(ComputePassError { + inner: + ComputePassErrorInner::EncoderState( + err @ (EncoderStateError::Locked | EncoderStateError::Ended), + ), + scope: _, + }) = base + { + // Most encoding errors are detected and raised within `finish()`. + // + // However, we raise a validation error here if the pass was opened + // within another pass, or on a finished encoder. The latter is + // particularly important, because in that case reporting errors via + // `CommandEncoder::finish` is not possible. + return Err(err.clone()); + } + + cmd_buf_data.push_with(|| -> Result<_, ComputePassError> { + Ok(ArcCommand::RunComputePass { + pass: base?, + timestamp_writes: pass.timestamp_writes.take(), + }) + }) + } +} + +pub(super) fn encode_compute_pass( + parent_state: &mut EncodingState, + mut base: BasePass, + mut timestamp_writes: Option, +) -> Result<(), ComputePassError> { + let pass_scope = PassErrorScope::Pass; + + let device = parent_state.device; + + // We automatically keep extending command buffers over time, and because + // we want to insert a command buffer _before_ what we're about to record, + // we need to make sure to close the previous one. + parent_state + .raw_encoder + .close_if_open() + .map_pass_err(pass_scope)?; + let raw_encoder = parent_state + .raw_encoder + .open_pass(base.label.as_deref()) + .map_pass_err(pass_scope)?; + + let mut debug_scope_depth = 0; + + let mut state = State { + pipeline: None, + + pass: pass::PassState { + base: EncodingState { + device, + raw_encoder, + tracker: parent_state.tracker, + buffer_memory_init_actions: parent_state.buffer_memory_init_actions, + texture_memory_actions: parent_state.texture_memory_actions, + as_actions: parent_state.as_actions, + temp_resources: parent_state.temp_resources, + indirect_draw_validation_resources: parent_state.indirect_draw_validation_resources, + snatch_guard: parent_state.snatch_guard, + debug_scope_depth: &mut debug_scope_depth, + }, + binder: Binder::new(), + temp_offsets: Vec::new(), + dynamic_offset_count: 0, + pending_discard_init_fixups: SurfacesInDiscardState::new(), + scope: device.new_usage_scope(), + string_offset: 0, + }, + active_query: None, + + immediates: Vec::new(), + + intermediate_trackers: Tracker::new( + device.ordered_buffer_usages, + device.ordered_texture_usages, + ), + }; + + let indices = &device.tracker_indices; + state + .pass + .base + .tracker + .buffers + .set_size(indices.buffers.size()); + state + .pass + .base + .tracker + .textures + .set_size(indices.textures.size()); + + let timestamp_writes: Option> = + if let Some(tw) = timestamp_writes.take() { + tw.query_set.same_device(device).map_pass_err(pass_scope)?; + + let query_set = state + .pass + .base + .tracker + .query_sets + .insert_single(tw.query_set); + + // Unlike in render passes we can't delay resetting the query sets since + // there is no auxiliary pass. + let range = if let (Some(index_a), Some(index_b)) = + (tw.beginning_of_pass_write_index, tw.end_of_pass_write_index) + { + Some(index_a.min(index_b)..index_a.max(index_b) + 1) + } else { + tw.beginning_of_pass_write_index + .or(tw.end_of_pass_write_index) + .map(|i| i..i + 1) + }; + // Range should always be Some, both values being None should lead to a validation error. + // But no point in erroring over that nuance here! + if let Some(range) = range { + unsafe { + state + .pass + .base + .raw_encoder + .reset_queries(query_set.raw(), range); + } + } + + Some(hal::PassTimestampWrites { + query_set: query_set.raw(), + beginning_of_pass_write_index: tw.beginning_of_pass_write_index, + end_of_pass_write_index: tw.end_of_pass_write_index, + }) + } else { + None + }; + + let hal_desc = hal::ComputePassDescriptor { + label: hal_label(base.label.as_deref(), device.instance_flags), + timestamp_writes, + }; + + unsafe { + state.pass.base.raw_encoder.begin_compute_pass(&hal_desc); + } + + for command in base.commands.drain(..) { + match command { + ArcComputeCommand::SetBindGroup { + index, + num_dynamic_offsets, + bind_group, + } => { + let scope = PassErrorScope::SetBindGroup; + pass::set_bind_group::( + &mut state.pass, + device, + &base.dynamic_offsets, + index, + num_dynamic_offsets, + bind_group, + false, + ) + .map_pass_err(scope)?; + } + ArcComputeCommand::SetPipeline(pipeline) => { + let scope = PassErrorScope::SetPipelineCompute; + set_pipeline(&mut state, device, pipeline).map_pass_err(scope)?; + } + ArcComputeCommand::SetImmediate { + offset, + size_bytes, + values_offset, + } => { + let scope = PassErrorScope::SetImmediate; + pass::set_immediates::( + &mut state.pass, + &base.immediates_data, + offset, + size_bytes, + Some(values_offset), + |data_slice| { + let offset_in_elements = (offset / wgt::IMMEDIATE_DATA_ALIGNMENT) as usize; + let size_in_elements = + (size_bytes / wgt::IMMEDIATE_DATA_ALIGNMENT) as usize; + state.immediates[offset_in_elements..][..size_in_elements] + .copy_from_slice(data_slice); + }, + ) + .map_pass_err(scope)?; + } + ArcComputeCommand::Dispatch(groups) => { + let scope = PassErrorScope::Dispatch { indirect: false }; + dispatch(&mut state, groups).map_pass_err(scope)?; + } + ArcComputeCommand::DispatchIndirect { buffer, offset } => { + let scope = PassErrorScope::Dispatch { indirect: true }; + dispatch_indirect(&mut state, device, buffer, offset).map_pass_err(scope)?; + } + ArcComputeCommand::PushDebugGroup { color: _, len } => { + pass::push_debug_group(&mut state.pass, &base.string_data, len); + } + ArcComputeCommand::PopDebugGroup => { + let scope = PassErrorScope::PopDebugGroup; + pass::pop_debug_group::(&mut state.pass) + .map_pass_err(scope)?; + } + ArcComputeCommand::InsertDebugMarker { color: _, len } => { + pass::insert_debug_marker(&mut state.pass, &base.string_data, len); + } + ArcComputeCommand::WriteTimestamp { + query_set, + query_index, + } => { + let scope = PassErrorScope::WriteTimestamp; + pass::write_timestamp::( + &mut state.pass, + device, + None, // compute passes do not attempt to coalesce query resets + query_set, + query_index, + ) + .map_pass_err(scope)?; + } + ArcComputeCommand::BeginPipelineStatisticsQuery { + query_set, + query_index, + } => { + let scope = PassErrorScope::BeginPipelineStatisticsQuery; + validate_and_begin_pipeline_statistics_query( + query_set, + state.pass.base.raw_encoder, + &mut state.pass.base.tracker.query_sets, + device, + query_index, + None, + &mut state.active_query, + ) + .map_pass_err(scope)?; + } + ArcComputeCommand::EndPipelineStatisticsQuery => { + let scope = PassErrorScope::EndPipelineStatisticsQuery; + end_pipeline_statistics_query(state.pass.base.raw_encoder, &mut state.active_query) + .map_pass_err(scope)?; + } + } + } + + if *state.pass.base.debug_scope_depth > 0 { + Err( + ComputePassErrorInner::DebugGroupError(DebugGroupError::MissingPop) + .map_pass_err(pass_scope), + )?; + } + + unsafe { + state.pass.base.raw_encoder.end_compute_pass(); + } + + let State { + pass: pass::PassState { + pending_discard_init_fixups, + .. + }, + intermediate_trackers, + .. + } = state; + + // Stop the current command encoder. + parent_state.raw_encoder.close().map_pass_err(pass_scope)?; + + // Create a new command encoder, which we will insert _before_ the body of the compute pass. + // + // Use that buffer to insert barriers and clear discarded images. + let transit = parent_state + .raw_encoder + .open_pass(hal_label( + Some("(wgpu internal) Pre Pass"), + device.instance_flags, + )) + .map_pass_err(pass_scope)?; + fixup_discarded_surfaces( + pending_discard_init_fixups.into_iter(), + transit, + &mut parent_state.tracker.textures, + device, + parent_state.snatch_guard, + ); + CommandEncoder::insert_barriers_from_tracker( + transit, + parent_state.tracker, + &intermediate_trackers, + parent_state.snatch_guard, + ); + // Close the command encoder, and swap it with the previous. + parent_state + .raw_encoder + .close_and_swap() + .map_pass_err(pass_scope)?; + + Ok(()) +} + +fn set_pipeline( + state: &mut State, + device: &Arc, + pipeline: Arc, +) -> Result<(), ComputePassErrorInner> { + pipeline.same_device(device)?; + + state.pipeline = Some(pipeline.clone()); + + let pipeline = state + .pass + .base + .tracker + .compute_pipelines + .insert_single(pipeline) + .clone(); + + unsafe { + state + .pass + .base + .raw_encoder + .set_compute_pipeline(pipeline.raw()); + } + + // Rebind resources + pass::change_pipeline_layout::( + &mut state.pass, + &pipeline.layout, + &pipeline.late_sized_buffer_groups, + || { + // This only needs to be here for compute pipelines because they use immediates for + // validating indirect draws. + state.immediates.clear(); + // Note that can only be one range for each stage. See the `MoreThanOneImmediateRangePerStage` error. + if pipeline.layout.immediate_size != 0 { + // Note that non-0 range start doesn't work anyway https://github.com/gfx-rs/wgpu/issues/4502 + let len = pipeline.layout.immediate_size as usize + / wgt::IMMEDIATE_DATA_ALIGNMENT as usize; + state.immediates.extend(core::iter::repeat_n(0, len)); + } + }, + ) +} + +fn dispatch(state: &mut State, groups: [u32; 3]) -> Result<(), ComputePassErrorInner> { + api_log!("ComputePass::dispatch {groups:?}"); + + state.is_ready()?; + + state.flush_bindings(None, false)?; + + let groups_size_limit = state + .pass + .base + .device + .limits + .max_compute_workgroups_per_dimension; + + if groups.iter().copied().any(|g| g > groups_size_limit) { + return Err(ComputePassErrorInner::Dispatch( + DispatchError::InvalidGroupSize { + current: groups, + limit: groups_size_limit, + }, + )); + } + + unsafe { + state.pass.base.raw_encoder.dispatch(groups); + } + Ok(()) +} + +fn dispatch_indirect( + state: &mut State, + device: &Arc, + buffer: Arc, + offset: u64, +) -> Result<(), ComputePassErrorInner> { + api_log!("ComputePass::dispatch_indirect"); + + buffer.same_device(device)?; + + state.is_ready()?; + + state + .pass + .base + .device + .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?; + + buffer.check_usage(wgt::BufferUsages::INDIRECT)?; + + if !offset.is_multiple_of(4) { + return Err(ComputePassErrorInner::UnalignedIndirectBufferOffset(offset)); + } + + let end_offset = offset + size_of::() as u64; + if end_offset > buffer.size { + return Err(ComputePassErrorInner::IndirectBufferOverrun { + offset, + end_offset, + buffer_size: buffer.size, + }); + } + + buffer.check_destroyed(state.pass.base.snatch_guard)?; + + let stride = 3 * 4; // 3 integers, x/y/z group size + state.pass.base.buffer_memory_init_actions.extend( + buffer.initialization_status.read().create_action( + &buffer, + offset..(offset + stride), + MemoryInitKind::NeedsInitializedMemory, + ), + ); + + if let Some(ref indirect_validation) = state.pass.base.device.indirect_validation { + let params = indirect_validation.dispatch.params( + &state.pass.base.device.limits, + offset, + buffer.size, + ); + + unsafe { + state + .pass + .base + .raw_encoder + .set_compute_pipeline(params.pipeline); + } + + unsafe { + state.pass.base.raw_encoder.set_immediates( + params.pipeline_layout, + 0, + &[params.offset_remainder as u32 / 4], + ); + } + + unsafe { + state.pass.base.raw_encoder.set_bind_group( + params.pipeline_layout, + 0, + params.dst_bind_group, + &[], + ); + } + unsafe { + state.pass.base.raw_encoder.set_bind_group( + params.pipeline_layout, + 1, + buffer + .indirect_validation_bind_groups + .get(state.pass.base.snatch_guard) + .unwrap() + .dispatch + .as_ref(), + &[params.aligned_offset as u32], + ); + } + + let src_transition = state + .intermediate_trackers + .buffers + .set_single(&buffer, wgt::BufferUses::STORAGE_READ_ONLY); + let src_barrier = src_transition + .map(|transition| transition.into_hal(&buffer, state.pass.base.snatch_guard)); + unsafe { + state + .pass + .base + .raw_encoder + .transition_buffers(src_barrier.as_slice()); + } + + unsafe { + state + .pass + .base + .raw_encoder + .transition_buffers(&[hal::BufferBarrier { + buffer: params.dst_buffer, + usage: hal::StateTransition { + from: wgt::BufferUses::INDIRECT, + to: wgt::BufferUses::STORAGE_READ_WRITE, + }, + }]); + } + + unsafe { + state.pass.base.raw_encoder.dispatch([1, 1, 1]); + } + + // reset state + { + let pipeline = state.pipeline.as_ref().unwrap(); + + unsafe { + state + .pass + .base + .raw_encoder + .set_compute_pipeline(pipeline.raw()); + } + + if !state.immediates.is_empty() { + unsafe { + state.pass.base.raw_encoder.set_immediates( + pipeline.layout.raw(), + 0, + &state.immediates, + ); + } + } + + for (i, group, dynamic_offsets) in state.pass.binder.list_valid() { + let raw_bg = group.try_raw(state.pass.base.snatch_guard)?; + unsafe { + state.pass.base.raw_encoder.set_bind_group( + pipeline.layout.raw(), + i as u32, + raw_bg, + dynamic_offsets, + ); + } + } + } + + unsafe { + state + .pass + .base + .raw_encoder + .transition_buffers(&[hal::BufferBarrier { + buffer: params.dst_buffer, + usage: hal::StateTransition { + from: wgt::BufferUses::STORAGE_READ_WRITE, + to: wgt::BufferUses::INDIRECT, + }, + }]); + } + + state.flush_bindings(Some(&buffer), false)?; + unsafe { + state + .pass + .base + .raw_encoder + .dispatch_indirect(params.dst_buffer, 0); + } + } else { + state.flush_bindings(Some(&buffer), true)?; + + let buf_raw = buffer.try_raw(state.pass.base.snatch_guard)?; + unsafe { + state + .pass + .base + .raw_encoder + .dispatch_indirect(buf_raw, offset); + } + } + + Ok(()) +} + +// Recording a compute pass. +// +// The only error that should be returned from these methods is +// `EncoderStateError::Ended`, when the pass has already ended and an immediate +// validation error is raised. +// +// All other errors should be stored in the pass for later reporting when +// `CommandEncoder.finish()` is called. +// +// The `pass_try!` macro should be used to handle errors appropriately. Note +// that the `pass_try!` and `pass_base!` macros may return early from the +// function that invokes them, like the `?` operator. +impl Global { + pub fn compute_pass_set_bind_group( + &self, + pass: &mut ComputePass, + index: u32, + bind_group_id: Option, + offsets: &[DynamicOffset], + ) -> Result<(), PassStateError> { + let scope = PassErrorScope::SetBindGroup; + + // This statement will return an error if the pass is ended. It's + // important the error check comes before the early-out for + // `set_and_check_redundant`. + let base = pass_base!(pass, scope); + + if pass.current_bind_groups.set_and_check_redundant( + bind_group_id, + index, + &mut base.dynamic_offsets, + offsets, + ) { + return Ok(()); + } + + let mut bind_group = None; + if let Some(bind_group_id) = bind_group_id { + let hub = &self.hub; + bind_group = Some(pass_try!( + base, + scope, + hub.bind_groups.get(bind_group_id).get(), + )); + } + + base.commands.push(ArcComputeCommand::SetBindGroup { + index, + num_dynamic_offsets: offsets.len(), + bind_group, + }); + + Ok(()) + } + + pub fn compute_pass_set_pipeline( + &self, + pass: &mut ComputePass, + pipeline_id: id::ComputePipelineId, + ) -> Result<(), PassStateError> { + let redundant = pass.current_pipeline.set_and_check_redundant(pipeline_id); + + let scope = PassErrorScope::SetPipelineCompute; + + // This statement will return an error if the pass is ended. + // Its important the error check comes before the early-out for `redundant`. + let base = pass_base!(pass, scope); + + if redundant { + return Ok(()); + } + + let hub = &self.hub; + let pipeline = pass_try!(base, scope, hub.compute_pipelines.get(pipeline_id).get()); + + base.commands.push(ArcComputeCommand::SetPipeline(pipeline)); + + Ok(()) + } + + pub fn compute_pass_set_immediates( + &self, + pass: &mut ComputePass, + offset: u32, + data: &[u8], + ) -> Result<(), PassStateError> { + let scope = PassErrorScope::SetImmediate; + let base = pass_base!(pass, scope); + + if offset & (wgt::IMMEDIATE_DATA_ALIGNMENT - 1) != 0 { + pass_try!( + base, + scope, + Err(ComputePassErrorInner::ImmediateOffsetAlignment), + ); + } + + if data.len() as u32 & (wgt::IMMEDIATE_DATA_ALIGNMENT - 1) != 0 { + pass_try!( + base, + scope, + Err(ComputePassErrorInner::ImmediateDataizeAlignment), + ) + } + let value_offset = pass_try!( + base, + scope, + base.immediates_data + .len() + .try_into() + .map_err(|_| ComputePassErrorInner::ImmediateOutOfMemory) + ); + + base.immediates_data.extend( + data.chunks_exact(wgt::IMMEDIATE_DATA_ALIGNMENT as usize) + .map(|arr| u32::from_ne_bytes([arr[0], arr[1], arr[2], arr[3]])), + ); + + base.commands.push(ArcComputeCommand::SetImmediate { + offset, + size_bytes: data.len() as u32, + values_offset: value_offset, + }); + + Ok(()) + } + + pub fn compute_pass_dispatch_workgroups( + &self, + pass: &mut ComputePass, + groups_x: u32, + groups_y: u32, + groups_z: u32, + ) -> Result<(), PassStateError> { + let scope = PassErrorScope::Dispatch { indirect: false }; + + pass_base!(pass, scope) + .commands + .push(ArcComputeCommand::Dispatch([groups_x, groups_y, groups_z])); + + Ok(()) + } + + pub fn compute_pass_dispatch_workgroups_indirect( + &self, + pass: &mut ComputePass, + buffer_id: id::BufferId, + offset: BufferAddress, + ) -> Result<(), PassStateError> { + let hub = &self.hub; + let scope = PassErrorScope::Dispatch { indirect: true }; + let base = pass_base!(pass, scope); + + let buffer = pass_try!(base, scope, hub.buffers.get(buffer_id).get()); + + base.commands + .push(ArcComputeCommand::DispatchIndirect { buffer, offset }); + + Ok(()) + } + + pub fn compute_pass_push_debug_group( + &self, + pass: &mut ComputePass, + label: &str, + color: u32, + ) -> Result<(), PassStateError> { + let base = pass_base!(pass, PassErrorScope::PushDebugGroup); + + let bytes = label.as_bytes(); + base.string_data.extend_from_slice(bytes); + + base.commands.push(ArcComputeCommand::PushDebugGroup { + color, + len: bytes.len(), + }); + + Ok(()) + } + + pub fn compute_pass_pop_debug_group( + &self, + pass: &mut ComputePass, + ) -> Result<(), PassStateError> { + let base = pass_base!(pass, PassErrorScope::PopDebugGroup); + + base.commands.push(ArcComputeCommand::PopDebugGroup); + + Ok(()) + } + + pub fn compute_pass_insert_debug_marker( + &self, + pass: &mut ComputePass, + label: &str, + color: u32, + ) -> Result<(), PassStateError> { + let base = pass_base!(pass, PassErrorScope::InsertDebugMarker); + + let bytes = label.as_bytes(); + base.string_data.extend_from_slice(bytes); + + base.commands.push(ArcComputeCommand::InsertDebugMarker { + color, + len: bytes.len(), + }); + + Ok(()) + } + + pub fn compute_pass_write_timestamp( + &self, + pass: &mut ComputePass, + query_set_id: id::QuerySetId, + query_index: u32, + ) -> Result<(), PassStateError> { + let scope = PassErrorScope::WriteTimestamp; + let base = pass_base!(pass, scope); + + let hub = &self.hub; + let query_set = pass_try!(base, scope, hub.query_sets.get(query_set_id).get()); + + base.commands.push(ArcComputeCommand::WriteTimestamp { + query_set, + query_index, + }); + + Ok(()) + } + + pub fn compute_pass_begin_pipeline_statistics_query( + &self, + pass: &mut ComputePass, + query_set_id: id::QuerySetId, + query_index: u32, + ) -> Result<(), PassStateError> { + let scope = PassErrorScope::BeginPipelineStatisticsQuery; + let base = pass_base!(pass, scope); + + let hub = &self.hub; + let query_set = pass_try!(base, scope, hub.query_sets.get(query_set_id).get()); + + base.commands + .push(ArcComputeCommand::BeginPipelineStatisticsQuery { + query_set, + query_index, + }); + + Ok(()) + } + + pub fn compute_pass_end_pipeline_statistics_query( + &self, + pass: &mut ComputePass, + ) -> Result<(), PassStateError> { + pass_base!(pass, PassErrorScope::EndPipelineStatisticsQuery) + .commands + .push(ArcComputeCommand::EndPipelineStatisticsQuery); + + Ok(()) + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/compute_command.rs b/third_party/wgpu-core-29.0.4-patched/src/command/compute_command.rs new file mode 100644 index 00000000..ce653a7d --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/compute_command.rs @@ -0,0 +1,70 @@ +#[cfg(feature = "serde")] +use crate::command::serde_object_reference_struct; +use crate::command::{ArcReferences, ReferenceType}; + +#[cfg(feature = "serde")] +use macro_rules_attribute::apply; + +/// cbindgen:ignore +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", apply(serde_object_reference_struct))] +pub enum ComputeCommand { + SetBindGroup { + index: u32, + num_dynamic_offsets: usize, + bind_group: Option, + }, + + SetPipeline(R::ComputePipeline), + + /// Set a range of immediates to values stored in `immediates_data`. + SetImmediate { + /// The byte offset within the immediate data storage to write to. This + /// must be a multiple of four. + offset: u32, + + /// The number of bytes to write. This must be a multiple of four. + size_bytes: u32, + + /// Index in `immediates_data` of the start of the data + /// to be written. + /// + /// Note: this is not a byte offset like `offset`. Rather, it is the + /// index of the first `u32` element in `immediates_data` to read. + values_offset: u32, + }, + + Dispatch([u32; 3]), + + DispatchIndirect { + buffer: R::Buffer, + offset: wgt::BufferAddress, + }, + + PushDebugGroup { + color: u32, + len: usize, + }, + + PopDebugGroup, + + InsertDebugMarker { + color: u32, + len: usize, + }, + + WriteTimestamp { + query_set: R::QuerySet, + query_index: u32, + }, + + BeginPipelineStatisticsQuery { + query_set: R::QuerySet, + query_index: u32, + }, + + EndPipelineStatisticsQuery, +} + +/// cbindgen:ignore +pub type ArcComputeCommand = ComputeCommand; diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/draw.rs b/third_party/wgpu-core-29.0.4-patched/src/command/draw.rs new file mode 100644 index 00000000..642ba95c --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/draw.rs @@ -0,0 +1,164 @@ +use alloc::boxed::Box; + +use thiserror::Error; + +use wgt::error::{ErrorType, WebGpuError}; + +use super::bind::BinderError; +use crate::command::pass; +use crate::{ + binding_model::{BindingError, ImmediateUploadError, LateMinBufferBindingSizeMismatch}, + resource::{ + DestroyedResourceError, MissingBufferUsageError, MissingTextureUsageError, + ResourceErrorIdent, + }, + track::ResourceUsageCompatibilityError, +}; + +/// Error validating a draw call. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum DrawError { + #[error("Blend constant needs to be set")] + MissingBlendConstant, + #[error("Render pipeline must be set")] + MissingPipeline(#[from] pass::MissingPipeline), + #[error("Currently set {pipeline} requires vertex buffer {index} to be set")] + MissingVertexBuffer { + pipeline: ResourceErrorIdent, + index: u32, + }, + #[error("Index buffer must be set")] + MissingIndexBuffer, + #[error(transparent)] + IncompatibleBindGroup(#[from] Box), + #[error("Vertex {last_vertex} extends beyond limit {vertex_limit} imposed by the buffer in slot {slot}. Did you bind the correct `Vertex` step-rate vertex buffer?")] + VertexBeyondLimit { + last_vertex: u64, + vertex_limit: u64, + slot: u32, + }, + #[error("Instance {last_instance} extends beyond limit {instance_limit} imposed by the buffer in slot {slot}. Did you bind the correct `Instance` step-rate vertex buffer?")] + InstanceBeyondLimit { + last_instance: u64, + instance_limit: u64, + slot: u32, + }, + #[error("Index {last_index} extends beyond limit {index_limit}. Did you bind the correct index buffer?")] + IndexBeyondLimit { last_index: u64, index_limit: u64 }, + #[error("For indexed drawing with strip topology, {pipeline}'s strip index format {strip_index_format:?} must match index buffer format {buffer_format:?}")] + UnmatchedStripIndexFormat { + pipeline: ResourceErrorIdent, + strip_index_format: Option, + buffer_format: wgt::IndexFormat, + }, + #[error(transparent)] + BindingSizeTooSmall(#[from] LateMinBufferBindingSizeMismatch), + #[error( + "Wrong pipeline type for this draw command. Attempted to call {} draw command on {} pipeline", + if *wanted_mesh_pipeline {"mesh shader"} else {"standard"}, + if *wanted_mesh_pipeline {"standard"} else {"mesh shader"}, + )] + WrongPipelineType { wanted_mesh_pipeline: bool }, + #[error( + "Each current draw group size dimension ({current:?}) must be less or equal to {limit}, and the product must be less or equal to {max_total}" + )] + InvalidGroupSize { + current: [u32; 3], + limit: u32, + max_total: u32, + }, + #[error( + "Mesh shader calls in multiview render passes require enabling the `EXPERIMENTAL_MESH_SHADER_MULTIVIEW` feature, and the highest bit ({highest_view_index}) in the multiview mask must be <= `Limits::max_multiview_view_count` ({max_multiviews})" + )] + MeshPipelineMultiviewLimitsViolated { + highest_view_index: u32, + max_multiviews: u32, + }, +} + +impl WebGpuError for DrawError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +/// Error encountered when encoding a render command. +/// This is the shared error set between render bundles and passes. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum RenderCommandError { + #[error(transparent)] + BindGroupIndexOutOfRange(#[from] pass::BindGroupIndexOutOfRange), + #[error("Vertex buffer index {index} is greater than the device's requested `max_vertex_buffers` limit {max}")] + VertexBufferIndexOutOfRange { index: u32, max: u32 }, + #[error( + "Offset {offset} for vertex buffer in slot {slot} is not a multiple of `VERTEX_ALIGNMENT`" + )] + UnalignedVertexBuffer { slot: u32, offset: u64 }, + #[error("Offset {offset} for index buffer is not a multiple of {alignment}")] + UnalignedIndexBuffer { offset: u64, alignment: usize }, + #[error("Render pipeline targets are incompatible with render pass")] + IncompatiblePipelineTargets(#[from] crate::device::RenderPassCompatibilityError), + #[error("{0} writes to depth, while the pass has read-only depth access")] + IncompatibleDepthAccess(ResourceErrorIdent), + #[error("{0} writes to stencil, while the pass has read-only stencil access")] + IncompatibleStencilAccess(ResourceErrorIdent), + #[error(transparent)] + ResourceUsageCompatibility(#[from] ResourceUsageCompatibilityError), + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + #[error(transparent)] + MissingTextureUsage(#[from] MissingTextureUsageError), + #[error(transparent)] + ImmediateData(#[from] ImmediateUploadError), + #[error(transparent)] + BindingError(#[from] BindingError), + #[error("Viewport size {{ w: {w}, h: {h} }} greater than device's requested `max_texture_dimension_2d` limit {max}, or less than zero")] + InvalidViewportRectSize { w: f32, h: f32, max: u32 }, + #[error("Viewport has invalid rect {rect:?} for device's requested `max_texture_dimension_2d` limit; Origin less than -2 * `max_texture_dimension_2d` ({min}), or rect extends past 2 * `max_texture_dimension_2d` - 1 ({max})")] + InvalidViewportRectPosition { rect: Rect, min: f32, max: f32 }, + #[error("Viewport minDepth {0} and/or maxDepth {1} are not in [0, 1]")] + InvalidViewportDepth(f32, f32), + #[error("Scissor {0:?} is not contained in the render target {1:?}")] + InvalidScissorRect(Rect, wgt::Extent3d), + #[error("Support for {0} is not implemented yet")] + Unimplemented(&'static str), +} + +impl WebGpuError for RenderCommandError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::IncompatiblePipelineTargets(e) => e.webgpu_error_type(), + Self::ResourceUsageCompatibility(e) => e.webgpu_error_type(), + Self::DestroyedResource(e) => e.webgpu_error_type(), + Self::MissingBufferUsage(e) => e.webgpu_error_type(), + Self::MissingTextureUsage(e) => e.webgpu_error_type(), + Self::ImmediateData(e) => e.webgpu_error_type(), + Self::BindingError(e) => e.webgpu_error_type(), + + Self::BindGroupIndexOutOfRange { .. } + | Self::VertexBufferIndexOutOfRange { .. } + | Self::UnalignedIndexBuffer { .. } + | Self::UnalignedVertexBuffer { .. } + | Self::IncompatibleDepthAccess(..) + | Self::IncompatibleStencilAccess(..) + | Self::InvalidViewportRectSize { .. } + | Self::InvalidViewportRectPosition { .. } + | Self::InvalidViewportDepth(..) + | Self::InvalidScissorRect(..) + | Self::Unimplemented(..) => ErrorType::Validation, + } + } +} + +#[derive(Clone, Copy, Debug, Default)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Rect { + pub x: T, + pub y: T, + pub w: T, + pub h: T, +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/encoder.rs b/third_party/wgpu-core-29.0.4-patched/src/command/encoder.rs new file mode 100644 index 00000000..19222c6e --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/encoder.rs @@ -0,0 +1,53 @@ +use alloc::{sync::Arc, vec::Vec}; + +use crate::{ + command::memory_init::CommandBufferTextureMemoryActions, + device::{queue::TempResource, Device}, + init_tracker::BufferInitTrackerAction, + ray_tracing::AsAction, + snatch::SnatchGuard, + track::Tracker, +}; + +/// State applicable when encoding commands onto a compute pass, render pass, or +/// directly to a command encoder. +/// +/// Most encoding routines just want to receive an open encoder, write +/// command(s) to it, and leave it open for whatever is next. In this case the +/// `E` type parameter has the default value of `dyn hal::DynCommandEncoder`. To +/// avoid confusion about encoder state, we set the convention that _the encoder +/// in an `EncodingState` holding a bare HAL reference must always be open_. +/// +/// Compute and render passes are more complicated. Because they record a +/// command buffer for a housekeeping pre-pass which is inserted before the pass +/// itself, the first thing they will do is close and reopen the encoder if it +/// is already open. Unnecessary empty HAL passes can be avoided by passing them +/// the encoder in whatever state it happens to be. In this case, `E` is +/// `InnerCommandEncoder`, which tracks the state of the encoder. The callee +/// (the render or compute pass) will open and close the encoder as necessary. +/// +/// This structure is not supported by cbindgen because it contains a trait +/// object reference. +/// +/// cbindgen:ignore +pub(crate) struct EncodingState<'snatch_guard, 'cmd_enc, E: ?Sized = dyn hal::DynCommandEncoder> { + pub(crate) device: &'cmd_enc Arc, + + pub(crate) raw_encoder: &'cmd_enc mut E, + + pub(crate) tracker: &'cmd_enc mut Tracker, + pub(crate) buffer_memory_init_actions: &'cmd_enc mut Vec, + pub(crate) texture_memory_actions: &'cmd_enc mut CommandBufferTextureMemoryActions, + pub(crate) as_actions: &'cmd_enc mut Vec, + pub(crate) temp_resources: &'cmd_enc mut Vec, + pub(crate) indirect_draw_validation_resources: + &'cmd_enc mut crate::indirect_validation::DrawResources, + + pub(crate) snatch_guard: &'snatch_guard SnatchGuard<'snatch_guard>, + + /// Current debug scope nesting depth. + /// + /// When encoding a compute or render pass, this is the depth of debug + /// scopes in the pass, not the depth of debug scopes in the parent encoder. + pub(crate) debug_scope_depth: &'cmd_enc mut u32, +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/encoder_command.rs b/third_party/wgpu-core-29.0.4-patched/src/command/encoder_command.rs new file mode 100644 index 00000000..2dfe2459 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/encoder_command.rs @@ -0,0 +1,186 @@ +use core::{convert::Infallible, num::NonZero}; + +use alloc::{string::String, sync::Arc, vec::Vec}; +#[cfg(feature = "serde")] +use macro_rules_attribute::{apply, attribute_alias}; + +use crate::{ + command::ColorAttachments, + id, + instance::Surface, + resource::{Buffer, QuerySet, Texture}, +}; + +pub trait ReferenceType { + type Buffer: Clone + core::fmt::Debug; + type Surface: Clone; // Surface does not implement Debug, although it probably could. + type Texture: Clone + core::fmt::Debug; + type TextureView: Clone + core::fmt::Debug; + type ExternalTexture: Clone + core::fmt::Debug; + type QuerySet: Clone + core::fmt::Debug; + type BindGroup: Clone + core::fmt::Debug; + type RenderPipeline: Clone + core::fmt::Debug; + type RenderBundle: Clone + core::fmt::Debug; + type ComputePipeline: Clone + core::fmt::Debug; + type Blas: Clone + core::fmt::Debug; + type Tlas: Clone + core::fmt::Debug; +} + +/// Reference wgpu objects via numeric IDs assigned by [`crate::identity::IdentityManager`]. +#[derive(Clone, Debug)] +pub struct IdReferences; + +/// Reference wgpu objects via the integer value of pointers. +/// +/// This is used for trace recording and playback. Recording stores the pointer +/// value of `Arc` references in the trace. Playback uses the integer values +/// as keys to a `HashMap`. +#[cfg(any(feature = "trace", feature = "replay"))] +#[doc(hidden)] +#[derive(Clone, Debug)] +pub struct PointerReferences; + +/// Reference wgpu objects via `Arc`s. +#[derive(Clone, Debug)] +pub struct ArcReferences; + +impl ReferenceType for IdReferences { + type Buffer = id::BufferId; + type Surface = id::SurfaceId; + type Texture = id::TextureId; + type TextureView = id::TextureViewId; + type ExternalTexture = id::ExternalTextureId; + type QuerySet = id::QuerySetId; + type BindGroup = id::BindGroupId; + type RenderPipeline = id::RenderPipelineId; + type RenderBundle = id::RenderBundleId; + type ComputePipeline = id::ComputePipelineId; + type Blas = id::BlasId; + type Tlas = id::TlasId; +} + +#[cfg(any(feature = "trace", feature = "replay"))] +impl ReferenceType for PointerReferences { + type Buffer = id::PointerId; + type Surface = id::PointerId; + type Texture = id::PointerId; + type TextureView = id::PointerId; + type ExternalTexture = id::PointerId; + type QuerySet = id::PointerId; + type BindGroup = id::PointerId; + type RenderPipeline = id::PointerId; + type RenderBundle = id::PointerId; + type ComputePipeline = id::PointerId; + type Blas = id::PointerId; + type Tlas = id::PointerId; +} + +impl ReferenceType for ArcReferences { + type Buffer = Arc; + type Surface = Arc; + type Texture = Arc; + type TextureView = Arc; + type ExternalTexture = Arc; + type QuerySet = Arc; + type BindGroup = Arc; + type RenderPipeline = Arc; + type RenderBundle = Arc; + type ComputePipeline = Arc; + type Blas = Arc; + type Tlas = Arc; +} + +#[cfg(feature = "serde")] +attribute_alias! { + #[apply(serde_object_reference_struct)] = + #[derive(serde::Serialize, serde::Deserialize)] + #[serde(bound = + "R::Buffer: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::Surface: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::Texture: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::TextureView: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::ExternalTexture: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::QuerySet: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::BindGroup: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::RenderPipeline: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::RenderBundle: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::ComputePipeline: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::Blas: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + R::Tlas: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + wgt::BufferTransition: serde::Serialize + for<'d> serde::Deserialize<'d>,\ + wgt::TextureTransition: serde::Serialize + for<'d> serde::Deserialize<'d>" + )]; +} + +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", apply(serde_object_reference_struct))] +pub enum Command { + CopyBufferToBuffer { + src: R::Buffer, + src_offset: wgt::BufferAddress, + dst: R::Buffer, + dst_offset: wgt::BufferAddress, + size: Option, + }, + CopyBufferToTexture { + src: wgt::TexelCopyBufferInfo, + dst: wgt::TexelCopyTextureInfo, + size: wgt::Extent3d, + }, + CopyTextureToBuffer { + src: wgt::TexelCopyTextureInfo, + dst: wgt::TexelCopyBufferInfo, + size: wgt::Extent3d, + }, + CopyTextureToTexture { + src: wgt::TexelCopyTextureInfo, + dst: wgt::TexelCopyTextureInfo, + size: wgt::Extent3d, + }, + ClearBuffer { + dst: R::Buffer, + offset: wgt::BufferAddress, + size: Option, + }, + ClearTexture { + dst: R::Texture, + subresource_range: wgt::ImageSubresourceRange, + }, + WriteTimestamp { + query_set: R::QuerySet, + query_index: u32, + }, + ResolveQuerySet { + query_set: R::QuerySet, + start_query: u32, + query_count: u32, + destination: R::Buffer, + destination_offset: wgt::BufferAddress, + }, + PushDebugGroup(String), + PopDebugGroup, + InsertDebugMarker(String), + RunComputePass { + pass: crate::command::BasePass, Infallible>, + timestamp_writes: Option>, + }, + RunRenderPass { + pass: crate::command::BasePass, Infallible>, + color_attachments: ColorAttachments, + depth_stencil_attachment: + Option>, + timestamp_writes: Option>, + occlusion_query_set: Option, + multiview_mask: Option>, + }, + BuildAccelerationStructures { + blas: Vec>, + tlas: Vec>, + }, + TransitionResources { + buffer_transitions: Vec>, + texture_transitions: Vec>, + }, +} + +pub type ArcCommand = Command; diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/ffi.rs b/third_party/wgpu-core-29.0.4-patched/src/command/ffi.rs new file mode 100644 index 00000000..4b9a5eff --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/ffi.rs @@ -0,0 +1,9 @@ +//! Types that are useful for FFI bindings to `wgpu`. + +use crate::{command::IdReferences, id}; + +pub type TexelCopyBufferInfo = wgt::TexelCopyBufferInfo; +pub type TexelCopyTextureInfo = wgt::TexelCopyTextureInfo; +pub type CopyExternalImageDestInfo = wgt::CopyExternalImageDestInfo; + +pub type Command = super::Command; diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/memory_init.rs b/third_party/wgpu-core-29.0.4-patched/src/command/memory_init.rs new file mode 100644 index 00000000..faca4170 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/memory_init.rs @@ -0,0 +1,338 @@ +use alloc::{ + sync::Arc, + vec::{Drain, Vec}, +}; +use core::ops::Range; + +use hashbrown::hash_map::Entry; + +use crate::{ + device::Device, + init_tracker::*, + resource::{DestroyedResourceError, ParentDevice, RawResourceAccess, Texture, Trackable}, + snatch::SnatchGuard, + track::{DeviceTracker, TextureTracker}, + FastHashMap, +}; + +use super::{clear_texture, BakedCommands, ClearError}; + +/// Surface that was discarded by `StoreOp::Discard` of a preceding renderpass. +/// Any read access to this surface needs to be preceded by a texture initialization. +#[derive(Clone)] +pub(crate) struct TextureSurfaceDiscard { + pub texture: Arc, + pub mip_level: u32, + pub layer: u32, +} + +pub(crate) type SurfacesInDiscardState = Vec; + +#[derive(Default)] +pub(crate) struct CommandBufferTextureMemoryActions { + /// The tracker actions that we need to be executed before the command + /// buffer is executed. + init_actions: Vec, + /// All the discards that haven't been followed by init again within the + /// command buffer i.e. everything in this list resets the texture init + /// state *after* the command buffer execution + discards: Vec, +} + +impl CommandBufferTextureMemoryActions { + pub(crate) fn drain_init_actions(&mut self) -> Drain<'_, TextureInitTrackerAction> { + self.init_actions.drain(..) + } + + pub(crate) fn discard(&mut self, discard: TextureSurfaceDiscard) { + self.discards.push(discard); + } + + // Registers a TextureInitTrackerAction. + // Returns previously discarded surface that need to be initialized *immediately* now. + // Only returns a non-empty list if action is MemoryInitKind::NeedsInitializedMemory. + #[must_use] + pub(crate) fn register_init_action( + &mut self, + action: &TextureInitTrackerAction, + ) -> SurfacesInDiscardState { + let mut immediately_necessary_clears = SurfacesInDiscardState::new(); + + // Note that within a command buffer we may stack arbitrary memory init + // actions on the same texture Since we react to them in sequence, they + // are going to be dropped again at queue submit + // + // We don't need to add MemoryInitKind::NeedsInitializedMemory to + // init_actions if a surface is part of the discard list. But that would + // mean splitting up the action which is more than we'd win here. + self.init_actions.extend( + action + .texture + .initialization_status + .read() + .check_action(action), + ); + + // We expect very few discarded surfaces at any point in time which is + // why a simple linear search is likely best. (i.e. most of the time + // self.discards is empty!) + let init_actions = &mut self.init_actions; + self.discards.retain(|discarded_surface| { + if discarded_surface.texture.is_equal(&action.texture) + && action.range.layer_range.contains(&discarded_surface.layer) + && action + .range + .mip_range + .contains(&discarded_surface.mip_level) + { + if let MemoryInitKind::NeedsInitializedMemory = action.kind { + immediately_necessary_clears.push(discarded_surface.clone()); + + // Mark surface as implicitly initialized (this is relevant + // because it might have been uninitialized prior to + // discarding + init_actions.push(TextureInitTrackerAction { + texture: discarded_surface.texture.clone(), + range: TextureInitRange { + mip_range: discarded_surface.mip_level + ..(discarded_surface.mip_level + 1), + layer_range: discarded_surface.layer..(discarded_surface.layer + 1), + }, + kind: MemoryInitKind::ImplicitlyInitialized, + }); + } + false + } else { + true + } + }); + + immediately_necessary_clears + } + + // Shortcut for register_init_action when it is known that the action is an + // implicit init, not requiring any immediate resource init. + pub(crate) fn register_implicit_init( + &mut self, + texture: &Arc, + range: TextureInitRange, + ) { + let must_be_empty = self.register_init_action(&TextureInitTrackerAction { + texture: texture.clone(), + range, + kind: MemoryInitKind::ImplicitlyInitialized, + }); + assert!(must_be_empty.is_empty()); + } +} + +// Utility function that takes discarded surfaces from (several calls to) +// register_init_action and initializes them on the spot. +// +// Takes care of barriers as well! +pub(crate) fn fixup_discarded_surfaces>( + inits: InitIter, + encoder: &mut dyn hal::DynCommandEncoder, + texture_tracker: &mut TextureTracker, + device: &Device, + snatch_guard: &SnatchGuard<'_>, +) { + for init in inits { + clear_texture( + &init.texture, + TextureInitRange { + mip_range: init.mip_level..(init.mip_level + 1), + layer_range: init.layer..(init.layer + 1), + }, + encoder, + texture_tracker, + &device.alignments, + device.zero_buffer.as_ref(), + snatch_guard, + device.instance_flags, + ) + .unwrap(); + } +} + +impl BakedCommands { + // inserts all buffer initializations that are going to be needed for + // executing the commands and updates resource init states accordingly + pub(crate) fn initialize_buffer_memory( + &mut self, + device_tracker: &mut DeviceTracker, + snatch_guard: &SnatchGuard<'_>, + ) -> Result<(), DestroyedResourceError> { + profiling::scope!("initialize_buffer_memory"); + + // Gather init ranges for each buffer so we can collapse them. + // It is not possible to do this at an earlier point since previously + // executed command buffer change the resource init state. + let mut uninitialized_ranges_per_buffer = FastHashMap::default(); + for buffer_use in self.buffer_memory_init_actions.drain(..) { + let mut initialization_status = buffer_use.buffer.initialization_status.write(); + + // align the end to 4 + let end_remainder = buffer_use.range.end % wgt::COPY_BUFFER_ALIGNMENT; + let end = if end_remainder == 0 { + buffer_use.range.end + } else { + buffer_use.range.end + wgt::COPY_BUFFER_ALIGNMENT - end_remainder + }; + let uninitialized_ranges = initialization_status.drain(buffer_use.range.start..end); + + match buffer_use.kind { + MemoryInitKind::ImplicitlyInitialized => {} + MemoryInitKind::NeedsInitializedMemory => { + match uninitialized_ranges_per_buffer.entry(buffer_use.buffer.tracker_index()) { + Entry::Vacant(e) => { + e.insert(( + buffer_use.buffer.clone(), + uninitialized_ranges.collect::>>(), + )); + } + Entry::Occupied(mut e) => { + e.get_mut().1.extend(uninitialized_ranges); + } + } + } + } + } + + for (buffer, mut ranges) in uninitialized_ranges_per_buffer.into_values() { + // Collapse touching ranges. + ranges.sort_by_key(|r| r.start); + for i in (1..ranges.len()).rev() { + // The memory init tracker made sure of this! + assert!(ranges[i - 1].end <= ranges[i].start); + if ranges[i].start == ranges[i - 1].end { + ranges[i - 1].end = ranges[i].end; + ranges.swap_remove(i); // Ordering not important at this point + } + } + + // Don't do use_replace since the buffer may already no longer have + // a ref_count. + // + // However, we *know* that it is currently in use, so the tracker + // must already know about it. + let transition = device_tracker + .buffers + .set_single(&buffer, wgt::BufferUses::COPY_DST); + + let raw_buf = buffer.try_raw(snatch_guard)?; + + unsafe { + self.encoder.raw.transition_buffers( + transition + .map(|pending| pending.into_hal(&buffer, snatch_guard)) + .as_slice(), + ); + } + + for range in ranges.iter() { + assert!( + range.start % wgt::COPY_BUFFER_ALIGNMENT == 0, + "Buffer {:?} has an uninitialized range with a start \ + not aligned to 4 (start was {})", + raw_buf, + range.start + ); + assert!( + range.end % wgt::COPY_BUFFER_ALIGNMENT == 0, + "Buffer {:?} has an uninitialized range with an end \ + not aligned to 4 (end was {})", + raw_buf, + range.end + ); + + unsafe { + self.encoder.raw.clear_buffer(raw_buf, range.clone()); + } + } + } + Ok(()) + } + + // inserts all texture initializations that are going to be needed for + // executing the commands and updates resource init states accordingly any + // textures that are left discarded by this command buffer will be marked as + // uninitialized + pub(crate) fn initialize_texture_memory( + &mut self, + device_tracker: &mut DeviceTracker, + device: &Device, + snatch_guard: &SnatchGuard<'_>, + ) -> Result<(), DestroyedResourceError> { + profiling::scope!("initialize_texture_memory"); + + let mut ranges: Vec = Vec::new(); + for texture_use in self.texture_memory_actions.drain_init_actions() { + let mut initialization_status = texture_use.texture.initialization_status.write(); + let use_range = texture_use.range; + let affected_mip_trackers = initialization_status + .mips + .iter_mut() + .enumerate() + .skip(use_range.mip_range.start as usize) + .take((use_range.mip_range.end - use_range.mip_range.start) as usize); + + match texture_use.kind { + MemoryInitKind::ImplicitlyInitialized => { + for (_, mip_tracker) in affected_mip_trackers { + mip_tracker.drain(use_range.layer_range.clone()); + } + } + MemoryInitKind::NeedsInitializedMemory => { + for (mip_level, mip_tracker) in affected_mip_trackers { + for layer_range in mip_tracker.drain(use_range.layer_range.clone()) { + ranges.push(TextureInitRange { + mip_range: (mip_level as u32)..(mip_level as u32 + 1), + layer_range, + }); + } + } + } + } + + // TODO: Could we attempt some range collapsing here? + for range in ranges.drain(..) { + let clear_result = clear_texture( + &texture_use.texture, + range, + self.encoder.raw.as_mut(), + &mut device_tracker.textures, + &device.alignments, + device.zero_buffer.as_ref(), + snatch_guard, + device.instance_flags, + ); + + // A Texture can be destroyed between the command recording + // and now, this is out of our control so we have to handle + // it gracefully. + if let Err(ClearError::DestroyedResource(e)) = clear_result { + return Err(e); + } + + // Other errors are unexpected. + if let Err(error) = clear_result { + panic!("{error}"); + } + } + } + + // Now that all buffers/textures have the proper init state for before + // cmdbuf start, we discard init states for textures it left discarded + // after its execution. + for surface_discard in self.texture_memory_actions.discards.iter() { + surface_discard + .texture + .initialization_status + .write() + .discard(surface_discard.mip_level, surface_discard.layer); + } + + Ok(()) + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/command/mod.rs b/third_party/wgpu-core-29.0.4-patched/src/command/mod.rs new file mode 100644 index 00000000..9160cdb1 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/command/mod.rs @@ -0,0 +1,2076 @@ +//! # Command Encoding +//! +//! TODO: High-level description of command encoding. +//! +//! The convention in this module is that functions accepting a [`&mut dyn +//! hal::DynCommandEncoder`] are low-level helpers and may assume the encoder is +//! in the open state, ready to encode commands. Encoders that are not open +//! should be nested within some other container that provides additional +//! state tracking, like [`InnerCommandEncoder`]. + +mod allocator; +mod bind; +mod bundle; +mod clear; +mod compute; +mod compute_command; +mod draw; +mod encoder; +mod encoder_command; +pub mod ffi; +mod memory_init; +mod pass; +mod query; +mod ray_tracing; +mod render; +mod render_command; +mod timestamp_writes; +mod transfer; +mod transition_resources; + +use alloc::{borrow::ToOwned as _, boxed::Box, string::String, sync::Arc, vec::Vec}; +use core::convert::Infallible; +use core::mem::{self, ManuallyDrop}; +use core::{ops, panic}; + +#[cfg(feature = "serde")] +pub(crate) use self::encoder_command::serde_object_reference_struct; +#[cfg(any(feature = "trace", feature = "replay"))] +#[doc(hidden)] +pub use self::encoder_command::PointerReferences; +// This module previously did `pub use *` for some of the submodules. When that +// was removed, every type that was previously public via `use *` was listed +// here. Some types (in particular `CopySide`) may be exported unnecessarily. +pub use self::{ + bundle::{ + bundle_ffi, CreateRenderBundleError, ExecutionError, RenderBundle, RenderBundleDescriptor, + RenderBundleEncoder, RenderBundleEncoderDescriptor, RenderBundleError, + RenderBundleErrorInner, + }, + clear::ClearError, + compute::{ + ComputeBasePass, ComputePass, ComputePassDescriptor, ComputePassError, + ComputePassErrorInner, DispatchError, + }, + compute_command::ArcComputeCommand, + draw::{DrawError, Rect, RenderCommandError}, + encoder_command::{ArcCommand, ArcReferences, Command, IdReferences, ReferenceType}, + query::{QueryError, QueryUseError, ResolveError, SimplifiedQueryType}, + render::{ + ArcRenderPassColorAttachment, AttachmentError, AttachmentErrorLocation, + ColorAttachmentError, ColorAttachments, LoadOp, PassChannel, RenderBasePass, RenderPass, + RenderPassColorAttachment, RenderPassDepthStencilAttachment, RenderPassDescriptor, + RenderPassError, RenderPassErrorInner, ResolvedPassChannel, + ResolvedRenderPassDepthStencilAttachment, StoreOp, + }, + render_command::ArcRenderCommand, + transfer::{CopySide, TransferError}, + transition_resources::TransitionResourcesError, +}; +pub(crate) use self::{ + clear::clear_texture, + encoder::EncodingState, + memory_init::CommandBufferTextureMemoryActions, + render::{get_dst_stride_of_indirect_args, get_src_stride_of_indirect_args, VertexLimits}, + transfer::{ + extract_texture_selector, validate_linear_texture_data, validate_texture_buffer_copy, + validate_texture_copy_dst_format, validate_texture_copy_range, + }, +}; + +pub(crate) use allocator::CommandAllocator; + +/// cbindgen:ignore +pub use self::{compute_command::ComputeCommand, render_command::RenderCommand}; + +pub(crate) use timestamp_writes::ArcPassTimestampWrites; +pub use timestamp_writes::PassTimestampWrites; + +use crate::binding_model::BindingError; +use crate::device::queue::TempResource; +use crate::device::{Device, DeviceError, MissingFeatures}; +use crate::id::Id; +use crate::lock::{rank, Mutex}; +use crate::snatch::SnatchGuard; + +use crate::init_tracker::BufferInitTrackerAction; +use crate::ray_tracing::{AsAction, BuildAccelerationStructureError}; +use crate::resource::{ + DestroyedResourceError, Fallible, InvalidResourceError, Labeled, ParentDevice as _, QuerySet, +}; +use crate::storage::Storage; +use crate::track::{DeviceTracker, ResourceUsageCompatibilityError, Tracker, UsageScope}; +use crate::{api_log, global::Global, id, resource_log, Label}; +use crate::{hal_label, LabelHelpers}; + +use wgt::error::{ErrorType, WebGpuError}; + +use thiserror::Error; + +/// cbindgen:ignore +pub type TexelCopyBufferInfo = ffi::TexelCopyBufferInfo; +/// cbindgen:ignore +pub type TexelCopyTextureInfo = ffi::TexelCopyTextureInfo; +/// cbindgen:ignore +pub type CopyExternalImageDestInfo = ffi::CopyExternalImageDestInfo; + +const IMMEDIATES_CLEAR_ARRAY: &[u32] = &[0_u32; 64]; + +pub(crate) struct EncoderErrorState { + error: CommandEncoderError, + + #[cfg(feature = "trace")] + trace_commands: Option>>, +} + +/// Construct an `EncoderErrorState` with only a `CommandEncoderError` (without +/// any traced commands). +/// +/// This is used in cases where pass begin/end were mismatched, if the same +/// encoder was finished multiple times, or in the status of a command buffer +/// (in which case the commands were already saved to the trace). In some of +/// these cases there may be commands that could be saved to the trace, but if +/// the application is that confused about using encoders, it's not clear +/// whether it's worth the effort to try and preserve the commands. +fn make_error_state>(error: E) -> CommandEncoderStatus { + CommandEncoderStatus::Error(EncoderErrorState { + error: error.into(), + + #[cfg(feature = "trace")] + trace_commands: None, + }) +} + +/// The current state of a command or pass encoder. +/// +/// In the WebGPU spec, the state of an encoder (open, locked, or ended) is +/// orthogonal to the validity of the encoder. However, this enum does not +/// represent the state of an invalid encoder. +pub(crate) enum CommandEncoderStatus { + /// Ready to record commands. An encoder's initial state. + /// + /// Command building methods like [`command_encoder_clear_buffer`] and + /// [`compute_pass_end`] require the encoder to be in this + /// state. + /// + /// This corresponds to WebGPU's "open" state. + /// See + /// + /// [`command_encoder_clear_buffer`]: Global::command_encoder_clear_buffer + /// [`compute_pass_end`]: Global::compute_pass_end + Recording(CommandBufferMutable), + + /// Locked by a render or compute pass. + /// + /// This state is entered when a render/compute pass is created, + /// and exited when the pass is ended. + /// + /// As long as the command encoder is locked, any command building operation + /// on it will fail and put the encoder into the [`Self::Error`] state. See + /// + Locked(CommandBufferMutable), + + Consumed, + + /// Command recording is complete, and the buffer is ready for submission. + /// + /// [`Global::command_encoder_finish`] transitions a + /// `CommandBuffer` from the `Recording` state into this state. + /// + /// [`Global::queue_submit`] requires that command buffers are + /// in this state. + /// + /// This corresponds to WebGPU's "ended" state. + /// See + Finished(CommandBufferMutable), + + /// The command encoder is invalid. + /// + /// The error that caused the invalidation is stored here, and will + /// be raised by `CommandEncoder.finish()`. + Error(EncoderErrorState), + + /// Temporary state used internally by methods on `CommandEncoderStatus`. + /// Encoder should never be left in this state. + Transitioning, +} + +impl CommandEncoderStatus { + #[doc(hidden)] + fn replay(&mut self, commands: Vec>) { + let Self::Recording(cmd_buf_data) = self else { + panic!("encoder should be in the recording state"); + }; + cmd_buf_data.commands.extend(commands); + } + + /// Push a command provided by a closure onto the encoder. + /// + /// If the encoder is in the [`Self::Recording`] state, calls the closure to + /// obtain a command, and pushes it onto the encoder. If the closure returns + /// an error, stores that error in the encoder for later reporting when + /// `finish()` is called. Returns `Ok(())` even if the closure returned an + /// error. + /// + /// If the encoder is not in the [`Self::Recording`] state, the closure will + /// not be called and nothing will be recorded. The encoder will be + /// invalidated (if it is not already). If the error is a [validation error + /// that should be raised immediately][ves], returns it in `Err`, otherwise, + /// returns `Ok(())`. + /// + /// [ves]: https://www.w3.org/TR/webgpu/#abstract-opdef-validate-the-encoder-state + fn push_with Result, E: Clone + Into>( + &mut self, + f: F, + ) -> Result<(), EncoderStateError> { + match self { + Self::Recording(cmd_buf_data) => { + cmd_buf_data.encoder.api.set(EncodingApi::Wgpu); + match f() { + Ok(cmd) => cmd_buf_data.commands.push(cmd), + Err(err) => { + self.invalidate(err); + } + } + Ok(()) + } + Self::Locked(_) => { + // Invalidate the encoder and do not record anything, but do not + // return an immediate validation error. + self.invalidate(EncoderStateError::Locked); + Ok(()) + } + // Encoder is ended. Invalidate the encoder, do not record anything, + // and return an immediate validation error. + Self::Finished(_) => Err(self.invalidate(EncoderStateError::Ended)), + Self::Consumed => Err(EncoderStateError::Ended), + // Encoder is already invalid. Do not record anything, but do not + // return an immediate validation error. + Self::Error(_) => Ok(()), + Self::Transitioning => unreachable!(), + } + } + + /// Call a closure with the inner command buffer structure. + /// + /// If the encoder is in the [`Self::Recording`] state, calls the provided + /// closure. If the closure returns an error, stores that error in the + /// encoder for later reporting when `finish()` is called. Returns `Ok(())` + /// even if the closure returned an error. + /// + /// If the encoder is not in the [`Self::Recording`] state, the closure will + /// not be called. The encoder will be invalidated (if it is not already). + /// If the error is a [validation error that should be raised + /// immediately][ves], returns it in `Err`, otherwise, returns `Ok(())`. + /// + /// [ves]: https://www.w3.org/TR/webgpu/#abstract-opdef-validate-the-encoder-state + fn with_buffer< + F: FnOnce(&mut CommandBufferMutable) -> Result<(), E>, + E: Clone + Into, + >( + &mut self, + api: EncodingApi, + f: F, + ) -> Result<(), EncoderStateError> { + match self { + Self::Recording(inner) => { + inner.encoder.api.set(api); + RecordingGuard { inner: self }.record(f); + Ok(()) + } + Self::Locked(_) => { + // Invalidate the encoder and do not record anything, but do not + // return an immediate validation error. + self.invalidate(EncoderStateError::Locked); + Ok(()) + } + // Encoder is ended. Invalidate the encoder, do not record anything, + // and return an immediate validation error. + Self::Finished(_) => Err(self.invalidate(EncoderStateError::Ended)), + Self::Consumed => Err(EncoderStateError::Ended), + // Encoder is already invalid. Do not record anything, but do not + // return an immediate validation error. + Self::Error(_) => Ok(()), + Self::Transitioning => unreachable!(), + } + } + + /// Special version of record used by `command_encoder_as_hal_mut`. This + /// differs from the regular version in two ways: + /// + /// 1. The recording closure is infallible. + /// 2. The recording closure takes `Option<&mut CommandBufferMutable>`, and + /// in the case that the encoder is not in a valid state for recording, the + /// closure is still called, with `None` as its argument. + pub(crate) fn record_as_hal_mut) -> T>( + &mut self, + f: F, + ) -> T { + match self { + Self::Recording(inner) => { + inner.encoder.api.set(EncodingApi::Raw); + RecordingGuard { inner: self }.record_as_hal_mut(f) + } + Self::Locked(_) => { + self.invalidate(EncoderStateError::Locked); + f(None) + } + Self::Finished(_) => { + self.invalidate(EncoderStateError::Ended); + f(None) + } + Self::Consumed => f(None), + Self::Error(_) => f(None), + Self::Transitioning => unreachable!(), + } + } + + /// Locks the encoder by putting it in the [`Self::Locked`] state. + /// + /// Render or compute passes call this on start. At the end of the pass, + /// they call [`Self::unlock_encoder`] to put the [`CommandBuffer`] back + /// into the [`Self::Recording`] state. + fn lock_encoder(&mut self) -> Result<(), EncoderStateError> { + match mem::replace(self, Self::Transitioning) { + Self::Recording(inner) => { + *self = Self::Locked(inner); + Ok(()) + } + st @ Self::Finished(_) => { + // Attempting to open a pass on a finished encoder raises a + // validation error but does not invalidate the encoder. This is + // related to https://github.com/gpuweb/gpuweb/issues/5207. + *self = st; + Err(EncoderStateError::Ended) + } + Self::Locked(_) => Err(self.invalidate(EncoderStateError::Locked)), + st @ Self::Consumed => { + *self = st; + Err(EncoderStateError::Ended) + } + st @ Self::Error(_) => { + *self = st; + Err(EncoderStateError::Invalid) + } + Self::Transitioning => unreachable!(), + } + } + + /// Unlocks the encoder and puts it back into the [`Self::Recording`] state. + /// + /// This function is the unlocking counterpart to [`Self::lock_encoder`]. It + /// is only valid to call this function if the encoder is in the + /// [`Self::Locked`] state. + /// + /// If the encoder is in a state other than [`Self::Locked`] and a + /// validation error should be raised immediately, returns it in `Err`, + /// otherwise, stores the error in the encoder and returns `Ok(())`. + fn unlock_encoder(&mut self) -> Result<(), EncoderStateError> { + match mem::replace(self, Self::Transitioning) { + Self::Locked(inner) => { + *self = Self::Recording(inner); + Ok(()) + } + st @ Self::Finished(_) => { + *self = st; + Err(EncoderStateError::Ended) + } + Self::Recording(_) => { + *self = make_error_state(EncoderStateError::Unlocked); + Err(EncoderStateError::Unlocked) + } + st @ Self::Consumed => { + *self = st; + Err(EncoderStateError::Ended) + } + st @ Self::Error(_) => { + // Encoder is already invalid. The error will be reported by + // `CommandEncoder.finish`. + *self = st; + Ok(()) + } + Self::Transitioning => unreachable!(), + } + } + + fn finish(&mut self) -> Self { + // Replace our state with `Consumed`, and return either the inner + // state or an error, to be transferred to the command buffer. + match mem::replace(self, Self::Consumed) { + Self::Recording(inner) => { + // Raw encoding leaves the encoder open in `command_encoder_as_hal_mut`. + // Otherwise, nothing should have opened it yet. + if inner.encoder.api != EncodingApi::Raw { + assert!(!inner.encoder.is_open); + } + Self::Finished(inner) + } + Self::Consumed | Self::Finished(_) => make_error_state(EncoderStateError::Ended), + Self::Locked(_) => make_error_state(EncoderStateError::Locked), + st @ Self::Error(_) => st, + Self::Transitioning => unreachable!(), + } + } + + /// Invalidate the command encoder due to an error. + /// + /// The error `err` is stored so that it can be reported when the encoder is + /// finished. If tracing is enabled, the traced commands are also stored. + /// + /// Since we do not track the state of an invalid encoder, it is not + /// necessary to unlock an encoder that has been invalidated. + fn invalidate>(&mut self, err: E) -> E { + #[cfg(feature = "trace")] + let trace_commands = match self { + Self::Recording(cmd_buf_data) => Some( + mem::take(&mut cmd_buf_data.commands) + .into_iter() + .map(crate::device::trace::IntoTrace::into_trace) + .collect(), + ), + _ => None, + }; + + let enc_err = err.clone().into(); + api_log!("Invalidating command encoder: {enc_err:?}"); + *self = Self::Error(EncoderErrorState { + error: enc_err, + #[cfg(feature = "trace")] + trace_commands, + }); + err + } +} + +/// A guard to enforce error reporting, for a [`CommandBuffer`] in the [`Recording`] state. +/// +/// An [`RecordingGuard`] holds a mutable reference to a [`CommandEncoderStatus`] that +/// has been verified to be in the [`Recording`] state. The [`RecordingGuard`] dereferences +/// mutably to the [`CommandBufferMutable`] that the status holds. +/// +/// Dropping an [`RecordingGuard`] sets the [`CommandBuffer`]'s state to +/// [`CommandEncoderStatus::Error`]. If your use of the guard was +/// successful, call its [`mark_successful`] method to dispose of it. +/// +/// [`Recording`]: CommandEncoderStatus::Recording +/// [`mark_successful`]: Self::mark_successful +pub(crate) struct RecordingGuard<'a> { + inner: &'a mut CommandEncoderStatus, +} + +impl<'a> RecordingGuard<'a> { + pub(crate) fn mark_successful(self) { + mem::forget(self) + } + + fn record< + F: FnOnce(&mut CommandBufferMutable) -> Result<(), E>, + E: Clone + Into, + >( + mut self, + f: F, + ) { + match f(&mut self) { + Ok(()) => self.mark_successful(), + Err(err) => { + self.inner.invalidate(err); + } + } + } + + /// Special version of record used by `command_encoder_as_hal_mut`. This + /// version takes an infallible recording closure. + pub(crate) fn record_as_hal_mut) -> T>( + mut self, + f: F, + ) -> T { + let res = f(Some(&mut self)); + self.mark_successful(); + res + } +} + +impl<'a> Drop for RecordingGuard<'a> { + fn drop(&mut self) { + if matches!(*self.inner, CommandEncoderStatus::Error(_)) { + // Don't overwrite an error that is already present. + return; + } + self.inner.invalidate(EncoderStateError::Invalid); + } +} + +impl<'a> ops::Deref for RecordingGuard<'a> { + type Target = CommandBufferMutable; + + fn deref(&self) -> &Self::Target { + match &*self.inner { + CommandEncoderStatus::Recording(command_buffer_mutable) => command_buffer_mutable, + _ => unreachable!(), + } + } +} + +impl<'a> ops::DerefMut for RecordingGuard<'a> { + fn deref_mut(&mut self) -> &mut Self::Target { + match self.inner { + CommandEncoderStatus::Recording(command_buffer_mutable) => command_buffer_mutable, + _ => unreachable!(), + } + } +} + +pub(crate) struct CommandEncoder { + pub(crate) device: Arc, + + pub(crate) label: String, + + /// The mutable state of this command encoder. + pub(crate) data: Mutex, +} + +crate::impl_resource_type!(CommandEncoder); +crate::impl_labeled!(CommandEncoder); +crate::impl_parent_device!(CommandEncoder); +crate::impl_storage_item!(CommandEncoder); + +impl Drop for CommandEncoder { + fn drop(&mut self) { + resource_log!("Drop {}", self.error_ident()); + } +} + +/// The encoding API being used with a `CommandEncoder`. +/// +/// Mixing APIs on the same encoder is not allowed. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum EncodingApi { + // The regular wgpu encoding APIs are being used. + Wgpu, + + // The raw hal encoding API is being used. + Raw, + + // Neither encoding API has been called yet. + Undecided, + + // The encoder is used internally by wgpu. + InternalUse, +} + +impl EncodingApi { + pub(crate) fn set(&mut self, api: EncodingApi) { + match *self { + EncodingApi::Undecided => { + *self = api; + } + self_api if self_api != api => { + panic!("Mixing the wgpu encoding API with the raw encoding API is not permitted"); + } + _ => {} + } + } +} + +/// A raw [`CommandEncoder`][rce], and the raw [`CommandBuffer`][rcb]s built from it. +/// +/// Each wgpu-core [`CommandBuffer`] owns an instance of this type, which is +/// where the commands are actually stored. +/// +/// This holds a `Vec` of raw [`CommandBuffer`][rcb]s, not just one. We are not +/// always able to record commands in the order in which they must ultimately be +/// submitted to the queue, but raw command buffers don't permit inserting new +/// commands into the middle of a recorded stream. However, hal queue submission +/// accepts a series of command buffers at once, so we can simply break the +/// stream up into multiple buffers, and then reorder the buffers. See +/// [`InnerCommandEncoder::close_and_swap`] for a specific example of this. +/// +/// [rce]: hal::Api::CommandEncoder +/// [rcb]: hal::Api::CommandBuffer +pub(crate) struct InnerCommandEncoder { + /// The underlying `wgpu_hal` [`CommandEncoder`]. + /// + /// Successfully executed command buffers' encoders are saved in a + /// [`CommandAllocator`] for recycling. + /// + /// [`CommandEncoder`]: hal::Api::CommandEncoder + /// [`CommandAllocator`]: crate::command::CommandAllocator + pub(crate) raw: ManuallyDrop>, + + /// All the raw command buffers for our owning [`CommandBuffer`], in + /// submission order. + /// + /// These command buffers were all constructed with `raw`. The + /// [`wgpu_hal::CommandEncoder`] trait forbids these from outliving `raw`, + /// and requires that we provide all of these when we call + /// [`raw.reset_all()`][CE::ra], so the encoder and its buffers travel + /// together. + /// + /// [CE::ra]: hal::CommandEncoder::reset_all + /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder + pub(crate) list: Vec>, + + pub(crate) device: Arc, + + /// True if `raw` is in the "recording" state. + /// + /// See the documentation for [`wgpu_hal::CommandEncoder`] for + /// details on the states `raw` can be in. + /// + /// [`wgpu_hal::CommandEncoder`]: hal::CommandEncoder + pub(crate) is_open: bool, + + /// Tracks which API is being used to encode commands. + /// + /// Mixing the wgpu encoding API with access to the raw hal encoder via + /// `as_hal_mut` is not supported. this field tracks which API is being used + /// in order to detect and reject invalid usage. + pub(crate) api: EncodingApi, + + pub(crate) label: String, +} + +impl InnerCommandEncoder { + /// Finish the current command buffer and insert it just before + /// the last element in [`self.list`][l]. + /// + /// On return, the underlying hal encoder is closed. + /// + /// What is this for? + /// + /// The `wgpu_hal` contract requires that each render or compute pass's + /// commands be preceded by calls to [`transition_buffers`] and + /// [`transition_textures`], to put the resources the pass operates on in + /// the appropriate state. Unfortunately, we don't know which transitions + /// are needed until we're done recording the pass itself. Rather than + /// iterating over the pass twice, we note the necessary transitions as we + /// record its commands, finish the raw command buffer for the actual pass, + /// record a new raw command buffer for the transitions, and jam that buffer + /// in just before the pass's. This is the function that jams in the + /// transitions' command buffer. + /// + /// # Panics + /// + /// - If the encoder is not open. + /// + /// [l]: InnerCommandEncoder::list + /// [`transition_buffers`]: hal::CommandEncoder::transition_buffers + /// [`transition_textures`]: hal::CommandEncoder::transition_textures + fn close_and_swap(&mut self) -> Result<(), DeviceError> { + assert!(self.is_open); + self.is_open = false; + + let new = + unsafe { self.raw.end_encoding() }.map_err(|e| self.device.handle_hal_error(e))?; + self.list.insert(self.list.len() - 1, new); + + Ok(()) + } + + /// Finish the current command buffer and insert it at the beginning + /// of [`self.list`][l]. + /// + /// On return, the underlying hal encoder is closed. + /// + /// # Panics + /// + /// - If the encoder is not open. + /// + /// [l]: InnerCommandEncoder::list + pub(crate) fn close_and_push_front(&mut self) -> Result<(), DeviceError> { + assert!(self.is_open); + self.is_open = false; + + let new = + unsafe { self.raw.end_encoding() }.map_err(|e| self.device.handle_hal_error(e))?; + self.list.insert(0, new); + + Ok(()) + } + + /// Finish the current command buffer, and push it onto + /// the end of [`self.list`][l]. + /// + /// On return, the underlying hal encoder is closed. + /// + /// # Panics + /// + /// - If the encoder is not open. + /// + /// [l]: InnerCommandEncoder::list + pub(crate) fn close(&mut self) -> Result<(), DeviceError> { + assert!(self.is_open); + self.is_open = false; + + let cmd_buf = + unsafe { self.raw.end_encoding() }.map_err(|e| self.device.handle_hal_error(e))?; + self.list.push(cmd_buf); + + Ok(()) + } + + /// Finish the current command buffer, if any, and add it to the + /// end of [`self.list`][l]. + /// + /// If we have opened this command encoder, finish its current + /// command buffer, and push it onto the end of [`self.list`][l]. + /// If this command buffer is closed, do nothing. + /// + /// On return, the underlying hal encoder is closed. + /// + /// [l]: InnerCommandEncoder::list + fn close_if_open(&mut self) -> Result<(), DeviceError> { + if self.is_open { + self.is_open = false; + let cmd_buf = + unsafe { self.raw.end_encoding() }.map_err(|e| self.device.handle_hal_error(e))?; + self.list.push(cmd_buf); + } + + Ok(()) + } + + /// If the command encoder is not open, begin recording a new command buffer. + /// + /// If the command encoder was already open, does nothing. + /// + /// In both cases, returns a reference to the raw encoder. + fn open_if_closed(&mut self) -> Result<&mut dyn hal::DynCommandEncoder, DeviceError> { + if !self.is_open { + let hal_label = hal_label(Some(self.label.as_str()), self.device.instance_flags); + unsafe { self.raw.begin_encoding(hal_label) } + .map_err(|e| self.device.handle_hal_error(e))?; + self.is_open = true; + } + + Ok(self.raw.as_mut()) + } + + /// Begin recording a new command buffer, if we haven't already. + /// + /// The underlying hal encoder is put in the "recording" state. + pub(crate) fn open(&mut self) -> Result<&mut dyn hal::DynCommandEncoder, DeviceError> { + if !self.is_open { + let hal_label = hal_label(Some(self.label.as_str()), self.device.instance_flags); + unsafe { self.raw.begin_encoding(hal_label) } + .map_err(|e| self.device.handle_hal_error(e))?; + self.is_open = true; + } + + Ok(self.raw.as_mut()) + } + + /// Begin recording a new command buffer for a render or compute pass, with + /// its own label. + /// + /// The underlying hal encoder is put in the "recording" state. + /// + /// # Panics + /// + /// - If the encoder is already open. + pub(crate) fn open_pass( + &mut self, + label: Option<&str>, + ) -> Result<&mut dyn hal::DynCommandEncoder, DeviceError> { + assert!(!self.is_open); + + let hal_label = hal_label(label, self.device.instance_flags); + unsafe { self.raw.begin_encoding(hal_label) } + .map_err(|e| self.device.handle_hal_error(e))?; + self.is_open = true; + + Ok(self.raw.as_mut()) + } +} + +impl Drop for InnerCommandEncoder { + fn drop(&mut self) { + if self.is_open { + unsafe { self.raw.discard_encoding() }; + } + unsafe { + self.raw.reset_all(mem::take(&mut self.list)); + } + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + self.device.command_allocator.release_encoder(raw); + } +} + +/// Look at the documentation for [`CommandBufferMutable`] for an explanation of +/// the fields in this struct. This is the "built" counterpart to that type. +pub(crate) struct BakedCommands { + pub(crate) encoder: InnerCommandEncoder, + pub(crate) trackers: Tracker, + pub(crate) temp_resources: Vec, + pub(crate) indirect_draw_validation_resources: crate::indirect_validation::DrawResources, + buffer_memory_init_actions: Vec, + texture_memory_actions: CommandBufferTextureMemoryActions, +} + +/// The mutable state of a [`CommandBuffer`]. +pub struct CommandBufferMutable { + /// The [`wgpu_hal::Api::CommandBuffer`]s we've built so far, and the encoder + /// they belong to. + /// + /// [`wgpu_hal::Api::CommandBuffer`]: hal::Api::CommandBuffer + pub(crate) encoder: InnerCommandEncoder, + + /// All the resources that the commands recorded so far have referred to. + pub(crate) trackers: Tracker, + + /// The regions of buffers and textures these commands will read and write. + /// + /// This is used to determine which portions of which + /// buffers/textures we actually need to initialize. If we're + /// definitely going to write to something before we read from it, + /// we don't need to clear its contents. + buffer_memory_init_actions: Vec, + texture_memory_actions: CommandBufferTextureMemoryActions, + + as_actions: Vec, + temp_resources: Vec, + + indirect_draw_validation_resources: crate::indirect_validation::DrawResources, + + pub(crate) commands: Vec>, + + /// If tracing, `command_encoder_finish` replaces the `Arc`s in `commands` + /// with integer pointers, and moves them into `trace_commands`. + #[cfg(feature = "trace")] + pub(crate) trace_commands: Option>>, +} + +impl CommandBufferMutable { + pub(crate) fn into_baked_commands(self) -> BakedCommands { + BakedCommands { + encoder: self.encoder, + trackers: self.trackers, + temp_resources: self.temp_resources, + indirect_draw_validation_resources: self.indirect_draw_validation_resources, + buffer_memory_init_actions: self.buffer_memory_init_actions, + texture_memory_actions: self.texture_memory_actions, + } + } +} + +/// A buffer of commands to be submitted to the GPU for execution. +/// +/// Once a command buffer is submitted to the queue, its contents are taken +/// to construct a [`BakedCommands`], whose contents eventually become the +/// property of the submission queue. +pub struct CommandBuffer { + pub(crate) device: Arc, + /// The `label` from the descriptor used to create the resource. + label: String, + + /// The mutable state of this command buffer. + pub(crate) data: Mutex, +} + +impl Drop for CommandBuffer { + fn drop(&mut self) { + resource_log!("Drop {}", self.error_ident()); + } +} + +impl CommandEncoder { + pub(crate) fn new( + encoder: Box, + device: &Arc, + label: &Label, + ) -> Self { + CommandEncoder { + device: device.clone(), + label: label.to_string(), + data: Mutex::new( + rank::COMMAND_BUFFER_DATA, + CommandEncoderStatus::Recording(CommandBufferMutable { + encoder: InnerCommandEncoder { + raw: ManuallyDrop::new(encoder), + list: Vec::new(), + device: device.clone(), + is_open: false, + api: EncodingApi::Undecided, + label: label.to_string(), + }, + trackers: Tracker::new( + device.ordered_buffer_usages, + device.ordered_texture_usages, + ), + buffer_memory_init_actions: Default::default(), + texture_memory_actions: Default::default(), + as_actions: Default::default(), + temp_resources: Default::default(), + indirect_draw_validation_resources: + crate::indirect_validation::DrawResources::new(device.clone()), + commands: Vec::new(), + #[cfg(feature = "trace")] + trace_commands: if device.trace.lock().is_some() { + Some(Vec::new()) + } else { + None + }, + }), + ), + } + } + + pub(crate) fn new_invalid( + device: &Arc, + label: &Label, + err: CommandEncoderError, + ) -> Self { + CommandEncoder { + device: device.clone(), + label: label.to_string(), + data: Mutex::new(rank::COMMAND_BUFFER_DATA, make_error_state(err)), + } + } + + pub(crate) fn insert_barriers_from_tracker( + raw: &mut dyn hal::DynCommandEncoder, + base: &mut Tracker, + head: &Tracker, + snatch_guard: &SnatchGuard, + ) { + profiling::scope!("insert_barriers"); + + base.buffers.set_from_tracker(&head.buffers); + base.textures.set_from_tracker(&head.textures); + + Self::drain_barriers(raw, base, snatch_guard); + } + + pub(crate) fn insert_barriers_from_scope( + raw: &mut dyn hal::DynCommandEncoder, + base: &mut Tracker, + head: &UsageScope, + snatch_guard: &SnatchGuard, + ) { + profiling::scope!("insert_barriers"); + + base.buffers.set_from_usage_scope(&head.buffers); + base.textures.set_from_usage_scope(&head.textures); + + Self::drain_barriers(raw, base, snatch_guard); + } + + pub(crate) fn drain_barriers( + raw: &mut dyn hal::DynCommandEncoder, + base: &mut Tracker, + snatch_guard: &SnatchGuard, + ) { + profiling::scope!("drain_barriers"); + + let buffer_barriers = base + .buffers + .drain_transitions(snatch_guard) + .collect::>(); + let (transitions, textures) = base.textures.drain_transitions(snatch_guard); + let texture_barriers = transitions + .into_iter() + .enumerate() + .map(|(i, p)| p.into_hal(textures[i].unwrap().raw())) + .collect::>(); + + unsafe { + raw.transition_buffers(&buffer_barriers); + raw.transition_textures(&texture_barriers); + } + } + + pub(crate) fn insert_barriers_from_device_tracker( + raw: &mut dyn hal::DynCommandEncoder, + base: &mut DeviceTracker, + head: &Tracker, + snatch_guard: &SnatchGuard, + ) { + profiling::scope!("insert_barriers_from_device_tracker"); + + let buffer_barriers = base + .buffers + .set_from_tracker_and_drain_transitions(&head.buffers, snatch_guard) + .collect::>(); + + let texture_barriers = base + .textures + .set_from_tracker_and_drain_transitions(&head.textures, snatch_guard) + .collect::>(); + + unsafe { + raw.transition_buffers(&buffer_barriers); + raw.transition_textures(&texture_barriers); + } + } + + fn encode_commands( + device: &Arc, + cmd_buf_data: &mut CommandBufferMutable, + ) -> Result<(), CommandEncoderError> { + device.check_is_valid()?; + let snatch_guard = device.snatchable_lock.read(); + let mut debug_scope_depth = 0; + + if cmd_buf_data.encoder.api == EncodingApi::Raw { + // Should have panicked on the first call that switched APIs, + // but lets be sure. + assert!(cmd_buf_data.commands.is_empty()); + } + + let commands = mem::take(&mut cmd_buf_data.commands); + + #[cfg(feature = "trace")] + if device.trace.lock().is_some() { + cmd_buf_data.trace_commands = Some( + commands + .iter() + .map(crate::device::trace::IntoTrace::to_trace) + .collect(), + ); + } + + for command in commands { + if matches!( + command, + ArcCommand::RunRenderPass { .. } | ArcCommand::RunComputePass { .. } + ) { + // Compute passes and render passes can accept either an + // open or closed encoder. This state object holds an + // `InnerCommandEncoder`. See the documentation of + // [`EncodingState`]. + let mut state = EncodingState { + device, + raw_encoder: &mut cmd_buf_data.encoder, + tracker: &mut cmd_buf_data.trackers, + buffer_memory_init_actions: &mut cmd_buf_data.buffer_memory_init_actions, + texture_memory_actions: &mut cmd_buf_data.texture_memory_actions, + as_actions: &mut cmd_buf_data.as_actions, + temp_resources: &mut cmd_buf_data.temp_resources, + indirect_draw_validation_resources: &mut cmd_buf_data + .indirect_draw_validation_resources, + snatch_guard: &snatch_guard, + debug_scope_depth: &mut debug_scope_depth, + }; + + match command { + ArcCommand::RunRenderPass { + pass, + color_attachments, + depth_stencil_attachment, + timestamp_writes, + occlusion_query_set, + multiview_mask, + } => { + api_log!( + "Begin encoding render pass with '{}' label", + pass.label.as_deref().unwrap_or("") + ); + let res = render::encode_render_pass( + &mut state, + pass, + color_attachments, + depth_stencil_attachment, + timestamp_writes, + occlusion_query_set, + multiview_mask, + ); + match res.as_ref() { + Err(err) => { + api_log!("Finished encoding render pass ({err:?})") + } + Ok(_) => { + api_log!("Finished encoding render pass (success)") + } + } + res?; + } + ArcCommand::RunComputePass { + pass, + timestamp_writes, + } => { + api_log!( + "Begin encoding compute pass with '{}' label", + pass.label.as_deref().unwrap_or("") + ); + let res = compute::encode_compute_pass(&mut state, pass, timestamp_writes); + match res.as_ref() { + Err(err) => { + api_log!("Finished encoding compute pass ({err:?})") + } + Ok(_) => { + api_log!("Finished encoding compute pass (success)") + } + } + res?; + } + _ => unreachable!(), + } + } else { + // All the other non-pass encoding routines assume the + // encoder is open, so open it if necessary. This state + // object holds an `&mut dyn hal::DynCommandEncoder`. By + // convention, a bare HAL encoder reference in + // [`EncodingState`] must always be an open encoder. + let raw_encoder = cmd_buf_data.encoder.open_if_closed()?; + let mut state = EncodingState { + device, + raw_encoder, + tracker: &mut cmd_buf_data.trackers, + buffer_memory_init_actions: &mut cmd_buf_data.buffer_memory_init_actions, + texture_memory_actions: &mut cmd_buf_data.texture_memory_actions, + as_actions: &mut cmd_buf_data.as_actions, + temp_resources: &mut cmd_buf_data.temp_resources, + indirect_draw_validation_resources: &mut cmd_buf_data + .indirect_draw_validation_resources, + snatch_guard: &snatch_guard, + debug_scope_depth: &mut debug_scope_depth, + }; + match command { + ArcCommand::CopyBufferToBuffer { + src, + src_offset, + dst, + dst_offset, + size, + } => { + transfer::copy_buffer_to_buffer( + &mut state, &src, src_offset, &dst, dst_offset, size, + )?; + } + ArcCommand::CopyBufferToTexture { src, dst, size } => { + transfer::copy_buffer_to_texture(&mut state, &src, &dst, &size)?; + } + ArcCommand::CopyTextureToBuffer { src, dst, size } => { + transfer::copy_texture_to_buffer(&mut state, &src, &dst, &size)?; + } + ArcCommand::CopyTextureToTexture { src, dst, size } => { + transfer::copy_texture_to_texture(&mut state, &src, &dst, &size)?; + } + ArcCommand::ClearBuffer { dst, offset, size } => { + clear::clear_buffer(&mut state, dst, offset, size)?; + } + ArcCommand::ClearTexture { + dst, + subresource_range, + } => { + clear::clear_texture_cmd(&mut state, dst, &subresource_range)?; + } + ArcCommand::WriteTimestamp { + query_set, + query_index, + } => { + query::write_timestamp(&mut state, query_set, query_index)?; + } + ArcCommand::ResolveQuerySet { + query_set, + start_query, + query_count, + destination, + destination_offset, + } => { + query::resolve_query_set( + &mut state, + query_set, + start_query, + query_count, + destination, + destination_offset, + )?; + } + ArcCommand::PushDebugGroup(label) => { + push_debug_group(&mut state, &label)?; + } + ArcCommand::PopDebugGroup => { + pop_debug_group(&mut state)?; + } + ArcCommand::InsertDebugMarker(label) => { + insert_debug_marker(&mut state, &label)?; + } + ArcCommand::BuildAccelerationStructures { blas, tlas } => { + ray_tracing::build_acceleration_structures(&mut state, blas, tlas)?; + } + ArcCommand::TransitionResources { + buffer_transitions, + texture_transitions, + } => { + transition_resources::transition_resources( + &mut state, + buffer_transitions, + texture_transitions, + )?; + } + ArcCommand::RunComputePass { .. } | ArcCommand::RunRenderPass { .. } => { + unreachable!() + } + } + } + } + + if debug_scope_depth > 0 { + Err(CommandEncoderError::DebugGroupError( + DebugGroupError::MissingPop, + ))?; + } + + // Close the encoder, unless it was closed already by a render or compute pass. + cmd_buf_data.encoder.close_if_open()?; + + // Note: if we want to stop tracking the swapchain texture view, + // this is the place to do it. + + Ok(()) + } + + fn finish( + self: &Arc, + desc: &wgt::CommandBufferDescriptor(name.to_owned(), hal_instance), + surfaces: Registry::new(), + hub: Hub::new(), + } + } + + /// # Safety + /// + /// - The raw instance handle returned must not be manually destroyed. + pub unsafe fn instance_as_hal(&self) -> Option<&A::Instance> { + unsafe { self.instance.as_hal::() } + } + + /// # Safety + /// + /// - The raw handles obtained from the Instance must not be manually destroyed + pub unsafe fn from_instance(instance: Instance) -> Self { + profiling::scope!("Global::new"); + Self { + instance, + surfaces: Registry::new(), + hub: Hub::new(), + } + } + + pub fn generate_report(&self) -> GlobalReport { + GlobalReport { + surfaces: self.surfaces.generate_report(), + hub: self.hub.generate_report(), + } + } +} + +impl fmt::Debug for Global { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Global").finish() + } +} + +impl Drop for Global { + fn drop(&mut self) { + profiling::scope!("Global::drop"); + resource_log!("Global::drop"); + } +} + +#[cfg(send_sync)] +fn _test_send_sync(global: &Global) { + fn test_internal(_: T) {} + test_internal(global) +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/hash_utils.rs b/third_party/wgpu-core-29.0.4-patched/src/hash_utils.rs new file mode 100644 index 00000000..5c5b2538 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/hash_utils.rs @@ -0,0 +1,14 @@ +//! Module for hashing utilities. +//! +//! Named hash_utils to prevent clashing with the core::hash module. + +/// HashMap using a fast, non-cryptographic hash algorithm. +pub type FastHashMap = + hashbrown::HashMap>; +/// HashSet using a fast, non-cryptographic hash algorithm. +pub type FastHashSet = + hashbrown::HashSet>; + +/// IndexMap using a fast, non-cryptographic hash algorithm. +pub type FastIndexMap = + indexmap::IndexMap>; diff --git a/third_party/wgpu-core-29.0.4-patched/src/hub.rs b/third_party/wgpu-core-29.0.4-patched/src/hub.rs new file mode 100644 index 00000000..d21e6565 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/hub.rs @@ -0,0 +1,261 @@ +/*! Allocating resource ids, and tracking the resources they refer to. + +The `wgpu_core` API uses identifiers of type [`Id`] to refer to +resources of type `R`. For example, [`id::DeviceId`] is an alias for +`Id`, and [`id::BufferId`] is an alias for +`Id`. `Id` implements `Copy`, `Hash`, `Eq`, `Ord`, and +of course `Debug`. + +[`id::DeviceId`]: crate::id::DeviceId +[`id::BufferId`]: crate::id::BufferId + +Each `Id` contains not only an index for the resource it denotes but +also a Backend indicating which `wgpu` backend it belongs to. + +`Id`s also incorporate a generation number, for additional validation. + +The resources to which identifiers refer are freed explicitly. +Attempting to use an identifier for a resource that has been freed +elicits an error result. + +Eventually, we would like to remove numeric IDs from wgpu-core. +See . + +## Assigning ids to resources + +The users of `wgpu_core` generally want resource ids to be assigned +in one of two ways: + +- Users like `wgpu` want `wgpu_core` to assign ids to resources itself. + For example, `wgpu` expects to call `Global::device_create_buffer` + and have the return value indicate the newly created buffer's id. + +- Users like Firefox want to allocate ids themselves, and pass + `Global::device_create_buffer` and friends the id to assign the new + resource. + +To accommodate either pattern, `wgpu_core` methods that create +resources all expect an `id_in` argument that the caller can use to +specify the id, and they all return the id used. For example, the +declaration of `Global::device_create_buffer` looks like this: + +```ignore +impl Global { + /* ... */ + pub fn device_create_buffer( + &self, + device_id: id::DeviceId, + desc: &resource::BufferDescriptor, + id_in: Input, + ) -> (id::BufferId, Option) { + /* ... */ + } + /* ... */ +} +``` + +Users that want to assign resource ids themselves pass in the id they +want as the `id_in` argument, whereas users that want `wgpu_core` +itself to choose ids always pass `()`. In either case, the id +ultimately assigned is returned as the first element of the tuple. + +Producing true identifiers from `id_in` values is the job of an +[`crate::identity::IdentityManager`] or ids will be received from outside through `Option` arguments. + +## Id allocation and streaming + +Perhaps surprisingly, allowing users to assign resource ids themselves +enables major performance improvements in some applications. + +The `wgpu_core` API is designed for use by Firefox's [WebGPU] +implementation. For security, web content and GPU use must be kept +segregated in separate processes, with all interaction between them +mediated by an inter-process communication protocol. As web content uses +the WebGPU API, the content process sends messages to the GPU process, +which interacts with the platform's GPU APIs on content's behalf, +occasionally sending results back. + +In a classic Rust API, a resource allocation function takes parameters +describing the resource to create, and if creation succeeds, it returns +the resource id in a `Result::Ok` value. However, this design is a poor +fit for the split-process design described above: content must wait for +the reply to its buffer-creation message (say) before it can know which +id it can use in the next message that uses that buffer. On a common +usage pattern, the classic Rust design imposes the latency of a full +cross-process round trip. + +We can avoid incurring these round-trip latencies simply by letting the +content process assign resource ids itself. With this approach, content +can choose an id for the new buffer, send a message to create the +buffer, and then immediately send the next message operating on that +buffer, since it already knows its id. Allowing content and GPU process +activity to be pipelined greatly improves throughput. + +To help propagate errors correctly in this style of usage, when resource +creation fails, the id supplied for that resource is marked to indicate +as much, allowing subsequent operations using that id to be properly +flagged as errors as well. + +[`process`]: crate::identity::IdentityManager::process +[`Id`]: crate::id::Id +[wrapped in a mutex]: trait.IdentityHandler.html#impl-IdentityHandler%3CI%3E-for-Mutex%3CIdentityManager%3E +[WebGPU]: https://www.w3.org/TR/webgpu/ + +## IDs and tracing + +As of `wgpu` v27, commands are encoded all at once when +`CommandEncoder::finish` is called, not when the encoding methods are +called for each command. This implies storing a representation of the +commands in memory until `finish` is called. `Arc`s are more suitable +for this purpose than numeric ids. Rather than redundantly store both +`Id`s and `Arc`s, tracing has been changed to work with `Arc`s. The +serialized trace identifies resources by the integer value of +`Arc::as_ptr`. These IDs have the type [`crate::id::PointerId`]. The +trace player uses hash maps to go from `PointerId`s to `Arc`s +when replaying a trace. + +*/ + +use alloc::sync::Arc; +use core::fmt::Debug; + +use crate::{ + binding_model::{BindGroup, BindGroupLayout, PipelineLayout}, + command::{CommandBuffer, CommandEncoder, RenderBundle}, + device::{queue::Queue, Device}, + instance::Adapter, + pipeline::{ComputePipeline, PipelineCache, RenderPipeline, ShaderModule}, + registry::{Registry, RegistryReport}, + resource::{ + Blas, Buffer, ExternalTexture, Fallible, QuerySet, Sampler, StagingBuffer, Texture, + TextureView, Tlas, + }, +}; + +#[derive(Debug, PartialEq, Eq)] +pub struct HubReport { + pub adapters: RegistryReport, + pub devices: RegistryReport, + pub queues: RegistryReport, + pub pipeline_layouts: RegistryReport, + pub shader_modules: RegistryReport, + pub bind_group_layouts: RegistryReport, + pub bind_groups: RegistryReport, + pub command_encoders: RegistryReport, + pub command_buffers: RegistryReport, + pub render_bundles: RegistryReport, + pub render_pipelines: RegistryReport, + pub compute_pipelines: RegistryReport, + pub pipeline_caches: RegistryReport, + pub query_sets: RegistryReport, + pub buffers: RegistryReport, + pub textures: RegistryReport, + pub texture_views: RegistryReport, + pub external_textures: RegistryReport, + pub samplers: RegistryReport, +} + +impl HubReport { + pub fn is_empty(&self) -> bool { + self.adapters.is_empty() + } +} + +#[allow(rustdoc::private_intra_doc_links)] +/// All the resources tracked by a [`crate::global::Global`]. +/// +/// ## Locking +/// +/// Each field in `Hub` is a [`Registry`] holding all the values of a +/// particular type of resource, all protected by a single RwLock. +/// So for example, to access any [`Buffer`], you must acquire a read +/// lock on the `Hub`s entire buffers registry. The lock guard +/// gives you access to the `Registry`'s [`Storage`], which you can +/// then index with the buffer's id. (Yes, this design causes +/// contention; see [#2272].) +/// +/// But most `wgpu` operations require access to several different +/// kinds of resource, so you often need to hold locks on several +/// different fields of your [`Hub`] simultaneously. +/// +/// Inside the `Registry` there are `Arc` where `T` is a Resource +/// Lock of `Registry` happens only when accessing to get the specific resource +/// +/// [`Storage`]: crate::storage::Storage +pub struct Hub { + pub(crate) adapters: Registry>, + pub(crate) devices: Registry>, + pub(crate) queues: Registry>, + pub(crate) pipeline_layouts: Registry>, + pub(crate) shader_modules: Registry>, + pub(crate) bind_group_layouts: Registry>, + pub(crate) bind_groups: Registry>, + pub(crate) command_encoders: Registry>, + pub(crate) command_buffers: Registry>, + pub(crate) render_bundles: Registry>, + pub(crate) render_pipelines: Registry>, + pub(crate) compute_pipelines: Registry>, + pub(crate) pipeline_caches: Registry>, + pub(crate) query_sets: Registry>, + pub(crate) buffers: Registry>, + pub(crate) staging_buffers: Registry, + pub(crate) textures: Registry>, + pub(crate) texture_views: Registry>, + pub(crate) external_textures: Registry>, + pub(crate) samplers: Registry>, + pub(crate) blas_s: Registry>, + pub(crate) tlas_s: Registry>, +} + +impl Hub { + pub(crate) fn new() -> Self { + Self { + adapters: Registry::new(), + devices: Registry::new(), + queues: Registry::new(), + pipeline_layouts: Registry::new(), + shader_modules: Registry::new(), + bind_group_layouts: Registry::new(), + bind_groups: Registry::new(), + command_encoders: Registry::new(), + command_buffers: Registry::new(), + render_bundles: Registry::new(), + render_pipelines: Registry::new(), + compute_pipelines: Registry::new(), + pipeline_caches: Registry::new(), + query_sets: Registry::new(), + buffers: Registry::new(), + staging_buffers: Registry::new(), + textures: Registry::new(), + texture_views: Registry::new(), + external_textures: Registry::new(), + samplers: Registry::new(), + blas_s: Registry::new(), + tlas_s: Registry::new(), + } + } + + pub fn generate_report(&self) -> HubReport { + HubReport { + adapters: self.adapters.generate_report(), + devices: self.devices.generate_report(), + queues: self.queues.generate_report(), + pipeline_layouts: self.pipeline_layouts.generate_report(), + shader_modules: self.shader_modules.generate_report(), + bind_group_layouts: self.bind_group_layouts.generate_report(), + bind_groups: self.bind_groups.generate_report(), + command_encoders: self.command_encoders.generate_report(), + command_buffers: self.command_buffers.generate_report(), + render_bundles: self.render_bundles.generate_report(), + render_pipelines: self.render_pipelines.generate_report(), + compute_pipelines: self.compute_pipelines.generate_report(), + pipeline_caches: self.pipeline_caches.generate_report(), + query_sets: self.query_sets.generate_report(), + buffers: self.buffers.generate_report(), + textures: self.textures.generate_report(), + texture_views: self.texture_views.generate_report(), + external_textures: self.external_textures.generate_report(), + samplers: self.samplers.generate_report(), + } + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/id.rs b/third_party/wgpu-core-29.0.4-patched/src/id.rs new file mode 100644 index 00000000..fdfff591 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/id.rs @@ -0,0 +1,355 @@ +use crate::{Epoch, Index}; +use core::{ + cmp::Ordering, + fmt::{self, Debug}, + hash::Hash, + marker::PhantomData, + num::NonZeroU64, +}; +use wgt::WasmNotSendSync; + +const _: () = { + if size_of::() != 4 { + panic!() + } +}; +const _: () = { + if size_of::() != 4 { + panic!() + } +}; +const _: () = { + if size_of::() != 8 { + panic!() + } +}; + +/// The raw underlying representation of an identifier. +#[repr(transparent)] +#[cfg_attr( + any(feature = "serde", feature = "trace"), + derive(serde::Serialize), + serde(into = "SerialId") +)] +#[cfg_attr( + any(feature = "serde", feature = "replay"), + derive(serde::Deserialize), + serde(try_from = "SerialId") +)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RawId(NonZeroU64); + +impl RawId { + /// Zip together an identifier and return its raw underlying representation. + /// + /// # Panics + /// + /// If both ID components are zero. + pub fn zip(index: Index, epoch: Epoch) -> RawId { + let v = (index as u64) | ((epoch as u64) << 32); + Self(NonZeroU64::new(v).expect("IDs may not be zero")) + } + + /// Unzip a raw identifier into its components. + pub fn unzip(self) -> (Index, Epoch) { + (self.0.get() as Index, (self.0.get() >> 32) as Epoch) + } +} + +/// An identifier for a wgpu object. +/// +/// An `Id` value identifies a value stored in a [`Global`]'s [`Hub`]. +/// +/// ## Note on `Id` typing +/// +/// You might assume that an `Id` can only be used to retrieve a resource of +/// type `T`, but that is not quite the case. The id types in `wgpu-core`'s +/// public API ([`TextureId`], for example) can refer to resources belonging to +/// any backend, but the corresponding resource types ([`Texture`], for +/// example) are always parameterized by a specific backend `A`. +/// +/// So the `T` in `Id` is usually a resource type like `Texture`, +/// where [`Noop`] is the `wgpu_hal` dummy back end. These empty types are +/// never actually used, beyond just making sure you access each `Storage` with +/// the right kind of identifier. The members of [`Hub`] pair up each +/// `X` type with the resource type `X`, for some specific backend +/// `A`. +/// +/// [`Global`]: crate::global::Global +/// [`Hub`]: crate::hub::Hub +/// [`Hub`]: crate::hub::Hub +/// [`Texture`]: crate::resource::Texture +/// [`Registry`]: crate::registry::Registry +/// [`Noop`]: hal::api::Noop +#[repr(transparent)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct Id(RawId, PhantomData); + +// This type represents Id in a more readable (and editable) way. +#[cfg(feature = "serde")] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Clone, Debug)] +pub enum SerialId { + // The only variant forces RON to not ignore "Id" + Id(Index, Epoch), +} + +#[cfg(feature = "serde")] +impl From for SerialId { + fn from(id: RawId) -> Self { + let (index, epoch) = id.unzip(); + Self::Id(index, epoch) + } +} + +#[cfg(feature = "serde")] +pub struct ZeroIdError; + +#[cfg(feature = "serde")] +impl fmt::Display for ZeroIdError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "IDs may not be zero") + } +} + +#[cfg(feature = "serde")] +impl TryFrom for RawId { + type Error = ZeroIdError; + fn try_from(id: SerialId) -> Result { + let SerialId::Id(index, epoch) = id; + if index == 0 && epoch == 0 { + Err(ZeroIdError) + } else { + Ok(RawId::zip(index, epoch)) + } + } +} + +/// Identify an object by the pointer returned by `Arc::as_ptr`. +/// +/// This is used for tracing. See [IDs and tracing](crate::hub#ids-and-tracing). +#[allow(dead_code)] +#[cfg(feature = "serde")] +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub enum PointerId { + // The only variant forces RON to not ignore "Id" + PointerId(core::num::NonZeroUsize, #[serde(skip)] PhantomData), +} + +#[cfg(feature = "serde")] +impl Copy for PointerId {} + +#[cfg(feature = "serde")] +impl Clone for PointerId { + fn clone(&self) -> Self { + *self + } +} + +#[cfg(feature = "serde")] +impl PartialEq for PointerId { + fn eq(&self, other: &Self) -> bool { + let PointerId::PointerId(this, _) = self; + let PointerId::PointerId(other, _) = other; + this == other + } +} + +#[cfg(feature = "serde")] +impl Eq for PointerId {} + +#[cfg(feature = "serde")] +impl Hash for PointerId { + fn hash(&self, state: &mut H) { + let PointerId::PointerId(this, _) = self; + this.hash(state); + } +} + +#[cfg(feature = "serde")] +impl From<&alloc::sync::Arc> for PointerId { + fn from(arc: &alloc::sync::Arc) -> Self { + // Since the memory representation of `Arc` is just a pointer to + // `ArcInner`, it would be nice to use that pointer as the trace ID, + // since many `into_trace` implementations would then be no-ops at + // runtime. However, `Arc::as_ptr` returns a pointer to the contained + // data, not to the `ArcInner`. The `ArcInner` stores the reference + // counts before the data, so the machine code for this conversion has + // to add an offset to the pointer. + PointerId::PointerId( + core::num::NonZeroUsize::new(alloc::sync::Arc::as_ptr(arc) as usize).unwrap(), + PhantomData, + ) + } +} + +impl Id +where + T: Marker, +{ + /// # Safety + /// + /// The raw id must be valid for the type. + pub unsafe fn from_raw(raw: RawId) -> Self { + Self(raw, PhantomData) + } + + /// Coerce the identifiers into its raw underlying representation. + pub fn into_raw(self) -> RawId { + self.0 + } + + #[inline] + pub fn zip(index: Index, epoch: Epoch) -> Self { + Id(RawId::zip(index, epoch), PhantomData) + } + + #[inline] + pub fn unzip(self) -> (Index, Epoch) { + self.0.unzip() + } +} + +impl Copy for Id where T: Marker {} + +impl Clone for Id +where + T: Marker, +{ + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl Debug for Id +where + T: Marker, +{ + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let (index, epoch) = self.unzip(); + write!(formatter, "Id({index},{epoch})")?; + Ok(()) + } +} + +impl Hash for Id +where + T: Marker, +{ + #[inline] + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + +impl PartialEq for Id +where + T: Marker, +{ + #[inline] + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl Eq for Id where T: Marker {} + +impl PartialOrd for Id +where + T: Marker, +{ + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Id +where + T: Marker, +{ + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + self.0.cmp(&other.0) + } +} + +/// Marker trait used to determine which types uniquely identify a resource. +/// +/// For example, `Device` will have the same type of identifier as +/// `Device` because `Device` for any `T` defines the same maker type. +pub trait Marker: 'static + WasmNotSendSync {} + +// This allows `()` to be used as a marker type for tests. +// +// We don't want these in production code, since they essentially remove type +// safety, like how identifiers across different types can be compared. +#[cfg(test)] +impl Marker for () {} + +/// Define identifiers for each resource. +macro_rules! ids { + ($( + $(#[$($meta:meta)*])* + pub type $name:ident $marker:ident; + )*) => { + /// Marker types for each resource. + pub mod markers { + $( + #[derive(Debug)] + pub enum $marker {} + impl super::Marker for $marker {} + )* + } + + $( + $(#[$($meta)*])* + pub type $name = Id; + )* + } +} + +ids! { + pub type AdapterId Adapter; + pub type SurfaceId Surface; + pub type DeviceId Device; + pub type QueueId Queue; + pub type BufferId Buffer; + pub type StagingBufferId StagingBuffer; + pub type TextureViewId TextureView; + pub type TextureId Texture; + pub type ExternalTextureId ExternalTexture; + pub type SamplerId Sampler; + pub type BindGroupLayoutId BindGroupLayout; + pub type PipelineLayoutId PipelineLayout; + pub type BindGroupId BindGroup; + pub type ShaderModuleId ShaderModule; + pub type RenderPipelineId RenderPipeline; + pub type ComputePipelineId ComputePipeline; + pub type PipelineCacheId PipelineCache; + pub type CommandEncoderId CommandEncoder; + pub type CommandBufferId CommandBuffer; + pub type RenderPassEncoderId RenderPassEncoder; + pub type ComputePassEncoderId ComputePassEncoder; + pub type RenderBundleEncoderId RenderBundleEncoder; + pub type RenderBundleId RenderBundle; + pub type QuerySetId QuerySet; + pub type BlasId Blas; + pub type TlasId Tlas; +} + +#[test] +fn test_id() { + let indexes = [0, Index::MAX / 2 - 1, Index::MAX / 2 + 1, Index::MAX]; + let epochs = [1, Epoch::MAX / 2 - 1, Epoch::MAX / 2 + 1, Epoch::MAX]; + for &i in &indexes { + for &e in &epochs { + let id = Id::<()>::zip(i, e); + let (index, epoch) = id.unzip(); + assert_eq!(index, i); + assert_eq!(epoch, e); + } + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/identity.rs b/third_party/wgpu-core-29.0.4-patched/src/identity.rs new file mode 100644 index 00000000..ea2f6326 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/identity.rs @@ -0,0 +1,162 @@ +use alloc::vec::Vec; +use core::{fmt::Debug, marker::PhantomData}; + +use crate::{ + id::{Id, Marker}, + lock::{rank, Mutex}, + Epoch, Index, +}; + +#[derive(Copy, Clone, Debug, PartialEq)] +enum IdSource { + External, + Allocated, + None, +} + +/// A simple structure to allocate [`Id`] identifiers. +/// +/// Calling [`alloc`] returns a fresh, never-before-seen id. Calling [`release`] +/// marks an id as dead; it will never be returned again by `alloc`. +/// +/// `IdentityValues` returns `Id`s whose index values are suitable for use as +/// indices into a `Vec` that holds those ids' referents: +/// +/// - Every live id has a distinct index value. Every live id's index +/// selects a distinct element in the vector. +/// +/// - `IdentityValues` prefers low index numbers. If you size your vector to +/// accommodate the indices produced here, the vector's length will reflect +/// the highwater mark of actual occupancy. +/// +/// - `IdentityValues` reuses the index values of freed ids before returning +/// ids with new index values. Freed vector entries get reused. +/// +/// - The non-reuse property is achieved by storing an `epoch` alongside the +/// index in an `Id`. Index values are reused, but only with a different +/// epoch. +/// +/// `IdentityValues` can also be used to track the count of IDs allocated by +/// some external allocator. Combining internal and external allocation is not +/// allowed; calling both `alloc` and `mark_as_used` on the same +/// `IdentityValues` will result in a panic. The external mode is used when +/// [playing back a trace of wgpu operations][player]. +/// +/// [`Id`]: crate::id::Id +/// [`alloc`]: IdentityValues::alloc +/// [`release`]: IdentityValues::release +/// [player]: https://github.com/gfx-rs/wgpu/tree/trunk/player/ +#[derive(Debug)] +pub(super) struct IdentityValues { + free: Vec<(Index, Epoch)>, + next_index: Index, + count: usize, + // Sanity check: The allocation logic works under the assumption that we don't + // do a mix of allocating ids from here and providing ids manually for the same + // storage container. + id_source: IdSource, +} + +impl IdentityValues { + /// Allocate a fresh, never-before-seen ID. + /// + /// # Panics + /// + /// If `mark_as_used` has previously been called on this `IdentityValues`. + pub fn alloc(&mut self) -> Id { + assert!( + self.id_source != IdSource::External, + "Mix of internally allocated and externally provided IDs" + ); + self.id_source = IdSource::Allocated; + + self.count += 1; + match self.free.pop() { + Some((index, epoch)) => Id::zip(index, epoch + 1), + None => { + let index = self.next_index; + self.next_index += 1; + let epoch = 1; + Id::zip(index, epoch) + } + } + } + + /// Increment the count of used IDs. + /// + /// # Panics + /// + /// If `alloc` has previously been called on this `IdentityValues`. + pub fn mark_as_used(&mut self, id: Id) -> Id { + assert!( + self.id_source != IdSource::Allocated, + "Mix of internally allocated and externally provided IDs" + ); + self.id_source = IdSource::External; + + self.count += 1; + id + } + + /// Free `id` and/or decrement the count of used IDs. + /// + /// Freed IDs will never be returned from `alloc` again. + pub fn release(&mut self, id: Id) { + if let IdSource::Allocated = self.id_source { + let (index, epoch) = id.unzip(); + self.free.push((index, epoch)); + } + self.count -= 1; + } + + pub fn count(&self) -> usize { + self.count + } +} + +#[derive(Debug)] +pub struct IdentityManager { + pub(super) values: Mutex, + _phantom: PhantomData, +} + +impl IdentityManager { + pub fn process(&self) -> Id { + self.values.lock().alloc() + } + pub fn mark_as_used(&self, id: Id) -> Id { + self.values.lock().mark_as_used(id) + } + pub fn free(&self, id: Id) { + self.values.lock().release(id) + } +} + +impl IdentityManager { + pub fn new() -> Self { + Self { + values: Mutex::new( + rank::IDENTITY_MANAGER_VALUES, + IdentityValues { + free: Vec::new(), + next_index: 0, + count: 0, + id_source: IdSource::None, + }, + ), + _phantom: PhantomData, + } + } +} + +#[test] +fn test_epoch_end_of_life() { + use crate::id; + let man = IdentityManager::::new(); + let id1 = man.process(); + assert_eq!(id1.unzip(), (0, 1)); + man.free(id1); + let id2 = man.process(); + // confirm that the epoch 1 is no longer re-used + assert_eq!(id2.unzip(), (0, 2)); +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/dispatch.rs b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/dispatch.rs new file mode 100644 index 00000000..06002e3d --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/dispatch.rs @@ -0,0 +1,408 @@ +use super::CreateIndirectValidationPipelineError; +use crate::{ + device::DeviceError, + hal_label, + pipeline::{CreateComputePipelineError, CreateShaderModuleError}, +}; +use alloc::{boxed::Box, format, string::ToString as _}; +use core::num::NonZeroU64; + +/// This machinery requires the following limits: +/// +/// - max_bind_groups: 2, +/// - max_dynamic_storage_buffers_per_pipeline_layout: 1, +/// - max_storage_buffers_per_shader_stage: 2, +/// - max_storage_buffer_binding_size: 3 * min_storage_buffer_offset_alignment, +/// - max_immediate_size: 4, +/// - max_compute_invocations_per_workgroup 1 +/// +/// These are all indirectly satisfied by `DownlevelFlags::INDIRECT_EXECUTION`, which is also +/// required for this module's functionality to work. +#[derive(Debug)] +pub(crate) struct Dispatch { + module: Box, + dst_bind_group_layout: Box, + src_bind_group_layout: Box, + pipeline_layout: Box, + pipeline: Box, + dst_buffer: Box, + dst_bind_group: Box, +} + +pub struct Params<'a> { + pub pipeline_layout: &'a dyn hal::DynPipelineLayout, + pub pipeline: &'a dyn hal::DynComputePipeline, + pub dst_buffer: &'a dyn hal::DynBuffer, + pub dst_bind_group: &'a dyn hal::DynBindGroup, + pub aligned_offset: u64, + pub offset_remainder: u64, +} + +impl Dispatch { + pub(super) fn new( + device: &dyn hal::DynDevice, + instance_flags: wgt::InstanceFlags, + limits: &wgt::Limits, + ) -> Result { + let max_compute_workgroups_per_dimension = limits.max_compute_workgroups_per_dimension; + + let src = format!( + " + @group(0) @binding(0) + var dst: array; + @group(1) @binding(0) + var src: array; + struct OffsetPc {{ + inner: u32, + }} + var offset: OffsetPc; + + @compute @workgroup_size(1) + fn main() {{ + let src = vec3(src[offset.inner], src[offset.inner + 1], src[offset.inner + 2]); + let max_compute_workgroups_per_dimension = {max_compute_workgroups_per_dimension}u; + if ( + src.x > max_compute_workgroups_per_dimension || + src.y > max_compute_workgroups_per_dimension || + src.z > max_compute_workgroups_per_dimension + ) {{ + dst = array(0u, 0u, 0u, 0u, 0u, 0u); + }} else {{ + dst = array(src.x, src.y, src.z, src.x, src.y, src.z); + }} + }} + " + ); + + // SAFETY: The value we are passing to `new_unchecked` is not zero, so this is safe. + const SRC_BUFFER_SIZE: NonZeroU64 = NonZeroU64::new(size_of::() as u64 * 3).unwrap(); + + // SAFETY: The value we are passing to `new_unchecked` is not zero, so this is safe. + const DST_BUFFER_SIZE: NonZeroU64 = NonZeroU64::new(SRC_BUFFER_SIZE.get() * 2).unwrap(); + + #[cfg(feature = "wgsl")] + let module = naga::front::wgsl::parse_str(&src).map_err(|inner| { + CreateShaderModuleError::Parsing(naga::error::ShaderError { + source: src.clone(), + label: None, + inner: Box::new(inner), + }) + })?; + #[cfg(not(feature = "wgsl"))] + #[allow(clippy::diverging_sub_expression)] + let module = panic!("Indirect validation requires the wgsl feature flag to be enabled!"); + + let info = crate::device::create_validator( + wgt::Features::IMMEDIATES, + wgt::DownlevelFlags::empty(), + naga::valid::ValidationFlags::all(), + ) + .validate(&module) + .map_err(|inner| { + CreateShaderModuleError::Validation(naga::error::ShaderError { + source: src, + label: None, + inner: Box::new(inner), + }) + })?; + let hal_shader = hal::ShaderInput::Naga(hal::NagaShader { + module: alloc::borrow::Cow::Owned(module), + info, + debug_source: None, + }); + let hal_desc = hal::ShaderModuleDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect dispatch validation shader module"), + instance_flags, + ), + runtime_checks: wgt::ShaderRuntimeChecks::unchecked(), + }; + let module = + unsafe { device.create_shader_module(&hal_desc, hal_shader) }.map_err(|error| { + match error { + hal::ShaderError::Device(error) => { + CreateShaderModuleError::Device(DeviceError::from_hal(error)) + } + hal::ShaderError::Compilation(ref msg) => { + log::error!("Shader error: {msg}"); + CreateShaderModuleError::Generation + } + } + })?; + + let dst_bind_group_layout_desc = hal::BindGroupLayoutDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect dispatch validation destination bind group layout"), + instance_flags, + ), + flags: hal::BindGroupLayoutFlags::empty(), + entries: &[wgt::BindGroupLayoutEntry { + binding: 0, + visibility: wgt::ShaderStages::COMPUTE, + ty: wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Storage { read_only: false }, + has_dynamic_offset: false, + min_binding_size: Some(DST_BUFFER_SIZE), + }, + count: None, + }], + }; + let dst_bind_group_layout = unsafe { + device + .create_bind_group_layout(&dst_bind_group_layout_desc) + .map_err(DeviceError::from_hal)? + }; + + let src_bind_group_layout_desc = hal::BindGroupLayoutDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect dispatch validation source bind group layout"), + instance_flags, + ), + flags: hal::BindGroupLayoutFlags::empty(), + entries: &[wgt::BindGroupLayoutEntry { + binding: 0, + visibility: wgt::ShaderStages::COMPUTE, + ty: wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: true, + min_binding_size: Some(SRC_BUFFER_SIZE), + }, + count: None, + }], + }; + let src_bind_group_layout = unsafe { + device + .create_bind_group_layout(&src_bind_group_layout_desc) + .map_err(DeviceError::from_hal)? + }; + + let pipeline_layout_desc = hal::PipelineLayoutDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect dispatch validation pipeline layout"), + instance_flags, + ), + flags: hal::PipelineLayoutFlags::empty(), + bind_group_layouts: &[ + Some(dst_bind_group_layout.as_ref()), + Some(src_bind_group_layout.as_ref()), + ], + immediate_size: 4, + }; + let pipeline_layout = unsafe { + device + .create_pipeline_layout(&pipeline_layout_desc) + .map_err(DeviceError::from_hal)? + }; + + let pipeline_desc = hal::ComputePipelineDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect dispatch validation pipeline"), + instance_flags, + ), + layout: pipeline_layout.as_ref(), + stage: hal::ProgrammableStage { + module: module.as_ref(), + entry_point: "main", + constants: &Default::default(), + zero_initialize_workgroup_memory: false, + }, + cache: None, + }; + let pipeline = + unsafe { device.create_compute_pipeline(&pipeline_desc) }.map_err(|err| match err { + hal::PipelineError::Device(error) => { + CreateComputePipelineError::Device(DeviceError::from_hal(error)) + } + hal::PipelineError::Linkage(_stages, msg) => { + CreateComputePipelineError::Internal(msg) + } + hal::PipelineError::EntryPoint(_stage) => CreateComputePipelineError::Internal( + crate::device::ENTRYPOINT_FAILURE_ERROR.to_string(), + ), + hal::PipelineError::PipelineConstants(_, error) => { + CreateComputePipelineError::PipelineConstants(error) + } + })?; + + let dst_buffer_desc = hal::BufferDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect dispatch validation destination buffer"), + instance_flags, + ), + size: DST_BUFFER_SIZE.get(), + usage: wgt::BufferUses::INDIRECT | wgt::BufferUses::STORAGE_READ_WRITE, + memory_flags: hal::MemoryFlags::empty(), + }; + let dst_buffer = + unsafe { device.create_buffer(&dst_buffer_desc) }.map_err(DeviceError::from_hal)?; + + let dst_bind_group_desc = hal::BindGroupDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect dispatch validation destination bind group"), + instance_flags, + ), + layout: dst_bind_group_layout.as_ref(), + entries: &[hal::BindGroupEntry { + binding: 0, + resource_index: 0, + count: 1, + }], + // SAFETY: We just created the buffer with this size. + buffers: &[hal::BufferBinding::new_unchecked( + dst_buffer.as_ref(), + 0, + Some(DST_BUFFER_SIZE), + )], + samplers: &[], + textures: &[], + acceleration_structures: &[], + external_textures: &[], + }; + let dst_bind_group = unsafe { + device + .create_bind_group(&dst_bind_group_desc) + .map_err(DeviceError::from_hal) + }?; + + Ok(Self { + module, + dst_bind_group_layout, + src_bind_group_layout, + pipeline_layout, + pipeline, + dst_buffer, + dst_bind_group, + }) + } + + /// `Ok(None)` will only be returned if `buffer_size` is `0`. + pub(super) fn create_src_bind_group( + &self, + device: &dyn hal::DynDevice, + limits: &wgt::Limits, + buffer_size: u64, + buffer: &dyn hal::DynBuffer, + instance_flags: wgt::InstanceFlags, + ) -> Result>, DeviceError> { + let binding_size = calculate_src_buffer_binding_size(buffer_size, limits); + let Some(binding_size) = NonZeroU64::new(binding_size) else { + return Ok(None); + }; + let hal_desc = hal::BindGroupDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect dispatch validation source bind group"), + instance_flags, + ), + layout: self.src_bind_group_layout.as_ref(), + entries: &[hal::BindGroupEntry { + binding: 0, + resource_index: 0, + count: 1, + }], + // SAFETY: We calculated the binding size to fit within the buffer. + buffers: &[hal::BufferBinding::new_unchecked(buffer, 0, binding_size)], + samplers: &[], + textures: &[], + acceleration_structures: &[], + external_textures: &[], + }; + unsafe { + device + .create_bind_group(&hal_desc) + .map(Some) + .map_err(DeviceError::from_hal) + } + } + + pub fn params<'a>(&'a self, limits: &wgt::Limits, offset: u64, buffer_size: u64) -> Params<'a> { + // The offset we receive is only required to be aligned to 4 bytes. + // + // Binding offsets and dynamic offsets are required to be aligned to + // min_storage_buffer_offset_alignment (256 bytes by default). + // + // So, we work around this limitation by calculating an aligned offset + // and pass the remainder through a immediate data. + // + // We could bind the whole buffer and only have to pass the offset + // through a immediate data but we might run into the + // max_storage_buffer_binding_size limit. + // + // See the inner docs of `calculate_src_buffer_binding_size` to + // see how we get the appropriate `binding_size`. + let alignment = limits.min_storage_buffer_offset_alignment as u64; + let binding_size = calculate_src_buffer_binding_size(buffer_size, limits); + let aligned_offset = offset - offset % alignment; + // This works because `binding_size` is either `buffer_size` or `alignment * 2 + buffer_size % alignment`. + let max_aligned_offset = buffer_size - binding_size; + let aligned_offset = aligned_offset.min(max_aligned_offset); + let offset_remainder = offset - aligned_offset; + + Params { + pipeline_layout: self.pipeline_layout.as_ref(), + pipeline: self.pipeline.as_ref(), + dst_buffer: self.dst_buffer.as_ref(), + dst_bind_group: self.dst_bind_group.as_ref(), + aligned_offset, + offset_remainder, + } + } + + pub(super) fn dispose(self, device: &dyn hal::DynDevice) { + let Dispatch { + module, + dst_bind_group_layout, + src_bind_group_layout, + pipeline_layout, + pipeline, + dst_buffer, + dst_bind_group, + } = self; + + unsafe { + device.destroy_bind_group(dst_bind_group); + device.destroy_buffer(dst_buffer); + device.destroy_compute_pipeline(pipeline); + device.destroy_pipeline_layout(pipeline_layout); + device.destroy_bind_group_layout(src_bind_group_layout); + device.destroy_bind_group_layout(dst_bind_group_layout); + device.destroy_shader_module(module); + } + } +} + +fn calculate_src_buffer_binding_size(buffer_size: u64, limits: &wgt::Limits) -> u64 { + let alignment = limits.min_storage_buffer_offset_alignment as u64; + + // We need to choose a binding size that can address all possible sets of 12 contiguous bytes in the buffer taking + // into account that the dynamic offset needs to be a multiple of `min_storage_buffer_offset_alignment`. + + // Given the know variables: `offset`, `buffer_size`, `alignment` and the rule `offset + 12 <= buffer_size`. + + // Let `chunks = floor(buffer_size / alignment)`. + // Let `chunk` be the interval `[0, chunks]`. + // Let `offset = alignment * chunk + r` where `r` is the interval [0, alignment - 4]. + // Let `binding` be the interval `[offset, offset + 12]`. + // Let `aligned_offset = alignment * chunk`. + // Let `aligned_binding` be the interval `[aligned_offset, aligned_offset + r + 12]`. + // Let `aligned_binding_size = r + 12 = [12, alignment + 8]`. + // Let `min_aligned_binding_size = alignment + 8`. + + // `min_aligned_binding_size` is the minimum binding size required to address all 12 contiguous bytes in the buffer + // but the last aligned_offset + min_aligned_binding_size might overflow the buffer. In order to avoid this we must + // pick a larger `binding_size` that satisfies: `last_aligned_offset + binding_size = buffer_size` and + // `binding_size >= min_aligned_binding_size`. + + // Let `buffer_size = alignment * chunks + sr` where `sr` is the interval [0, alignment - 4]. + // Let `last_aligned_offset = alignment * (chunks - u)` where `u` is the interval [0, chunks]. + // => `binding_size = buffer_size - last_aligned_offset` + // => `binding_size = alignment * chunks + sr - alignment * (chunks - u)` + // => `binding_size = alignment * chunks + sr - alignment * chunks + alignment * u` + // => `binding_size = sr + alignment * u` + // => `min_aligned_binding_size <= sr + alignment * u` + // => `alignment + 8 <= sr + alignment * u` + // => `u` must be at least 2 + // => `binding_size = sr + alignment * 2` + + let binding_size = 2 * alignment + (buffer_size % alignment); + binding_size.min(buffer_size) +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/draw.rs b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/draw.rs new file mode 100644 index 00000000..03501b5a --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/draw.rs @@ -0,0 +1,1032 @@ +use super::{ + utils::{BufferBarrierScratch, BufferBarriers, UniqueIndexExt as _, UniqueIndexScratch}, + CreateIndirectValidationPipelineError, +}; +use crate::{ + command::RenderPassErrorInner, + device::{queue::TempResource, Device, DeviceError}, + hal_label, + lock::{rank, Mutex}, + pipeline::{CreateComputePipelineError, CreateShaderModuleError}, + resource::{RawResourceAccess as _, StagingBuffer, Trackable}, + snatch::SnatchGuard, + track::TrackerIndex, + FastHashMap, +}; +use alloc::{boxed::Box, string::ToString, sync::Arc, vec, vec::Vec}; +use core::{mem::size_of, num::NonZeroU64}; +use wgt::Limits; + +/// Note: This needs to be under: +/// +/// default max_compute_workgroups_per_dimension * size_of::() * `workgroup_size` used by the shader +/// +/// = (2^16 - 1) * 2^4 * 2^6 +/// +/// It is currently set to: +/// +/// = (2^16 - 1) * 2^4 +/// +/// This is enough space for: +/// +/// - 65535 [`wgt::DrawIndirectArgs`] / [`MetadataEntry`] +/// - 52428 [`wgt::DrawIndexedIndirectArgs`] +const BUFFER_SIZE: wgt::BufferSize = wgt::BufferSize::new(1_048_560).unwrap(); + +/// Holds all device-level resources that are needed to validate indirect draws. +/// +/// This machinery requires the following limits: +/// +/// - max_bind_groups: 3, +/// - max_dynamic_storage_buffers_per_pipeline_layout: 1, +/// - max_storage_buffers_per_shader_stage: 3, +/// - max_immediate_size: 8, +/// +/// These are all indirectly satisfied by `DownlevelFlags::INDIRECT_EXECUTION`, which is also +/// required for this module's functionality to work. +#[derive(Debug)] +pub(crate) struct Draw { + module: Box, + metadata_bind_group_layout: Box, + src_bind_group_layout: Box, + dst_bind_group_layout: Box, + pipeline_layout: Box, + pipeline: Box, + + free_indirect_entries: Mutex>, + free_metadata_entries: Mutex>, +} + +impl Draw { + pub(super) fn new( + device: &dyn hal::DynDevice, + required_features: &wgt::Features, + instance_flags: wgt::InstanceFlags, + backend: wgt::Backend, + ) -> Result { + let module = create_validation_module(device, instance_flags)?; + + let metadata_bind_group_layout = create_bind_group_layout( + device, + true, + false, + BUFFER_SIZE, + hal_label( + Some("(wgpu internal) Indirect draw validation metadata bind group layout"), + instance_flags, + ), + )?; + let src_bind_group_layout = create_bind_group_layout( + device, + true, + true, + wgt::BufferSize::new(4 * 4).unwrap(), + hal_label( + Some("(wgpu internal) Indirect draw validation source bind group layout"), + instance_flags, + ), + )?; + let dst_bind_group_layout = create_bind_group_layout( + device, + false, + false, + BUFFER_SIZE, + hal_label( + Some("(wgpu internal) Indirect draw validation destination bind group layout"), + instance_flags, + ), + )?; + + let pipeline_layout_desc = hal::PipelineLayoutDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect draw validation pipeline layout"), + instance_flags, + ), + flags: hal::PipelineLayoutFlags::empty(), + bind_group_layouts: &[ + Some(metadata_bind_group_layout.as_ref()), + Some(src_bind_group_layout.as_ref()), + Some(dst_bind_group_layout.as_ref()), + ], + immediate_size: 8, + }; + let pipeline_layout = unsafe { + device + .create_pipeline_layout(&pipeline_layout_desc) + .map_err(DeviceError::from_hal)? + }; + + let supports_indirect_first_instance = + required_features.contains(wgt::Features::INDIRECT_FIRST_INSTANCE); + let write_d3d12_special_constants = backend == wgt::Backend::Dx12; + let pipeline = create_validation_pipeline( + device, + module.as_ref(), + pipeline_layout.as_ref(), + supports_indirect_first_instance, + write_d3d12_special_constants, + instance_flags, + )?; + + Ok(Self { + module, + metadata_bind_group_layout, + src_bind_group_layout, + dst_bind_group_layout, + pipeline_layout, + pipeline, + + free_indirect_entries: Mutex::new(rank::BUFFER_POOL, Vec::new()), + free_metadata_entries: Mutex::new(rank::BUFFER_POOL, Vec::new()), + }) + } + + /// `Ok(None)` will only be returned if `buffer_size` is `0`. + pub(super) fn create_src_bind_group( + &self, + device: &dyn hal::DynDevice, + limits: &Limits, + buffer_size: u64, + buffer: &dyn hal::DynBuffer, + instance_flags: wgt::InstanceFlags, + ) -> Result>, DeviceError> { + let binding_size = calculate_src_buffer_binding_size(buffer_size, limits); + let Some(binding_size) = NonZeroU64::new(binding_size) else { + return Ok(None); + }; + let hal_desc = hal::BindGroupDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect draw validation source bind group"), + instance_flags, + ), + layout: self.src_bind_group_layout.as_ref(), + entries: &[hal::BindGroupEntry { + binding: 0, + resource_index: 0, + count: 1, + }], + // SAFETY: We calculated the binding size to fit within the buffer. + buffers: &[hal::BufferBinding::new_unchecked(buffer, 0, binding_size)], + samplers: &[], + textures: &[], + acceleration_structures: &[], + external_textures: &[], + }; + unsafe { + device + .create_bind_group(&hal_desc) + .map(Some) + .map_err(DeviceError::from_hal) + } + } + + fn acquire_dst_entry( + &self, + device: &dyn hal::DynDevice, + instance_flags: wgt::InstanceFlags, + ) -> Result { + let mut free_buffers = self.free_indirect_entries.lock(); + match free_buffers.pop() { + Some(buffer) => Ok(buffer), + None => { + let usage = wgt::BufferUses::INDIRECT | wgt::BufferUses::STORAGE_READ_WRITE; + create_buffer_and_bind_group( + device, + usage, + self.dst_bind_group_layout.as_ref(), + hal_label(Some("(wgpu internal) Indirect draw validation destination buffer"), instance_flags), + hal_label(Some("(wgpu internal) Indirect draw validation destination bind group layout"), instance_flags), + ) + } + } + } + + fn release_dst_entries(&self, entries: impl Iterator) { + self.free_indirect_entries.lock().extend(entries); + } + + fn acquire_metadata_entry( + &self, + device: &dyn hal::DynDevice, + instance_flags: wgt::InstanceFlags, + ) -> Result { + let mut free_buffers = self.free_metadata_entries.lock(); + match free_buffers.pop() { + Some(buffer) => Ok(buffer), + None => { + let usage = wgt::BufferUses::COPY_DST | wgt::BufferUses::STORAGE_READ_ONLY; + create_buffer_and_bind_group( + device, + usage, + self.metadata_bind_group_layout.as_ref(), + hal_label( + Some("(wgpu internal) Indirect draw validation metadata buffer"), + instance_flags, + ), + hal_label( + Some("(wgpu internal) Indirect draw validation metadata bind group layout"), + instance_flags, + ), + ) + } + } + } + + fn release_metadata_entries(&self, entries: impl Iterator) { + self.free_metadata_entries.lock().extend(entries); + } + + /// Injects a compute pass that will validate all indirect draws in the current render pass. + pub(crate) fn inject_validation_pass( + &self, + device: &Arc, + snatch_guard: &SnatchGuard, + resources: &mut DrawResources, + temp_resources: &mut Vec, + encoder: &mut dyn hal::DynCommandEncoder, + batcher: DrawBatcher, + ) -> Result<(), RenderPassErrorInner> { + let mut batches = batcher.batches; + + if batches.is_empty() { + return Ok(()); + } + + let max_staging_buffer_size = 1 << 26; // ~67MiB + + let mut staging_buffers = Vec::new(); + + let mut current_size = 0; + for batch in batches.values_mut() { + let data = batch.metadata(); + let offset = if current_size + data.len() > max_staging_buffer_size { + let staging_buffer = + StagingBuffer::new(device, NonZeroU64::new(current_size as u64).unwrap())?; + staging_buffers.push(staging_buffer); + current_size = data.len(); + 0 + } else { + let offset = current_size; + current_size += data.len(); + offset as u64 + }; + batch.staging_buffer_index = staging_buffers.len(); + batch.staging_buffer_offset = offset; + } + if current_size != 0 { + let staging_buffer = + StagingBuffer::new(device, NonZeroU64::new(current_size as u64).unwrap())?; + staging_buffers.push(staging_buffer); + } + + for batch in batches.values() { + let data = batch.metadata(); + let staging_buffer = &mut staging_buffers[batch.staging_buffer_index]; + unsafe { + staging_buffer.write_with_offset( + data, + 0, + batch.staging_buffer_offset as isize, + data.len(), + ) + }; + } + + let staging_buffers: Vec<_> = staging_buffers + .into_iter() + .map(|buffer| buffer.flush()) + .collect(); + + let mut current_metadata_entry = None; + for batch in batches.values_mut() { + let data = batch.metadata(); + let (metadata_resource_index, metadata_buffer_offset) = + resources.get_metadata_subrange(data.len() as u64, &mut current_metadata_entry)?; + batch.metadata_resource_index = metadata_resource_index; + batch.metadata_buffer_offset = metadata_buffer_offset; + } + + let buffer_barrier_scratch = &mut BufferBarrierScratch::new(); + let unique_index_scratch = &mut UniqueIndexScratch::new(); + + BufferBarriers::new(buffer_barrier_scratch) + .extend( + batches + .values() + .map(|batch| batch.staging_buffer_index) + .unique(unique_index_scratch) + .map(|index| hal::BufferBarrier { + buffer: staging_buffers[index].raw(), + usage: hal::StateTransition { + from: wgt::BufferUses::MAP_WRITE, + to: wgt::BufferUses::COPY_SRC, + }, + }), + ) + .extend( + batches + .values() + .map(|batch| batch.metadata_resource_index) + .unique(unique_index_scratch) + .map(|index| hal::BufferBarrier { + buffer: resources.get_metadata_buffer(index), + usage: hal::StateTransition { + from: wgt::BufferUses::STORAGE_READ_ONLY, + to: wgt::BufferUses::COPY_DST, + }, + }), + ) + .encode(encoder); + + for batch in batches.values() { + let data = batch.metadata(); + let data_size = NonZeroU64::new(data.len() as u64).unwrap(); + + let staging_buffer = &staging_buffers[batch.staging_buffer_index]; + + let metadata_buffer = resources.get_metadata_buffer(batch.metadata_resource_index); + + unsafe { + encoder.copy_buffer_to_buffer( + staging_buffer.raw(), + metadata_buffer, + &[hal::BufferCopy { + src_offset: batch.staging_buffer_offset, + dst_offset: batch.metadata_buffer_offset, + size: data_size, + }], + ); + } + } + + for staging_buffer in staging_buffers { + temp_resources.push(TempResource::StagingBuffer(staging_buffer)); + } + + BufferBarriers::new(buffer_barrier_scratch) + .extend( + batches + .values() + .map(|batch| batch.metadata_resource_index) + .unique(unique_index_scratch) + .map(|index| hal::BufferBarrier { + buffer: resources.get_metadata_buffer(index), + usage: hal::StateTransition { + from: wgt::BufferUses::COPY_DST, + to: wgt::BufferUses::STORAGE_READ_ONLY, + }, + }), + ) + .extend( + batches + .values() + .map(|batch| batch.dst_resource_index) + .unique(unique_index_scratch) + .map(|index| hal::BufferBarrier { + buffer: resources.get_dst_buffer(index), + usage: hal::StateTransition { + from: wgt::BufferUses::INDIRECT, + to: wgt::BufferUses::STORAGE_READ_WRITE, + }, + }), + ) + .encode(encoder); + + let desc = hal::ComputePassDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect draw validation pass"), + device.instance_flags, + ), + timestamp_writes: None, + }; + unsafe { + encoder.begin_compute_pass(&desc); + } + unsafe { + encoder.set_compute_pipeline(self.pipeline.as_ref()); + } + + for batch in batches.values() { + let pipeline_layout = self.pipeline_layout.as_ref(); + + let metadata_start = + (batch.metadata_buffer_offset / size_of::() as u64) as u32; + let metadata_count = batch.entries.len() as u32; + unsafe { + encoder.set_immediates(pipeline_layout, 0, &[metadata_start, metadata_count]); + } + + let metadata_bind_group = + resources.get_metadata_bind_group(batch.metadata_resource_index); + unsafe { + encoder.set_bind_group(pipeline_layout, 0, metadata_bind_group, &[]); + } + + // Make sure the indirect buffer is still valid. + batch.src_buffer.try_raw(snatch_guard)?; + + let src_bind_group = batch + .src_buffer + .indirect_validation_bind_groups + .get(snatch_guard) + .unwrap() + .draw + .as_ref(); + unsafe { + encoder.set_bind_group( + pipeline_layout, + 1, + src_bind_group, + &[batch.src_dynamic_offset as u32], + ); + } + + let dst_bind_group = resources.get_dst_bind_group(batch.dst_resource_index); + unsafe { + encoder.set_bind_group(pipeline_layout, 2, dst_bind_group, &[]); + } + + unsafe { + encoder.dispatch([(batch.entries.len() as u32).div_ceil(64), 1, 1]); + } + } + + unsafe { + encoder.end_compute_pass(); + } + + BufferBarriers::new(buffer_barrier_scratch) + .extend( + batches + .values() + .map(|batch| batch.dst_resource_index) + .unique(unique_index_scratch) + .map(|index| hal::BufferBarrier { + buffer: resources.get_dst_buffer(index), + usage: hal::StateTransition { + from: wgt::BufferUses::STORAGE_READ_WRITE, + to: wgt::BufferUses::INDIRECT, + }, + }), + ) + .encode(encoder); + + Ok(()) + } + + pub(super) fn dispose(self, device: &dyn hal::DynDevice) { + let Draw { + module, + metadata_bind_group_layout, + src_bind_group_layout, + dst_bind_group_layout, + pipeline_layout, + pipeline, + + free_indirect_entries, + free_metadata_entries, + } = self; + + for entry in free_indirect_entries.into_inner().drain(..) { + unsafe { + device.destroy_bind_group(entry.bind_group); + device.destroy_buffer(entry.buffer); + } + } + + for entry in free_metadata_entries.into_inner().drain(..) { + unsafe { + device.destroy_bind_group(entry.bind_group); + device.destroy_buffer(entry.buffer); + } + } + + unsafe { + device.destroy_compute_pipeline(pipeline); + device.destroy_pipeline_layout(pipeline_layout); + device.destroy_bind_group_layout(metadata_bind_group_layout); + device.destroy_bind_group_layout(src_bind_group_layout); + device.destroy_bind_group_layout(dst_bind_group_layout); + device.destroy_shader_module(module); + } + } +} + +fn create_validation_module( + device: &dyn hal::DynDevice, + instance_flags: wgt::InstanceFlags, +) -> Result, CreateIndirectValidationPipelineError> { + let src = include_str!("./validate_draw.wgsl"); + + #[cfg(feature = "wgsl")] + let module = naga::front::wgsl::parse_str(src).map_err(|inner| { + CreateShaderModuleError::Parsing(naga::error::ShaderError { + source: src.to_string(), + label: None, + inner: Box::new(inner), + }) + })?; + #[cfg(not(feature = "wgsl"))] + #[allow(clippy::diverging_sub_expression)] + let module = panic!("Indirect validation requires the wgsl feature flag to be enabled!"); + + let info = crate::device::create_validator( + wgt::Features::IMMEDIATES, + wgt::DownlevelFlags::empty(), + naga::valid::ValidationFlags::all(), + ) + .validate(&module) + .map_err(|inner| { + CreateShaderModuleError::Validation(naga::error::ShaderError { + source: src.to_string(), + label: None, + inner: Box::new(inner), + }) + })?; + let hal_shader = hal::ShaderInput::Naga(hal::NagaShader { + module: alloc::borrow::Cow::Owned(module), + info, + debug_source: None, + }); + let hal_desc = hal::ShaderModuleDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect draw validation shader module"), + instance_flags, + ), + runtime_checks: wgt::ShaderRuntimeChecks::unchecked(), + }; + let module = unsafe { device.create_shader_module(&hal_desc, hal_shader) }.map_err( + |error| match error { + hal::ShaderError::Device(error) => { + CreateShaderModuleError::Device(DeviceError::from_hal(error)) + } + hal::ShaderError::Compilation(ref msg) => { + log::error!("Shader error: {msg}"); + CreateShaderModuleError::Generation + } + }, + )?; + + Ok(module) +} + +fn create_validation_pipeline( + device: &dyn hal::DynDevice, + module: &dyn hal::DynShaderModule, + pipeline_layout: &dyn hal::DynPipelineLayout, + supports_indirect_first_instance: bool, + write_d3d12_special_constants: bool, + instance_flags: wgt::InstanceFlags, +) -> Result, CreateIndirectValidationPipelineError> { + let pipeline_desc = hal::ComputePipelineDescriptor { + label: hal_label( + Some("(wgpu internal) Indirect draw validation pipeline"), + instance_flags, + ), + layout: pipeline_layout, + stage: hal::ProgrammableStage { + module, + entry_point: "main", + constants: &hashbrown::HashMap::from([ + ( + "supports_indirect_first_instance".to_string(), + f64::from(supports_indirect_first_instance), + ), + ( + "write_d3d12_special_constants".to_string(), + f64::from(write_d3d12_special_constants), + ), + ]), + zero_initialize_workgroup_memory: false, + }, + cache: None, + }; + let pipeline = + unsafe { device.create_compute_pipeline(&pipeline_desc) }.map_err(|err| match err { + hal::PipelineError::Device(error) => { + CreateComputePipelineError::Device(DeviceError::from_hal(error)) + } + hal::PipelineError::Linkage(_stages, msg) => CreateComputePipelineError::Internal(msg), + hal::PipelineError::EntryPoint(_stage) => CreateComputePipelineError::Internal( + crate::device::ENTRYPOINT_FAILURE_ERROR.to_string(), + ), + hal::PipelineError::PipelineConstants(_, error) => { + CreateComputePipelineError::PipelineConstants(error) + } + })?; + + Ok(pipeline) +} + +fn create_bind_group_layout( + device: &dyn hal::DynDevice, + read_only: bool, + has_dynamic_offset: bool, + min_binding_size: wgt::BufferSize, + label: Option<&'static str>, +) -> Result, CreateIndirectValidationPipelineError> { + let bind_group_layout_desc = hal::BindGroupLayoutDescriptor { + label, + flags: hal::BindGroupLayoutFlags::empty(), + entries: &[wgt::BindGroupLayoutEntry { + binding: 0, + visibility: wgt::ShaderStages::COMPUTE, + ty: wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Storage { read_only }, + has_dynamic_offset, + min_binding_size: Some(min_binding_size), + }, + count: None, + }], + }; + let bind_group_layout = unsafe { + device + .create_bind_group_layout(&bind_group_layout_desc) + .map_err(DeviceError::from_hal)? + }; + + Ok(bind_group_layout) +} + +/// Returns the largest binding size that when combined with dynamic offsets can address the whole buffer. +fn calculate_src_buffer_binding_size(buffer_size: u64, limits: &Limits) -> u64 { + let max_storage_buffer_binding_size = limits.max_storage_buffer_binding_size; + let min_storage_buffer_offset_alignment = limits.min_storage_buffer_offset_alignment as u64; + + if buffer_size <= max_storage_buffer_binding_size { + buffer_size + } else { + let buffer_rem = buffer_size % min_storage_buffer_offset_alignment; + let binding_rem = max_storage_buffer_binding_size % min_storage_buffer_offset_alignment; + + // Can the buffer remainder fit in the binding remainder? + // If so, align max binding size and add buffer remainder + if buffer_rem <= binding_rem { + max_storage_buffer_binding_size - binding_rem + buffer_rem + } + // If not, align max binding size, shorten it by a chunk and add buffer remainder + else { + max_storage_buffer_binding_size - binding_rem - min_storage_buffer_offset_alignment + + buffer_rem + } + } +} + +/// Splits the given `offset` into a dynamic offset & offset. +fn calculate_src_offsets(buffer_size: u64, limits: &Limits, offset: u64) -> (u64, u64) { + let binding_size = calculate_src_buffer_binding_size(buffer_size, limits); + + let min_storage_buffer_offset_alignment = limits.min_storage_buffer_offset_alignment as u64; + + let chunk_adjustment = match min_storage_buffer_offset_alignment { + // No need to adjust since the src_offset is 4 byte aligned. + 4 => 0, + // With 16/20 bytes of data we can straddle up to 2 8 byte boundaries: + // - 16 bytes of data: (4|8|4) + // - 20 bytes of data: (4|8|8, 8|8|4) + 8 => 2, + // With 16/20 bytes of data we can straddle up to 1 16+ byte boundary: + // - 16 bytes of data: (4|12, 8|8, 12|4) + // - 20 bytes of data: (4|16, 8|12, 12|8, 16|4) + 16.. => 1, + _ => unreachable!(), + }; + + let chunks = binding_size / min_storage_buffer_offset_alignment; + let dynamic_offset_stride = + chunks.saturating_sub(chunk_adjustment) * min_storage_buffer_offset_alignment; + + if dynamic_offset_stride == 0 { + return (0, offset); + } + + let max_dynamic_offset = buffer_size - binding_size; + let max_dynamic_offset_index = max_dynamic_offset / dynamic_offset_stride; + + let src_dynamic_offset_index = offset / dynamic_offset_stride; + + let src_dynamic_offset = + src_dynamic_offset_index.min(max_dynamic_offset_index) * dynamic_offset_stride; + let src_offset = offset - src_dynamic_offset; + + (src_dynamic_offset, src_offset) +} + +#[derive(Debug)] +struct BufferPoolEntry { + buffer: Box, + bind_group: Box, +} + +fn create_buffer_and_bind_group( + device: &dyn hal::DynDevice, + usage: wgt::BufferUses, + bind_group_layout: &dyn hal::DynBindGroupLayout, + buffer_label: Option<&'static str>, + bind_group_label: Option<&'static str>, +) -> Result { + let buffer_desc = hal::BufferDescriptor { + label: buffer_label, + size: BUFFER_SIZE.get(), + usage, + memory_flags: hal::MemoryFlags::empty(), + }; + let buffer = unsafe { device.create_buffer(&buffer_desc) }?; + let bind_group_desc = hal::BindGroupDescriptor { + label: bind_group_label, + layout: bind_group_layout, + entries: &[hal::BindGroupEntry { + binding: 0, + resource_index: 0, + count: 1, + }], + // SAFETY: We just created the buffer with this size. + buffers: &[hal::BufferBinding::new_unchecked( + buffer.as_ref(), + 0, + BUFFER_SIZE, + )], + samplers: &[], + textures: &[], + acceleration_structures: &[], + external_textures: &[], + }; + let bind_group = unsafe { device.create_bind_group(&bind_group_desc) }?; + Ok(BufferPoolEntry { buffer, bind_group }) +} + +#[derive(Clone)] +struct CurrentEntry { + index: usize, + offset: u64, +} + +/// Holds all command buffer-level resources that are needed to validate indirect draws. +pub(crate) struct DrawResources { + device: Arc, + dst_entries: Vec, + metadata_entries: Vec, +} + +impl Drop for DrawResources { + fn drop(&mut self) { + if let Some(ref indirect_validation) = self.device.indirect_validation { + let indirect_draw_validation = &indirect_validation.draw; + indirect_draw_validation.release_dst_entries(self.dst_entries.drain(..)); + indirect_draw_validation.release_metadata_entries(self.metadata_entries.drain(..)); + } + } +} + +impl DrawResources { + pub(crate) fn new(device: Arc) -> Self { + DrawResources { + device, + dst_entries: Vec::new(), + metadata_entries: Vec::new(), + } + } + + pub(crate) fn get_dst_buffer(&self, index: usize) -> &dyn hal::DynBuffer { + self.dst_entries.get(index).unwrap().buffer.as_ref() + } + + fn get_dst_bind_group(&self, index: usize) -> &dyn hal::DynBindGroup { + self.dst_entries.get(index).unwrap().bind_group.as_ref() + } + + fn get_metadata_buffer(&self, index: usize) -> &dyn hal::DynBuffer { + self.metadata_entries.get(index).unwrap().buffer.as_ref() + } + + fn get_metadata_bind_group(&self, index: usize) -> &dyn hal::DynBindGroup { + self.metadata_entries + .get(index) + .unwrap() + .bind_group + .as_ref() + } + + fn get_dst_subrange( + &mut self, + size: u64, + current_entry: &mut Option, + ) -> Result<(usize, u64), DeviceError> { + let indirect_draw_validation = &self.device.indirect_validation.as_ref().unwrap().draw; + let ensure_entry = |index: usize| { + if self.dst_entries.len() <= index { + let entry = indirect_draw_validation + .acquire_dst_entry(self.device.raw(), self.device.instance_flags)?; + self.dst_entries.push(entry); + } + Ok(()) + }; + let entry_data = Self::get_subrange_impl(ensure_entry, current_entry, size)?; + Ok((entry_data.index, entry_data.offset)) + } + + fn get_metadata_subrange( + &mut self, + size: u64, + current_entry: &mut Option, + ) -> Result<(usize, u64), DeviceError> { + let indirect_draw_validation = &self.device.indirect_validation.as_ref().unwrap().draw; + let ensure_entry = |index: usize| { + if self.metadata_entries.len() <= index { + let entry = indirect_draw_validation + .acquire_metadata_entry(self.device.raw(), self.device.instance_flags)?; + self.metadata_entries.push(entry); + } + Ok(()) + }; + let entry_data = Self::get_subrange_impl(ensure_entry, current_entry, size)?; + Ok((entry_data.index, entry_data.offset)) + } + + fn get_subrange_impl( + ensure_entry: impl FnOnce(usize) -> Result<(), hal::DeviceError>, + current_entry: &mut Option, + size: u64, + ) -> Result { + let index = if let Some(current_entry) = current_entry.as_mut() { + if current_entry.offset + size <= BUFFER_SIZE.get() { + let entry_data = current_entry.clone(); + current_entry.offset += size; + return Ok(entry_data); + } else { + current_entry.index + 1 + } + } else { + 0 + }; + + ensure_entry(index).map_err(DeviceError::from_hal)?; + + let entry_data = CurrentEntry { index, offset: 0 }; + + *current_entry = Some(CurrentEntry { + index, + offset: size, + }); + + Ok(entry_data) + } +} + +/// This must match the `MetadataEntry` struct used by the shader. +#[repr(C)] +struct MetadataEntry { + src_offset: u32, + dst_offset: u32, + vertex_or_index_limit: u32, + instance_limit: u32, +} + +impl MetadataEntry { + fn new( + indexed: bool, + src_offset: u64, + dst_offset: u64, + vertex_or_index_limit: u64, + instance_limit: u64, + ) -> Self { + const U32_MAX_AS_U64: u64 = u32::MAX as u64; + + // NOTE: buffer sizes should never exceed `u32::MAX`. + assert!(src_offset <= U32_MAX_AS_U64); + assert!(dst_offset <= U32_MAX_AS_U64); + + let src_offset = src_offset as u32; + let src_offset = src_offset / 4; // translate byte offset to offset in u32's + + // `src_offset` needs at most 30 bits, + // pack `indexed` in bit 31 of `src_offset` + let src_offset = src_offset | ((indexed as u32) << 31); + + // max value for limits since first_X and X_count indirect draw arguments are u32 + let max_limit = U32_MAX_AS_U64 + U32_MAX_AS_U64; // 1 11111111 11111111 11111111 11111110 + + let vertex_or_index_limit = vertex_or_index_limit.min(max_limit); + let vertex_or_index_limit_bit_32 = (vertex_or_index_limit >> 32) as u32; // extract bit 32 + let vertex_or_index_limit = vertex_or_index_limit as u32; // truncate the limit to a u32 + + let instance_limit = instance_limit.min(max_limit); + let instance_limit_bit_32 = (instance_limit >> 32) as u32; // extract bit 32 + let instance_limit = instance_limit as u32; // truncate the limit to a u32 + + let dst_offset = dst_offset as u32; + let dst_offset = dst_offset / 4; // translate byte offset to offset in u32's + + // `dst_offset` needs at most 30 bits, + // pack `vertex_or_index_limit_bit_32` in bit 30 of `dst_offset` and + // pack `instance_limit_bit_32` in bit 31 of `dst_offset` + let dst_offset = + dst_offset | (vertex_or_index_limit_bit_32 << 30) | (instance_limit_bit_32 << 31); + + Self { + src_offset, + dst_offset, + vertex_or_index_limit, + instance_limit, + } + } +} + +struct DrawIndirectValidationBatch { + src_buffer: Arc, + src_dynamic_offset: u64, + dst_resource_index: usize, + entries: Vec, + + staging_buffer_index: usize, + staging_buffer_offset: u64, + metadata_resource_index: usize, + metadata_buffer_offset: u64, +} + +impl DrawIndirectValidationBatch { + /// Data to be written to the metadata buffer. + fn metadata(&self) -> &[u8] { + unsafe { + core::slice::from_raw_parts( + self.entries.as_ptr().cast::(), + self.entries.len() * size_of::(), + ) + } + } +} + +/// Accumulates all needed data needed to validate indirect draws. +pub(crate) struct DrawBatcher { + batches: FastHashMap<(TrackerIndex, u64, usize), DrawIndirectValidationBatch>, + current_dst_entry: Option, +} + +impl DrawBatcher { + pub(crate) fn new() -> Self { + Self { + batches: FastHashMap::default(), + current_dst_entry: None, + } + } + + /// Add an indirect draw to be validated. + /// + /// Returns the index of the indirect buffer in `indirect_draw_validation_resources` + /// and the offset to be used for the draw. + pub(crate) fn add<'a>( + &mut self, + indirect_draw_validation_resources: &'a mut DrawResources, + device: &Device, + src_buffer: &Arc, + offset: u64, + family: crate::command::DrawCommandFamily, + vertex_or_index_limit: u64, + instance_limit: u64, + ) -> Result<(usize, u64), DeviceError> { + let stride = crate::command::get_dst_stride_of_indirect_args(device.backend(), family); + + let (dst_resource_index, dst_offset) = indirect_draw_validation_resources + .get_dst_subrange(stride, &mut self.current_dst_entry)?; + + let buffer_size = src_buffer.size; + let limits = device.adapter.limits(); + let (src_dynamic_offset, src_offset) = calculate_src_offsets(buffer_size, &limits, offset); + + let src_buffer_tracker_index = src_buffer.tracker_index(); + + let entry = MetadataEntry::new( + family == crate::command::DrawCommandFamily::DrawIndexed, + src_offset, + dst_offset, + vertex_or_index_limit, + instance_limit, + ); + + match self.batches.entry(( + src_buffer_tracker_index, + src_dynamic_offset, + dst_resource_index, + )) { + hashbrown::hash_map::Entry::Occupied(mut occupied_entry) => { + occupied_entry.get_mut().entries.push(entry) + } + hashbrown::hash_map::Entry::Vacant(vacant_entry) => { + vacant_entry.insert(DrawIndirectValidationBatch { + src_buffer: src_buffer.clone(), + src_dynamic_offset, + dst_resource_index, + entries: vec![entry], + + // these will be initialized once we accumulated all entries for the batch + staging_buffer_index: 0, + staging_buffer_offset: 0, + metadata_resource_index: 0, + metadata_buffer_offset: 0, + }); + } + } + + Ok((dst_resource_index, dst_offset)) + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/mod.rs b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/mod.rs new file mode 100644 index 00000000..00b3bbc2 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/mod.rs @@ -0,0 +1,109 @@ +use crate::{ + device::DeviceError, + pipeline::{CreateComputePipelineError, CreateShaderModuleError}, +}; +use alloc::boxed::Box; +use thiserror::Error; + +mod dispatch; +mod draw; +mod utils; + +pub(crate) use dispatch::Dispatch; +pub(crate) use draw::{Draw, DrawBatcher, DrawResources}; + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +enum CreateIndirectValidationPipelineError { + #[error(transparent)] + DeviceError(#[from] DeviceError), + #[error(transparent)] + ShaderModule(#[from] CreateShaderModuleError), + #[error(transparent)] + ComputePipeline(#[from] CreateComputePipelineError), +} + +pub(crate) struct IndirectValidation { + pub(crate) dispatch: Dispatch, + pub(crate) draw: Draw, +} + +impl IndirectValidation { + pub(crate) fn new( + device: &dyn hal::DynDevice, + required_limits: &wgt::Limits, + required_features: &wgt::Features, + instance_flags: wgt::InstanceFlags, + backend: wgt::Backend, + ) -> Result { + let dispatch = match Dispatch::new(device, instance_flags, required_limits) { + Ok(dispatch) => dispatch, + Err(e) => { + log::error!("indirect-validation error: {e:?}"); + return Err(DeviceError::Lost); + } + }; + let draw = match Draw::new(device, required_features, instance_flags, backend) { + Ok(draw) => draw, + Err(e) => { + log::error!("indirect-draw-validation error: {e:?}"); + return Err(DeviceError::Lost); + } + }; + Ok(Self { dispatch, draw }) + } + + pub(crate) fn dispose(self, device: &dyn hal::DynDevice) { + let Self { dispatch, draw } = self; + + dispatch.dispose(device); + draw.dispose(device); + } +} + +#[derive(Debug)] +pub(crate) struct BindGroups { + pub(crate) dispatch: Box, + draw: Box, +} + +impl BindGroups { + /// `Ok(None)` will only be returned if `buffer_size` is `0`. + pub(crate) fn new( + indirect_validation: &IndirectValidation, + device: &crate::device::Device, + buffer_size: u64, + buffer: &dyn hal::DynBuffer, + ) -> Result, DeviceError> { + let dispatch = indirect_validation.dispatch.create_src_bind_group( + device.raw(), + &device.limits, + buffer_size, + buffer, + device.instance_flags, + )?; + let draw = indirect_validation.draw.create_src_bind_group( + device.raw(), + &device.adapter.limits(), + buffer_size, + buffer, + device.instance_flags, + )?; + + match (dispatch, draw) { + (None, None) => Ok(None), + (None, Some(_)) => unreachable!(), + (Some(_), None) => unreachable!(), + (Some(dispatch), Some(draw)) => Ok(Some(Self { dispatch, draw })), + } + } + + pub(crate) fn dispose(self, device: &dyn hal::DynDevice) { + let Self { dispatch, draw } = self; + + unsafe { + device.destroy_bind_group(dispatch); + device.destroy_bind_group(draw); + } + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/utils.rs b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/utils.rs new file mode 100644 index 00000000..5700fe5c --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/utils.rs @@ -0,0 +1,84 @@ +use alloc::vec::Vec; + +pub(crate) struct UniqueIndexScratch(bit_set::BitSet); + +impl UniqueIndexScratch { + pub(crate) fn new() -> Self { + Self(bit_set::BitSet::new()) + } +} + +pub(crate) struct UniqueIndex<'a, I: Iterator> { + inner: I, + scratch: &'a mut UniqueIndexScratch, +} + +impl<'a, I: Iterator> UniqueIndex<'a, I> { + fn new(inner: I, scratch: &'a mut UniqueIndexScratch) -> Self { + scratch.0.make_empty(); + Self { inner, scratch } + } +} + +impl<'a, I: Iterator> Iterator for UniqueIndex<'a, I> { + type Item = usize; + + fn next(&mut self) -> Option { + self.inner.find(|&i| self.scratch.0.insert(i)) + } +} + +pub(crate) trait UniqueIndexExt: Iterator { + fn unique<'a>(self, scratch: &'a mut UniqueIndexScratch) -> UniqueIndex<'a, Self> + where + Self: Sized, + { + UniqueIndex::new(self, scratch) + } +} + +impl> UniqueIndexExt for T {} + +type BufferBarrier<'b> = hal::BufferBarrier<'b, dyn hal::DynBuffer>; + +pub(crate) struct BufferBarrierScratch<'b>(Vec>); + +impl<'b> BufferBarrierScratch<'b> { + pub(crate) fn new() -> Self { + Self(Vec::new()) + } +} + +pub(crate) struct BufferBarriers<'a, 'b> { + scratch: &'a mut BufferBarrierScratch<'b>, +} + +impl<'a, 'b> BufferBarriers<'a, 'b> { + pub(crate) fn new(scratch: &'a mut BufferBarrierScratch<'_>) -> Self { + // change lifetime of buffer reference, this is safe since `scratch` is empty, + // it was either just created or it has been cleared on `BufferBarriers::drop` + let scratch = unsafe { + core::mem::transmute::<&'a mut BufferBarrierScratch<'_>, &'a mut BufferBarrierScratch<'b>>( + scratch, + ) + }; + Self { scratch } + } + + pub(crate) fn extend(self, iter: impl Iterator>) -> Self { + self.scratch.0.extend(iter); + self + } + + pub(crate) fn encode(self, encoder: &mut dyn hal::DynCommandEncoder) { + unsafe { + encoder.transition_buffers(&self.scratch.0); + } + } +} + +impl<'a, 'b> Drop for BufferBarriers<'a, 'b> { + fn drop(&mut self) { + self.scratch.0.clear(); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/validate_draw.wgsl b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/validate_draw.wgsl new file mode 100644 index 00000000..39c41851 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/indirect_validation/validate_draw.wgsl @@ -0,0 +1,98 @@ +override supports_indirect_first_instance: bool; +override write_d3d12_special_constants: bool; + +struct MetadataEntry { + // bits 0..30 are an offset into `src` + // bit 31 signifies that we are validating an indexed draw + src_offset: u32, + // bits 0..30 are an offset into `dst` + // bit 30 is the most significant bit of `vertex_or_index_limit` + // bit 31 is the most significant bit of `instance_limit` + dst_offset: u32, + vertex_or_index_limit: u32, + instance_limit: u32, +} + +struct MetadataRange { + start: u32, + count: u32, +} +var metadata_range: MetadataRange; + +@group(0) @binding(0) +var metadata: array; +@group(1) @binding(0) +var src: array; +@group(2) @binding(0) +var dst: array; + +fn is_bit_set(data: u32, index: u32) -> bool { + return ((data >> index) & 1u) == 1u; +} + +@compute @workgroup_size(64) +fn main(@builtin(global_invocation_id) global_invocation_id: vec3u) { + if global_invocation_id.x >= metadata_range.count { return; } + + let metadata = metadata[metadata_range.start + global_invocation_id.x]; + var failed = false; + + let is_indexed = is_bit_set(metadata.src_offset, 31); + let src_base_offset = ((metadata.src_offset << 2) >> 2); + let dst_base_offset = ((metadata.dst_offset << 2) >> 2); + + let first_vertex_or_index = src[src_base_offset + 2]; + let vertex_or_index_count = src[src_base_offset + 0]; + + { + let can_overflow = is_bit_set(metadata.dst_offset, 30); + let sub_overflows = metadata.vertex_or_index_limit < first_vertex_or_index; + failed |= sub_overflows && !can_overflow; + let vertex_or_index_limit = metadata.vertex_or_index_limit - first_vertex_or_index; + failed |= vertex_or_index_limit < vertex_or_index_count; + } + + let first_instance = src[src_base_offset + 3 + u32(is_indexed)]; + let instance_count = src[src_base_offset + 1]; + + { + let can_overflow = is_bit_set(metadata.dst_offset, 31); + let sub_overflows = metadata.instance_limit < first_instance; + failed |= sub_overflows && !can_overflow; + let instance_limit = metadata.instance_limit - first_instance; + failed |= instance_limit < instance_count; + } + + if !supports_indirect_first_instance { + failed |= first_instance != 0u; + } + + let dst_offset = select(0u, 3u, write_d3d12_special_constants); + if failed { + if write_d3d12_special_constants { + dst[dst_base_offset + 0] = 0u; + dst[dst_base_offset + 1] = 0u; + dst[dst_base_offset + 2] = 0u; + } + dst[dst_base_offset + dst_offset + 0] = 0u; + dst[dst_base_offset + dst_offset + 1] = 0u; + dst[dst_base_offset + dst_offset + 2] = 0u; + dst[dst_base_offset + dst_offset + 3] = 0u; + if (is_indexed) { + dst[dst_base_offset + dst_offset + 4] = 0u; + } + } else { + if write_d3d12_special_constants { + dst[dst_base_offset + 0] = src[src_base_offset + 2 + u32(is_indexed)]; + dst[dst_base_offset + 1] = src[src_base_offset + 3 + u32(is_indexed)]; + dst[dst_base_offset + 2] = 0u; + } + dst[dst_base_offset + dst_offset + 0] = src[src_base_offset + 0]; + dst[dst_base_offset + dst_offset + 1] = src[src_base_offset + 1]; + dst[dst_base_offset + dst_offset + 2] = src[src_base_offset + 2]; + dst[dst_base_offset + dst_offset + 3] = src[src_base_offset + 3]; + if (is_indexed) { + dst[dst_base_offset + dst_offset + 4] = src[src_base_offset + 4]; + } + } +} \ No newline at end of file diff --git a/third_party/wgpu-core-29.0.4-patched/src/init_tracker/buffer.rs b/third_party/wgpu-core-29.0.4-patched/src/init_tracker/buffer.rs new file mode 100644 index 00000000..4d2cc42f --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/init_tracker/buffer.rs @@ -0,0 +1,40 @@ +use super::{InitTracker, MemoryInitKind}; +use crate::resource::Buffer; +use alloc::sync::Arc; +use core::ops::Range; + +#[derive(Debug, Clone)] +pub(crate) struct BufferInitTrackerAction { + pub buffer: Arc, + pub range: Range, + pub kind: MemoryInitKind, +} + +pub(crate) type BufferInitTracker = InitTracker; + +impl BufferInitTracker { + /// Checks if an action has/requires any effect on the initialization status + /// and shrinks its range if possible. + pub(crate) fn check_action( + &self, + action: &BufferInitTrackerAction, + ) -> Option { + self.create_action(&action.buffer, action.range.clone(), action.kind) + } + + /// Creates an action if it would have any effect on the initialization + /// status and shrinks the range if possible. + pub(crate) fn create_action( + &self, + buffer: &Arc, + query_range: Range, + kind: MemoryInitKind, + ) -> Option { + self.check(query_range) + .map(|range| BufferInitTrackerAction { + buffer: buffer.clone(), + range, + kind, + }) + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/init_tracker/mod.rs b/third_party/wgpu-core-29.0.4-patched/src/init_tracker/mod.rs new file mode 100644 index 00000000..879dd5b3 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/init_tracker/mod.rs @@ -0,0 +1,426 @@ +/*! Lazy initialization of texture and buffer memory. + +The WebGPU specification requires all texture & buffer memory to be +zero initialized on first read. To avoid unnecessary inits, we track +the initialization status of every resource and perform inits lazily. + +The granularity is different for buffers and textures: + +- Buffer: Byte granularity to support usecases with large, partially + bound buffers well. + +- Texture: Mip-level per layer. That is, a 2D surface is either + completely initialized or not, subrects are not tracked. + +Every use of a buffer/texture generates a InitTrackerAction which are +recorded and later resolved at queue submit by merging them with the +current state and each other in execution order. + +It is important to note that from the point of view of the memory init +system there are two kind of writes: + +- **Full writes**: Any kind of memcpy operation. These cause a + `MemoryInitKind.ImplicitlyInitialized` action. + +- **(Potentially) partial writes**: For example, write use in a + Shader. The system is not able to determine if a resource is fully + initialized afterwards but is no longer allowed to perform any + clears, therefore this leads to a + `MemoryInitKind.NeedsInitializedMemory` action, exactly like a read + would. + + */ + +use core::{fmt, iter, ops::Range}; + +use smallvec::SmallVec; + +mod buffer; +mod texture; + +pub(crate) use buffer::{BufferInitTracker, BufferInitTrackerAction}; +pub(crate) use texture::{ + has_copy_partial_init_tracker_coverage, TextureInitRange, TextureInitTracker, + TextureInitTrackerAction, +}; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum MemoryInitKind { + // The memory range is going to be written by an already initialized source, + // thus doesn't need extra attention other than marking as initialized. + ImplicitlyInitialized, + // The memory range is going to be read, therefore needs to ensure prior + // initialization. + NeedsInitializedMemory, +} + +// Most of the time a resource is either fully uninitialized (one element) or +// initialized (zero elements). +type UninitializedRangeVec = SmallVec<[Range; 1]>; + +/// Tracks initialization status of a linear range from 0..size +#[derive(Debug, Clone)] +pub(crate) struct InitTracker { + /// Non-overlapping list of all uninitialized ranges, sorted by + /// range end. + uninitialized_ranges: UninitializedRangeVec, +} + +pub(crate) struct UninitializedIter<'a, Idx: fmt::Debug + Ord + Copy> { + uninitialized_ranges: &'a UninitializedRangeVec, + drain_range: Range, + next_index: usize, +} + +impl<'a, Idx> Iterator for UninitializedIter<'a, Idx> +where + Idx: fmt::Debug + Ord + Copy, +{ + type Item = Range; + + fn next(&mut self) -> Option { + self.uninitialized_ranges + .get(self.next_index) + .and_then(|range| { + if range.start < self.drain_range.end { + self.next_index += 1; + Some( + range.start.max(self.drain_range.start) + ..range.end.min(self.drain_range.end), + ) + } else { + None + } + }) + } +} + +pub(crate) struct InitTrackerDrain<'a, Idx: fmt::Debug + Ord + Copy> { + uninitialized_ranges: &'a mut UninitializedRangeVec, + drain_range: Range, + first_index: usize, + next_index: usize, +} + +impl<'a, Idx> Iterator for InitTrackerDrain<'a, Idx> +where + Idx: fmt::Debug + Ord + Copy, +{ + type Item = Range; + + fn next(&mut self) -> Option { + if let Some(r) = self + .uninitialized_ranges + .get(self.next_index) + .and_then(|range| { + if range.start < self.drain_range.end { + Some(range.clone()) + } else { + None + } + }) + { + self.next_index += 1; + Some(r.start.max(self.drain_range.start)..r.end.min(self.drain_range.end)) + } else { + let num_affected = self.next_index - self.first_index; + if num_affected == 0 { + return None; + } + let first_range = &mut self.uninitialized_ranges[self.first_index]; + + // Split one "big" uninitialized range? + if num_affected == 1 + && first_range.start < self.drain_range.start + && first_range.end > self.drain_range.end + { + let old_start = first_range.start; + first_range.start = self.drain_range.end; + self.uninitialized_ranges + .insert(self.first_index, old_start..self.drain_range.start); + } + // Adjust border ranges and delete everything in-between. + else { + let remove_start = if first_range.start >= self.drain_range.start { + self.first_index + } else { + first_range.end = self.drain_range.start; + self.first_index + 1 + }; + + let last_range = &mut self.uninitialized_ranges[self.next_index - 1]; + let remove_end = if last_range.end <= self.drain_range.end { + self.next_index + } else { + last_range.start = self.drain_range.end; + self.next_index - 1 + }; + + self.uninitialized_ranges.drain(remove_start..remove_end); + } + + None + } + } +} + +impl<'a, Idx> Drop for InitTrackerDrain<'a, Idx> +where + Idx: fmt::Debug + Ord + Copy, +{ + fn drop(&mut self) { + if self.next_index <= self.first_index { + for _ in self {} + } + } +} + +impl InitTracker +where + Idx: fmt::Debug + Ord + Copy + Default, +{ + pub(crate) fn new(size: Idx) -> Self { + Self { + uninitialized_ranges: iter::once(Idx::default()..size).collect(), + } + } + + /// Checks for uninitialized ranges within a given query range. + /// + /// If `query_range` includes any uninitialized portions of this init + /// tracker's resource, return the smallest subrange of `query_range` that + /// covers all uninitialized regions. + /// + /// The returned range may be larger than necessary, to keep this function + /// O(log n). + pub(crate) fn check(&self, query_range: Range) -> Option> { + let index = self + .uninitialized_ranges + .partition_point(|r| r.end <= query_range.start); + self.uninitialized_ranges + .get(index) + .and_then(|start_range| { + if start_range.start < query_range.end { + let start = start_range.start.max(query_range.start); + match self.uninitialized_ranges.get(index + 1) { + Some(next_range) => { + if next_range.start < query_range.end { + // Would need to keep iterating for more + // accurate upper bound. Don't do that here. + Some(start..query_range.end) + } else { + Some(start..start_range.end.min(query_range.end)) + } + } + None => Some(start..start_range.end.min(query_range.end)), + } + } else { + None + } + }) + } + + // Returns an iterator over the uninitialized ranges in a query range. + pub(crate) fn uninitialized(&mut self, drain_range: Range) -> UninitializedIter<'_, Idx> { + let index = self + .uninitialized_ranges + .partition_point(|r| r.end <= drain_range.start); + UninitializedIter { + drain_range, + uninitialized_ranges: &self.uninitialized_ranges, + next_index: index, + } + } + + // Drains uninitialized ranges in a query range. + pub(crate) fn drain(&mut self, drain_range: Range) -> InitTrackerDrain<'_, Idx> { + let index = self + .uninitialized_ranges + .partition_point(|r| r.end <= drain_range.start); + InitTrackerDrain { + drain_range, + uninitialized_ranges: &mut self.uninitialized_ranges, + first_index: index, + next_index: index, + } + } +} + +impl InitTracker { + // Makes a single entry uninitialized if not already uninitialized + pub(crate) fn discard(&mut self, pos: u32) { + // first range where end>=idx + let r_idx = self.uninitialized_ranges.partition_point(|r| r.end < pos); + if let Some(r) = self.uninitialized_ranges.get(r_idx) { + // Extend range at end + if r.end == pos { + // merge with next? + if let Some(right) = self.uninitialized_ranges.get(r_idx + 1) { + if right.start == pos + 1 { + self.uninitialized_ranges[r_idx] = r.start..right.end; + self.uninitialized_ranges.remove(r_idx + 1); + return; + } + } + self.uninitialized_ranges[r_idx] = r.start..(pos + 1); + } else if r.start > pos { + // may still extend range at beginning + if r.start == pos + 1 { + self.uninitialized_ranges[r_idx] = pos..r.end; + } else { + // previous range end must be smaller than idx, therefore no merge possible + self.uninitialized_ranges.push(pos..(pos + 1)); + } + } + } else { + self.uninitialized_ranges.push(pos..(pos + 1)); + } + } +} + +#[cfg(test)] +mod test { + use alloc::{vec, vec::Vec}; + use core::ops::Range; + + type Tracker = super::InitTracker; + + #[test] + fn check_for_newly_created_tracker() { + let tracker = Tracker::new(10); + assert_eq!(tracker.check(0..10), Some(0..10)); + assert_eq!(tracker.check(0..3), Some(0..3)); + assert_eq!(tracker.check(3..4), Some(3..4)); + assert_eq!(tracker.check(4..10), Some(4..10)); + } + + #[test] + fn check_for_drained_tracker() { + let mut tracker = Tracker::new(10); + tracker.drain(0..10); + assert_eq!(tracker.check(0..10), None); + assert_eq!(tracker.check(0..3), None); + assert_eq!(tracker.check(3..4), None); + assert_eq!(tracker.check(4..10), None); + } + + #[test] + fn check_for_partially_filled_tracker() { + let mut tracker = Tracker::new(25); + // Two regions of uninitialized memory + tracker.drain(0..5); + tracker.drain(10..15); + tracker.drain(20..25); + + assert_eq!(tracker.check(0..25), Some(5..25)); // entire range + + assert_eq!(tracker.check(0..5), None); // left non-overlapping + assert_eq!(tracker.check(3..8), Some(5..8)); // left overlapping region + assert_eq!(tracker.check(3..17), Some(5..17)); // left overlapping region + contained region + + // right overlapping region + contained region (yes, doesn't fix range end!) + assert_eq!(tracker.check(8..22), Some(8..22)); + // right overlapping region + assert_eq!(tracker.check(17..22), Some(17..20)); + // right non-overlapping + assert_eq!(tracker.check(20..25), None); + } + + #[test] + fn drain_already_drained() { + let mut tracker = Tracker::new(30); + tracker.drain(10..20); + + // Overlapping with non-cleared + tracker.drain(5..15); // Left overlap + tracker.drain(15..25); // Right overlap + tracker.drain(0..30); // Inner overlap + + // Clear fully cleared + tracker.drain(0..30); + + assert_eq!(tracker.check(0..30), None); + } + + #[test] + fn drain_never_returns_ranges_twice_for_same_range() { + let mut tracker = Tracker::new(19); + assert_eq!(tracker.drain(0..19).count(), 1); + assert_eq!(tracker.drain(0..19).count(), 0); + + let mut tracker = Tracker::new(17); + assert_eq!(tracker.drain(5..8).count(), 1); + assert_eq!(tracker.drain(5..8).count(), 0); + assert_eq!(tracker.drain(1..3).count(), 1); + assert_eq!(tracker.drain(1..3).count(), 0); + assert_eq!(tracker.drain(7..13).count(), 1); + assert_eq!(tracker.drain(7..13).count(), 0); + } + + #[test] + fn drain_splits_ranges_correctly() { + let mut tracker = Tracker::new(1337); + assert_eq!( + tracker.drain(21..42).collect::>>(), + vec![21..42] + ); + assert_eq!( + tracker.drain(900..1000).collect::>>(), + vec![900..1000] + ); + + // Split ranges. + assert_eq!( + tracker.drain(5..1003).collect::>>(), + vec![5..21, 42..900, 1000..1003] + ); + assert_eq!( + tracker.drain(0..1337).collect::>>(), + vec![0..5, 1003..1337] + ); + } + + #[test] + fn discard_adds_range_on_cleared() { + let mut tracker = Tracker::new(10); + tracker.drain(0..10); + tracker.discard(0); + tracker.discard(5); + tracker.discard(9); + assert_eq!(tracker.check(0..1), Some(0..1)); + assert_eq!(tracker.check(1..5), None); + assert_eq!(tracker.check(5..6), Some(5..6)); + assert_eq!(tracker.check(6..9), None); + assert_eq!(tracker.check(9..10), Some(9..10)); + } + + #[test] + fn discard_does_nothing_on_uncleared() { + let mut tracker = Tracker::new(10); + tracker.discard(0); + tracker.discard(5); + tracker.discard(9); + assert_eq!(tracker.uninitialized_ranges.len(), 1); + assert_eq!(tracker.uninitialized_ranges[0], 0..10); + } + + #[test] + fn discard_extends_ranges() { + let mut tracker = Tracker::new(10); + tracker.drain(3..7); + tracker.discard(2); + tracker.discard(7); + assert_eq!(tracker.uninitialized_ranges.len(), 2); + assert_eq!(tracker.uninitialized_ranges[0], 0..3); + assert_eq!(tracker.uninitialized_ranges[1], 7..10); + } + + #[test] + fn discard_merges_ranges() { + let mut tracker = Tracker::new(10); + tracker.drain(3..4); + tracker.discard(3); + assert_eq!(tracker.uninitialized_ranges.len(), 1); + assert_eq!(tracker.uninitialized_ranges[0], 0..10); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/init_tracker/texture.rs b/third_party/wgpu-core-29.0.4-patched/src/init_tracker/texture.rs new file mode 100644 index 00000000..e03f8d3a --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/init_tracker/texture.rs @@ -0,0 +1,107 @@ +use super::{InitTracker, MemoryInitKind}; +use crate::resource::Texture; +use alloc::{sync::Arc, vec::Vec}; +use arrayvec::ArrayVec; +use core::ops::Range; +use wgt::TextureSelector; + +#[derive(Debug, Clone)] +pub(crate) struct TextureInitRange { + pub(crate) mip_range: Range, + // Strictly array layers. We do *not* track volume slices separately. + pub(crate) layer_range: Range, +} + +// Returns true if a copy operation doesn't fully cover the texture init +// tracking granularity. I.e. if this function returns true for a pending copy +// operation, the target texture needs to be ensured to be initialized first! +pub(crate) fn has_copy_partial_init_tracker_coverage( + copy_size: &wgt::Extent3d, + mip_level: u32, + desc: &wgt::TextureDescriptor<(), Vec>, +) -> bool { + let target_size = desc.mip_level_size(mip_level).unwrap(); + copy_size.width != target_size.width + || copy_size.height != target_size.height + || (desc.dimension == wgt::TextureDimension::D3 + && copy_size.depth_or_array_layers != target_size.depth_or_array_layers) +} + +impl From for TextureInitRange { + fn from(selector: TextureSelector) -> Self { + TextureInitRange { + mip_range: selector.mips, + layer_range: selector.layers, + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct TextureInitTrackerAction { + pub(crate) texture: Arc, + pub(crate) range: TextureInitRange, + pub(crate) kind: MemoryInitKind, +} + +pub(crate) type TextureLayerInitTracker = InitTracker; + +#[derive(Debug)] +pub(crate) struct TextureInitTracker { + pub mips: ArrayVec, +} + +impl TextureInitTracker { + pub(crate) fn new(mip_level_count: u32, depth_or_array_layers: u32) -> Self { + TextureInitTracker { + mips: core::iter::repeat_n( + TextureLayerInitTracker::new(depth_or_array_layers), + mip_level_count as usize, + ) + .collect(), + } + } + + pub(crate) fn check_action( + &self, + action: &TextureInitTrackerAction, + ) -> Option { + let mut mip_range_start = usize::MAX; + let mut mip_range_end = usize::MIN; + let mut layer_range_start = u32::MAX; + let mut layer_range_end = u32::MIN; + + for (i, mip_tracker) in self + .mips + .iter() + .enumerate() + .take(action.range.mip_range.end as usize) + .skip(action.range.mip_range.start as usize) + { + if let Some(uninitialized_layer_range) = + mip_tracker.check(action.range.layer_range.clone()) + { + mip_range_start = mip_range_start.min(i); + mip_range_end = i + 1; + layer_range_start = layer_range_start.min(uninitialized_layer_range.start); + layer_range_end = layer_range_end.max(uninitialized_layer_range.end); + }; + } + + if mip_range_start < mip_range_end && layer_range_start < layer_range_end { + Some(TextureInitTrackerAction { + texture: action.texture.clone(), + range: TextureInitRange { + mip_range: mip_range_start as u32..mip_range_end as u32, + layer_range: layer_range_start..layer_range_end, + }, + kind: action.kind, + }) + } else { + None + } + } + + pub(crate) fn discard(&mut self, mip_level: u32, layer: u32) { + self.mips[mip_level as usize].discard(layer); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/instance.rs b/third_party/wgpu-core-29.0.4-patched/src/instance.rs new file mode 100644 index 00000000..1f2f9c6a --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/instance.rs @@ -0,0 +1,1248 @@ +use alloc::{ + borrow::{Cow, ToOwned as _}, + boxed::Box, + string::String, + sync::Arc, + vec, + vec::Vec, +}; + +use hashbrown::HashMap; +use thiserror::Error; +use wgt::error::{ErrorType, WebGpuError}; + +use crate::{ + api_log, api_log_debug, + device::{queue::Queue, resource::Device, DeviceDescriptor, DeviceError}, + global::Global, + id::{markers, AdapterId, DeviceId, QueueId, SurfaceId}, + lock::{rank, Mutex}, + present::Presentation, + resource::ResourceType, + resource_log, + timestamp_normalization::TimestampNormalizerInitError, + DOWNLEVEL_WARNING_MESSAGE, +}; + +use wgt::{Backend, Backends, PowerPreference}; + +pub type RequestAdapterOptions = wgt::RequestAdapterOptions; + +#[derive(Clone, Debug, Error)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[error("Limit '{name}' value {requested} is better than allowed {allowed}")] +pub struct FailedLimit { + name: Cow<'static, str>, + requested: u64, + allowed: u64, +} + +impl WebGpuError for FailedLimit { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +fn check_limits(requested: &wgt::Limits, allowed: &wgt::Limits) -> Vec { + let mut failed = Vec::new(); + + requested.check_limits_with_fail_fn(allowed, false, |name, requested, allowed| { + failed.push(FailedLimit { + name: Cow::Borrowed(name), + requested, + allowed, + }) + }); + + failed +} + +#[test] +fn downlevel_default_limits_less_than_default_limits() { + let res = check_limits(&wgt::Limits::downlevel_defaults(), &wgt::Limits::default()); + assert!( + res.is_empty(), + "Downlevel limits are greater than default limits", + ) +} + +#[derive(Default)] +pub struct Instance { + _name: String, + + /// List of instances per `wgpu-hal` backend. + /// + /// The ordering in this list implies prioritization and needs to be preserved. + instance_per_backend: Vec<(Backend, Box)>, + + /// The backends that were requested by the user. + requested_backends: Backends, + + /// The backends that we could have attempted to obtain from `wgpu-hal` — + /// those for which support is compiled in, currently. + /// + /// The union of this and `requested_backends` is the set of backends that would be used, + /// independent of whether accessing the drivers/hardware for them succeeds. + /// To obtain the set of backends actually in use by this instance, check + /// `instance_per_backend` instead. + supported_backends: Backends, + + pub flags: wgt::InstanceFlags, + + /// Non-lifetimed [`raw_window_handle::DisplayHandle`], for keepalive and validation purposes in + /// [`Self::create_surface()`]. + /// + /// When used with `winit`, callers are expected to pass its `OwnedDisplayHandle` (created from + /// the `EventLoop`) here. + display: Option>, +} + +impl Instance { + pub fn new( + name: &str, + mut instance_desc: wgt::InstanceDescriptor, + telemetry: Option, + ) -> Self { + let mut this = Self { + _name: name.to_owned(), + instance_per_backend: Vec::new(), + requested_backends: instance_desc.backends, + supported_backends: Backends::empty(), + flags: instance_desc.flags, + // HACK: We must take ownership of the field here, without being able to pass it into + // try_add_hal(). Remove it from the mutable descriptor instead, while try_add_hal() + // borrows the handle from `this.display` instead. + display: instance_desc.display.take(), + }; + + #[cfg(all(vulkan, not(target_os = "netbsd")))] + this.try_add_hal(hal::api::Vulkan, &instance_desc, telemetry); + #[cfg(metal)] + this.try_add_hal(hal::api::Metal, &instance_desc, telemetry); + #[cfg(dx12)] + this.try_add_hal(hal::api::Dx12, &instance_desc, telemetry); + #[cfg(gles)] + this.try_add_hal(hal::api::Gles, &instance_desc, telemetry); + #[cfg(feature = "noop")] + this.try_add_hal(hal::api::Noop, &instance_desc, telemetry); + + this + } + + /// Helper for `Instance::new()`; attempts to add a single `wgpu-hal` backend to this instance. + fn try_add_hal( + &mut self, + _: A, + instance_desc: &wgt::InstanceDescriptor, + telemetry: Option, + ) { + // Whether or not the backend was requested, and whether or not it succeeds, + // note that we *could* try it. + self.supported_backends |= A::VARIANT.into(); + + if !instance_desc.backends.contains(A::VARIANT.into()) { + log::trace!("Instance::new: backend {:?} not requested", A::VARIANT); + return; + } + + // If this was Some, it was moved into self + assert!(instance_desc.display.is_none()); + + let hal_desc = hal::InstanceDescriptor { + name: "wgpu", + flags: self.flags, + memory_budget_thresholds: instance_desc.memory_budget_thresholds, + backend_options: instance_desc.backend_options.clone(), + telemetry, + // Pass a borrow, the core instance here keeps the owned handle alive already + // WARNING: Using self here, not instance_desc! + display: self.display.as_ref().map(|hdh| { + hdh.display_handle() + .expect("Implementation did not provide a DisplayHandle") + }), + }; + + use hal::Instance as _; + // SAFETY: ??? + match unsafe { A::Instance::init(&hal_desc) } { + Ok(instance) => { + log::debug!("Instance::new: created {:?} backend", A::VARIANT); + self.instance_per_backend + .push((A::VARIANT, Box::new(instance))); + } + Err(err) => { + log::debug!( + "Instance::new: failed to create {:?} backend: {:?}", + A::VARIANT, + err + ); + } + } + } + + pub(crate) fn from_hal_instance( + name: String, + hal_instance: ::Instance, + ) -> Self { + Self { + _name: name, + instance_per_backend: vec![(A::VARIANT, Box::new(hal_instance))], + requested_backends: A::VARIANT.into(), + supported_backends: A::VARIANT.into(), + flags: wgt::InstanceFlags::default(), + display: None, // TODO: Extract display from HAL instance if available? + } + } + + pub fn raw(&self, backend: Backend) -> Option<&dyn hal::DynInstance> { + self.instance_per_backend + .iter() + .find_map(|(instance_backend, instance)| { + (*instance_backend == backend).then(|| instance.as_ref()) + }) + } + + /// # Safety + /// + /// - The raw instance handle returned must not be manually destroyed. + pub unsafe fn as_hal(&self) -> Option<&A::Instance> { + self.raw(A::VARIANT).map(|instance| { + instance + .as_any() + .downcast_ref() + // This should be impossible. It would mean that backend instance and enum type are mismatching. + .expect("Stored instance is not of the correct type") + }) + } + + /// Creates a new surface targeting the given display/window handles. + /// + /// Internally attempts to create hal surfaces for all enabled backends. + /// + /// Fails only if creation for surfaces for all enabled backends fails in which case + /// the error for each enabled backend is listed. + /// Vice versa, if creation for any backend succeeds, success is returned. + /// Surface creation errors are logged to the debug log in any case. + /// + /// # Safety + /// + /// - `display_handle` must be a valid object to create a surface upon, + /// falls back to the instance display handle otherwise. + /// - `window_handle` must remain valid as long as the returned + /// [`SurfaceId`] is being used. + pub unsafe fn create_surface( + &self, + display_handle: Option, + window_handle: raw_window_handle::RawWindowHandle, + ) -> Result { + profiling::scope!("Instance::create_surface"); + + let instance_display_handle = self.display.as_ref().map(|d| { + d.display_handle() + .expect("Implementation did not provide a DisplayHandle") + .as_raw() + }); + let display_handle = match (instance_display_handle, display_handle) { + (Some(a), Some(b)) => { + if a != b { + return Err(CreateSurfaceError::MismatchingDisplayHandle); + } + a + } + (Some(hnd), None) => hnd, + (None, Some(hnd)) => hnd, + (None, None) => return Err(CreateSurfaceError::MissingDisplayHandle), + }; + + let mut errors = HashMap::default(); + let mut surface_per_backend = HashMap::default(); + + for (backend, instance) in &self.instance_per_backend { + match unsafe { + instance + .as_ref() + .create_surface(display_handle, window_handle) + } { + Ok(raw) => { + surface_per_backend.insert(*backend, raw); + } + Err(err) => { + log::debug!( + "Instance::create_surface: failed to create surface for {backend:?}: {err:?}" + ); + errors.insert(*backend, err); + } + } + } + + if surface_per_backend.is_empty() { + Err(CreateSurfaceError::FailedToCreateSurfaceForAnyBackend( + errors, + )) + } else { + let surface = Surface { + presentation: Mutex::new(rank::SURFACE_PRESENTATION, None), + surface_per_backend, + }; + + Ok(surface) + } + } + + /// Creates a new surface from the given drm configuration. + /// + /// # Safety + /// + /// - All parameters must point to valid DRM values. + /// + /// # Platform Support + /// + /// This function is only available on non-apple Unix-like platforms (Linux, FreeBSD) and + /// currently only works with the Vulkan backend. + #[cfg(all( + unix, + not(target_vendor = "apple"), + not(target_family = "wasm"), + not(target_os = "netbsd") + ))] + #[cfg_attr(not(vulkan), expect(unused_variables))] + pub unsafe fn create_surface_from_drm( + &self, + fd: i32, + plane: u32, + connector_id: u32, + width: u32, + height: u32, + refresh_rate: u32, + ) -> Result { + profiling::scope!("Instance::create_surface_from_drm"); + + let mut errors = HashMap::default(); + let mut surface_per_backend: HashMap> = + HashMap::default(); + + #[cfg(all(vulkan, not(target_os = "netbsd")))] + { + let instance = unsafe { self.as_hal::() } + .ok_or(CreateSurfaceError::BackendNotEnabled(Backend::Vulkan))?; + + // Safety must be upheld by the caller + match unsafe { + instance.create_surface_from_drm( + fd, + plane, + connector_id, + width, + height, + refresh_rate, + ) + } { + Ok(surface) => { + surface_per_backend.insert(Backend::Vulkan, Box::new(surface)); + } + Err(err) => { + errors.insert(Backend::Vulkan, err); + } + } + } + + if surface_per_backend.is_empty() { + Err(CreateSurfaceError::FailedToCreateSurfaceForAnyBackend( + errors, + )) + } else { + let surface = Surface { + presentation: Mutex::new(rank::SURFACE_PRESENTATION, None), + surface_per_backend, + }; + + Ok(surface) + } + } + + /// # Safety + /// + /// `layer` must be a valid pointer. + #[cfg(metal)] + pub unsafe fn create_surface_metal( + &self, + layer: *mut core::ffi::c_void, + ) -> Result { + profiling::scope!("Instance::create_surface_metal"); + + let instance = unsafe { self.as_hal::() } + .ok_or(CreateSurfaceError::BackendNotEnabled(Backend::Metal))?; + + let layer = layer.cast(); + // SAFETY: We do this cast and deref. (rather than using `metal` to get the + // object we want) to avoid direct coupling on the `metal` crate. + // + // To wit, this pointer… + // + // - …is properly aligned. + // - …is dereferenceable to a `MetalLayerRef` as an invariant of the `metal` + // field. + // - …points to an _initialized_ `MetalLayerRef`. + // - …is only ever aliased via an immutable reference that lives within this + // lexical scope. + let layer = unsafe { &*layer }; + let raw_surface: Box = + Box::new(instance.create_surface_from_layer(layer)); + + let surface = Surface { + presentation: Mutex::new(rank::SURFACE_PRESENTATION, None), + surface_per_backend: core::iter::once((Backend::Metal, raw_surface)).collect(), + }; + + Ok(surface) + } + + #[cfg(dx12)] + fn create_surface_dx12( + &self, + create_surface_func: impl FnOnce(&hal::dx12::Instance) -> hal::dx12::Surface, + ) -> Result { + let instance = unsafe { self.as_hal::() } + .ok_or(CreateSurfaceError::BackendNotEnabled(Backend::Dx12))?; + let surface: Box = Box::new(create_surface_func(instance)); + + let surface = Surface { + presentation: Mutex::new(rank::SURFACE_PRESENTATION, None), + surface_per_backend: core::iter::once((Backend::Dx12, surface)).collect(), + }; + + Ok(surface) + } + + #[cfg(dx12)] + /// # Safety + /// + /// The visual must be valid and able to be used to make a swapchain with. + pub unsafe fn create_surface_from_visual( + &self, + visual: *mut core::ffi::c_void, + ) -> Result { + profiling::scope!("Instance::instance_create_surface_from_visual"); + self.create_surface_dx12(|inst| unsafe { inst.create_surface_from_visual(visual) }) + } + + #[cfg(dx12)] + /// # Safety + /// + /// The surface_handle must be valid and able to be used to make a swapchain with. + pub unsafe fn create_surface_from_surface_handle( + &self, + surface_handle: *mut core::ffi::c_void, + ) -> Result { + profiling::scope!("Instance::instance_create_surface_from_surface_handle"); + self.create_surface_dx12(|inst| unsafe { + inst.create_surface_from_surface_handle(surface_handle) + }) + } + + #[cfg(dx12)] + /// # Safety + /// + /// The swap_chain_panel must be valid and able to be used to make a swapchain with. + pub unsafe fn create_surface_from_swap_chain_panel( + &self, + swap_chain_panel: *mut core::ffi::c_void, + ) -> Result { + profiling::scope!("Instance::instance_create_surface_from_swap_chain_panel"); + self.create_surface_dx12(|inst| unsafe { + inst.create_surface_from_swap_chain_panel(swap_chain_panel) + }) + } + + pub fn enumerate_adapters(&self, backends: Backends) -> Vec { + profiling::scope!("Instance::enumerate_adapters"); + api_log!("Instance::enumerate_adapters"); + + let mut adapters = Vec::new(); + for (_backend, instance) in self + .instance_per_backend + .iter() + .filter(|(backend, _)| backends.contains(Backends::from(*backend))) + { + // NOTE: We might be using `profiling` without any features. The empty backend of this + // macro emits no code, so unused code linting changes depending on the backend. + profiling::scope!("enumerating", &*alloc::format!("{_backend:?}")); + + let hal_adapters = unsafe { instance.enumerate_adapters(None) }; + for raw in hal_adapters { + let adapter = Adapter::new(raw); + api_log_debug!("Adapter {:?}", adapter.raw.info); + adapters.push(adapter); + } + } + adapters + } + + pub fn request_adapter( + &self, + desc: &wgt::RequestAdapterOptions<&Surface>, + backends: Backends, + ) -> Result { + profiling::scope!("Instance::request_adapter"); + api_log!("Instance::request_adapter"); + + let mut adapters = Vec::new(); + let mut incompatible_surface_backends = Backends::empty(); + let mut no_fallback_backends = Backends::empty(); + let mut no_adapter_backends = Backends::empty(); + + for &(backend, ref instance) in self + .instance_per_backend + .iter() + .filter(|&&(backend, _)| backends.contains(Backends::from(backend))) + { + let compatible_hal_surface = desc + .compatible_surface + .and_then(|surface| surface.raw(backend)); + + let mut backend_adapters = + unsafe { instance.enumerate_adapters(compatible_hal_surface) }; + if backend_adapters.is_empty() { + log::debug!("enabled backend `{backend:?}` has no adapters"); + no_adapter_backends |= Backends::from(backend); + // by continuing, we avoid setting the further error bits below + continue; + } + + if desc.force_fallback_adapter { + log::debug!("Filtering `{backend:?}` for `force_fallback_adapter`"); + backend_adapters.retain(|exposed| { + let keep = exposed.info.device_type == wgt::DeviceType::Cpu; + if !keep { + log::debug!("* Eliminating adapter `{}`", exposed.info.name); + } + keep + }); + if backend_adapters.is_empty() { + log::debug!("* Backend `{backend:?}` has no fallback adapters"); + no_fallback_backends |= Backends::from(backend); + continue; + } + } + + if let Some(surface) = desc.compatible_surface { + backend_adapters.retain(|exposed| { + let capabilities = surface.get_capabilities_with_raw(exposed); + if let Err(err) = capabilities { + log::debug!( + "Adapter {:?} not compatible with surface: {}", + exposed.info, + err + ); + incompatible_surface_backends |= Backends::from(backend); + false + } else { + true + } + }); + if backend_adapters.is_empty() { + incompatible_surface_backends |= Backends::from(backend); + continue; + } + } + adapters.extend(backend_adapters); + } + + match desc.power_preference { + PowerPreference::LowPower => { + sort(&mut adapters, true); + } + PowerPreference::HighPerformance => { + sort(&mut adapters, false); + } + PowerPreference::None => {} + }; + + fn sort(adapters: &mut [hal::DynExposedAdapter], prefer_integrated_gpu: bool) { + adapters + .sort_by_key(|adapter| get_order(adapter.info.device_type, prefer_integrated_gpu)); + } + + fn get_order(device_type: wgt::DeviceType, prefer_integrated_gpu: bool) -> u8 { + // Since devices of type "Other" might really be "Unknown" and come + // from APIs like OpenGL that don't specify device type, Prefer more + // Specific types over Other. + // + // This means that backends which do provide accurate device types + // will be preferred if their device type indicates an actual + // hardware GPU (integrated or discrete). + match device_type { + wgt::DeviceType::DiscreteGpu if prefer_integrated_gpu => 2, + wgt::DeviceType::IntegratedGpu if prefer_integrated_gpu => 1, + wgt::DeviceType::DiscreteGpu => 1, + wgt::DeviceType::IntegratedGpu => 2, + wgt::DeviceType::Other => 3, + wgt::DeviceType::VirtualGpu => 4, + wgt::DeviceType::Cpu => 5, + } + } + + // `request_adapter` can be a bit of a black box. + // Shine some light on its decision in debug log. + if adapters.is_empty() { + log::debug!("Request adapter didn't find compatible adapters."); + } else { + log::debug!( + "Found {} compatible adapters. Sorted by preference:", + adapters.len() + ); + for adapter in &adapters { + log::debug!("* {:?}", adapter.info); + } + } + + if let Some(adapter) = adapters.into_iter().next() { + api_log_debug!("Request adapter result {:?}", adapter.info); + let adapter = Adapter::new(adapter); + Ok(adapter) + } else { + Err(wgt::RequestAdapterError::NotFound { + supported_backends: self.supported_backends, + requested_backends: self.requested_backends, + active_backends: self.active_backends(), + no_fallback_backends, + no_adapter_backends, + incompatible_surface_backends, + }) + } + } + + fn active_backends(&self) -> Backends { + self.instance_per_backend + .iter() + .map(|&(backend, _)| Backends::from(backend)) + .collect() + } +} + +pub struct Surface { + pub(crate) presentation: Mutex>, + pub surface_per_backend: HashMap>, +} + +impl ResourceType for Surface { + const TYPE: &'static str = "Surface"; +} +impl crate::storage::StorageItem for Surface { + type Marker = markers::Surface; +} + +impl Surface { + pub fn get_capabilities( + &self, + adapter: &Adapter, + ) -> Result { + self.get_capabilities_with_raw(&adapter.raw) + } + + pub fn get_capabilities_with_raw( + &self, + adapter: &hal::DynExposedAdapter, + ) -> Result { + let backend = adapter.backend(); + let suf = self + .raw(backend) + .ok_or(GetSurfaceSupportError::NotSupportedByBackend(backend))?; + profiling::scope!("surface_capabilities"); + let caps = unsafe { adapter.adapter.surface_capabilities(suf) } + .ok_or(GetSurfaceSupportError::FailedToRetrieveSurfaceCapabilitiesForAdapter)?; + Ok(caps) + } + + pub fn raw(&self, backend: Backend) -> Option<&dyn hal::DynSurface> { + self.surface_per_backend + .get(&backend) + .map(|surface| surface.as_ref()) + } +} + +impl Drop for Surface { + fn drop(&mut self) { + if let Some(present) = self.presentation.lock().take() { + for (&backend, surface) in &self.surface_per_backend { + if backend == present.device.backend() { + unsafe { surface.unconfigure(present.device.raw()) }; + } + } + } + } +} + +pub struct Adapter { + pub(crate) raw: hal::DynExposedAdapter, +} + +impl Adapter { + pub fn new(mut raw: hal::DynExposedAdapter) -> Self { + // WebGPU requires this offset alignment as lower bound on all adapters. + const MIN_BUFFER_OFFSET_ALIGNMENT_LOWER_BOUND: u32 = 32; + + let limits = &mut raw.capabilities.limits; + + limits.min_uniform_buffer_offset_alignment = limits + .min_uniform_buffer_offset_alignment + .max(MIN_BUFFER_OFFSET_ALIGNMENT_LOWER_BOUND); + limits.min_storage_buffer_offset_alignment = limits + .min_storage_buffer_offset_alignment + .max(MIN_BUFFER_OFFSET_ALIGNMENT_LOWER_BOUND); + + Self { raw } + } + + /// Returns the backend this adapter is using. + pub fn backend(&self) -> Backend { + self.raw.backend() + } + + pub fn is_surface_supported(&self, surface: &Surface) -> bool { + // If get_capabilities returns Err, then the API does not advertise support for the surface. + // + // This could occur if the user is running their app on Wayland but Vulkan does not support + // VK_KHR_wayland_surface. + surface.get_capabilities(self).is_ok() + } + + pub fn get_info(&self) -> wgt::AdapterInfo { + self.raw.info.clone() + } + + pub fn features(&self) -> wgt::Features { + self.raw.features + } + + pub fn limits(&self) -> wgt::Limits { + self.raw.capabilities.limits.clone() + } + + pub fn downlevel_capabilities(&self) -> wgt::DownlevelCapabilities { + self.raw.capabilities.downlevel.clone() + } + + pub fn get_presentation_timestamp(&self) -> wgt::PresentationTimestamp { + unsafe { self.raw.adapter.get_presentation_timestamp() } + } + + pub fn cooperative_matrix_properties(&self) -> Vec { + self.raw.capabilities.cooperative_matrix_properties.clone() + } + + pub fn get_texture_format_features( + &self, + format: wgt::TextureFormat, + ) -> wgt::TextureFormatFeatures { + use hal::TextureFormatCapabilities as Tfc; + + let caps = unsafe { self.raw.adapter.texture_format_capabilities(format) }; + let mut allowed_usages = wgt::TextureUsages::empty(); + + allowed_usages.set(wgt::TextureUsages::COPY_SRC, caps.contains(Tfc::COPY_SRC)); + allowed_usages.set(wgt::TextureUsages::COPY_DST, caps.contains(Tfc::COPY_DST)); + allowed_usages.set( + wgt::TextureUsages::TEXTURE_BINDING, + caps.contains(Tfc::SAMPLED), + ); + allowed_usages.set( + wgt::TextureUsages::STORAGE_BINDING, + caps.intersects( + Tfc::STORAGE_WRITE_ONLY + | Tfc::STORAGE_READ_ONLY + | Tfc::STORAGE_READ_WRITE + | Tfc::STORAGE_ATOMIC, + ), + ); + allowed_usages.set( + wgt::TextureUsages::RENDER_ATTACHMENT | wgt::TextureUsages::TRANSIENT, + caps.intersects(Tfc::COLOR_ATTACHMENT | Tfc::DEPTH_STENCIL_ATTACHMENT), + ); + allowed_usages.set( + wgt::TextureUsages::STORAGE_ATOMIC, + caps.contains(Tfc::STORAGE_ATOMIC), + ); + + let mut flags = wgt::TextureFormatFeatureFlags::empty(); + flags.set( + wgt::TextureFormatFeatureFlags::STORAGE_READ_ONLY, + caps.contains(Tfc::STORAGE_READ_ONLY), + ); + flags.set( + wgt::TextureFormatFeatureFlags::STORAGE_WRITE_ONLY, + caps.contains(Tfc::STORAGE_WRITE_ONLY), + ); + flags.set( + wgt::TextureFormatFeatureFlags::STORAGE_READ_WRITE, + caps.contains(Tfc::STORAGE_READ_WRITE), + ); + + flags.set( + wgt::TextureFormatFeatureFlags::STORAGE_ATOMIC, + caps.contains(Tfc::STORAGE_ATOMIC), + ); + + flags.set( + wgt::TextureFormatFeatureFlags::FILTERABLE, + caps.contains(Tfc::SAMPLED_LINEAR), + ); + + flags.set( + wgt::TextureFormatFeatureFlags::BLENDABLE, + caps.contains(Tfc::COLOR_ATTACHMENT_BLEND), + ); + + flags.set( + wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2, + caps.contains(Tfc::MULTISAMPLE_X2), + ); + flags.set( + wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4, + caps.contains(Tfc::MULTISAMPLE_X4), + ); + flags.set( + wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8, + caps.contains(Tfc::MULTISAMPLE_X8), + ); + flags.set( + wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16, + caps.contains(Tfc::MULTISAMPLE_X16), + ); + + flags.set( + wgt::TextureFormatFeatureFlags::MULTISAMPLE_RESOLVE, + caps.contains(Tfc::MULTISAMPLE_RESOLVE), + ); + + wgt::TextureFormatFeatures { + allowed_usages, + flags, + } + } + + fn create_device_and_queue_from_hal( + self: &Arc, + hal_device: hal::DynOpenDevice, + desc: &DeviceDescriptor, + instance_flags: wgt::InstanceFlags, + ) -> Result<(Arc, Arc), RequestDeviceError> { + api_log!("Adapter::create_device"); + + let device = Device::new(hal_device.device, self, desc, instance_flags)?; + let device = Arc::new(device); + + let queue = Queue::new(device.clone(), hal_device.queue, instance_flags)?; + let queue = Arc::new(queue); + + device.set_queue(&queue); + device.late_init_resources_with_queue()?; + + Ok((device, queue)) + } + + pub fn create_device_and_queue( + self: &Arc, + desc: &DeviceDescriptor, + instance_flags: wgt::InstanceFlags, + ) -> Result<(Arc, Arc), RequestDeviceError> { + // Verify all features were exposed by the adapter + if !self.raw.features.contains(desc.required_features) { + return Err(RequestDeviceError::UnsupportedFeature( + desc.required_features - self.raw.features, + )); + } + + // Check if experimental features are permitted to be enabled. + if desc + .required_features + .intersects(wgt::Features::all_experimental_mask()) + && !desc.experimental_features.is_enabled() + { + return Err(RequestDeviceError::ExperimentalFeaturesNotEnabled( + desc.required_features + .intersection(wgt::Features::all_experimental_mask()), + )); + } + + let caps = &self.raw.capabilities; + if Backends::PRIMARY.contains(Backends::from(self.backend())) + && !caps.downlevel.is_webgpu_compliant() + { + let missing_flags = wgt::DownlevelFlags::compliant() - caps.downlevel.flags; + log::warn!("Missing downlevel flags: {missing_flags:?}\n{DOWNLEVEL_WARNING_MESSAGE}"); + log::warn!("{:#?}", caps.downlevel); + } + + // Verify feature preconditions + if desc + .required_features + .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS) + && self.raw.info.device_type == wgt::DeviceType::DiscreteGpu + { + log::warn!( + "Feature MAPPABLE_PRIMARY_BUFFERS enabled on a discrete gpu. \ + This is a massive performance footgun and likely not what you wanted" + ); + } + + if let Some(failed) = check_limits(&desc.required_limits, &caps.limits).pop() { + return Err(RequestDeviceError::LimitsExceeded(failed)); + } + + let open = unsafe { + self.raw.adapter.open( + desc.required_features, + &desc.required_limits, + &desc.memory_hints, + ) + } + .map_err(DeviceError::from_hal)?; + + self.create_device_and_queue_from_hal(open, desc, instance_flags) + } +} + +crate::impl_resource_type!(Adapter); +crate::impl_storage_item!(Adapter); + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum GetSurfaceSupportError { + #[error("Surface is not supported for the specified backend {0}")] + NotSupportedByBackend(Backend), + #[error("Failed to retrieve surface capabilities for the specified adapter.")] + FailedToRetrieveSurfaceCapabilitiesForAdapter, +} + +#[derive(Clone, Debug, Error)] +/// Error when requesting a device from the adapter +#[non_exhaustive] +pub enum RequestDeviceError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + LimitsExceeded(#[from] FailedLimit), + #[error("Failed to initialize Timestamp Normalizer")] + TimestampNormalizerInitFailed(#[from] TimestampNormalizerInitError), + #[error("Unsupported features were requested: {0}")] + UnsupportedFeature(wgt::Features), + #[error( + "Some experimental features, {0}, were requested, but experimental features are not enabled" + )] + ExperimentalFeaturesNotEnabled(wgt::Features), +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateSurfaceError { + #[error("The backend {0} was not enabled on the instance.")] + BackendNotEnabled(Backend), + #[error("Failed to create surface for any enabled backend: {0:?}")] + FailedToCreateSurfaceForAnyBackend(HashMap), + #[error("The display handle used to create this Instance does not match the one used to create a surface on it")] + MismatchingDisplayHandle, + #[error( + "No `DisplayHandle` is available to create this surface with. When creating a surface with `create_surface()` \ + you must specify a display handle in `InstanceDescriptor::display`. \ + Rarely, if you need to create surfaces from different `DisplayHandle`s (ex. different Wayland or X11 connections), \ + you must use `create_surface_unsafe()`." + )] + MissingDisplayHandle, +} + +impl Global { + /// Creates a new surface targeting the given display/window handles. + /// + /// Internally attempts to create hal surfaces for all enabled backends. + /// + /// Fails only if creation for surfaces for all enabled backends fails in which case + /// the error for each enabled backend is listed. + /// Vice versa, if creation for any backend succeeds, success is returned. + /// Surface creation errors are logged to the debug log in any case. + /// + /// id_in: + /// - If `Some`, the id to assign to the surface. A new one will be generated otherwise. + /// + /// # Safety + /// + /// - `display_handle` must be a valid object to create a surface upon, + /// falls back to the instance display handle otherwise. + /// - `window_handle` must remain valid as long as the returned + /// [`SurfaceId`] is being used. + pub unsafe fn instance_create_surface( + &self, + display_handle: Option, + window_handle: raw_window_handle::RawWindowHandle, + id_in: Option, + ) -> Result { + let surface = unsafe { self.instance.create_surface(display_handle, window_handle) }?; + let id = self.surfaces.prepare(id_in).assign(Arc::new(surface)); + Ok(id) + } + + /// Creates a new surface from the given drm configuration. + /// + /// # Safety + /// + /// - All parameters must point to valid DRM values. + /// + /// # Platform Support + /// + /// This function is only available on non-apple Unix-like platforms (Linux, FreeBSD) and + /// currently only works with the Vulkan backend. + #[cfg(all( + unix, + not(target_vendor = "apple"), + not(target_family = "wasm"), + not(target_os = "netbsd") + ))] + pub unsafe fn instance_create_surface_from_drm( + &self, + fd: i32, + plane: u32, + connector_id: u32, + width: u32, + height: u32, + refresh_rate: u32, + id_in: Option, + ) -> Result { + let surface = unsafe { + self.instance.create_surface_from_drm( + fd, + plane, + connector_id, + width, + height, + refresh_rate, + ) + }?; + let id = self.surfaces.prepare(id_in).assign(Arc::new(surface)); + + Ok(id) + } + + /// # Safety + /// + /// `layer` must be a valid pointer. + #[cfg(metal)] + pub unsafe fn instance_create_surface_metal( + &self, + layer: *mut core::ffi::c_void, + id_in: Option, + ) -> Result { + let surface = unsafe { self.instance.create_surface_metal(layer) }?; + let id = self.surfaces.prepare(id_in).assign(Arc::new(surface)); + Ok(id) + } + + #[cfg(dx12)] + /// # Safety + /// + /// The visual must be valid and able to be used to make a swapchain with. + pub unsafe fn instance_create_surface_from_visual( + &self, + visual: *mut core::ffi::c_void, + id_in: Option, + ) -> Result { + let surface = unsafe { self.instance.create_surface_from_visual(visual) }?; + let id = self.surfaces.prepare(id_in).assign(Arc::new(surface)); + Ok(id) + } + + #[cfg(dx12)] + /// # Safety + /// + /// The surface_handle must be valid and able to be used to make a swapchain with. + pub unsafe fn instance_create_surface_from_surface_handle( + &self, + surface_handle: *mut core::ffi::c_void, + id_in: Option, + ) -> Result { + let surface = unsafe { + self.instance + .create_surface_from_surface_handle(surface_handle) + }?; + let id = self.surfaces.prepare(id_in).assign(Arc::new(surface)); + Ok(id) + } + + #[cfg(dx12)] + /// # Safety + /// + /// The swap_chain_panel must be valid and able to be used to make a swapchain with. + pub unsafe fn instance_create_surface_from_swap_chain_panel( + &self, + swap_chain_panel: *mut core::ffi::c_void, + id_in: Option, + ) -> Result { + let surface = unsafe { + self.instance + .create_surface_from_swap_chain_panel(swap_chain_panel) + }?; + let id = self.surfaces.prepare(id_in).assign(Arc::new(surface)); + Ok(id) + } + + pub fn surface_drop(&self, id: SurfaceId) { + profiling::scope!("Surface::drop"); + + api_log!("Surface::drop {id:?}"); + + self.surfaces.remove(id); + } + + pub fn enumerate_adapters(&self, backends: Backends) -> Vec { + let adapters = self.instance.enumerate_adapters(backends); + adapters + .into_iter() + .map(|adapter| self.hub.adapters.prepare(None).assign(Arc::new(adapter))) + .collect() + } + + pub fn request_adapter( + &self, + desc: &RequestAdapterOptions, + backends: Backends, + id_in: Option, + ) -> Result { + let compatible_surface = desc.compatible_surface.map(|id| self.surfaces.get(id)); + let desc = wgt::RequestAdapterOptions { + power_preference: desc.power_preference, + force_fallback_adapter: desc.force_fallback_adapter, + compatible_surface: compatible_surface.as_deref(), + }; + let adapter = self.instance.request_adapter(&desc, backends)?; + let id = self.hub.adapters.prepare(id_in).assign(Arc::new(adapter)); + Ok(id) + } + + /// # Safety + /// + /// `hal_adapter` must be created from this global internal instance handle. + pub unsafe fn create_adapter_from_hal( + &self, + hal_adapter: hal::DynExposedAdapter, + input: Option, + ) -> AdapterId { + profiling::scope!("Instance::create_adapter_from_hal"); + + let fid = self.hub.adapters.prepare(input); + let id = fid.assign(Arc::new(Adapter::new(hal_adapter))); + + resource_log!("Created Adapter {:?}", id); + id + } + + pub fn adapter_get_info(&self, adapter_id: AdapterId) -> wgt::AdapterInfo { + let adapter = self.hub.adapters.get(adapter_id); + adapter.get_info() + } + + pub fn adapter_get_texture_format_features( + &self, + adapter_id: AdapterId, + format: wgt::TextureFormat, + ) -> wgt::TextureFormatFeatures { + let adapter = self.hub.adapters.get(adapter_id); + adapter.get_texture_format_features(format) + } + + pub fn adapter_features(&self, adapter_id: AdapterId) -> wgt::Features { + let adapter = self.hub.adapters.get(adapter_id); + adapter.features() + } + + pub fn adapter_limits(&self, adapter_id: AdapterId) -> wgt::Limits { + let adapter = self.hub.adapters.get(adapter_id); + adapter.limits() + } + + pub fn adapter_downlevel_capabilities( + &self, + adapter_id: AdapterId, + ) -> wgt::DownlevelCapabilities { + let adapter = self.hub.adapters.get(adapter_id); + adapter.downlevel_capabilities() + } + + pub fn adapter_get_presentation_timestamp( + &self, + adapter_id: AdapterId, + ) -> wgt::PresentationTimestamp { + let adapter = self.hub.adapters.get(adapter_id); + adapter.get_presentation_timestamp() + } + + pub fn adapter_cooperative_matrix_properties( + &self, + adapter_id: AdapterId, + ) -> Vec { + let adapter = self.hub.adapters.get(adapter_id); + adapter.cooperative_matrix_properties() + } + + pub fn adapter_drop(&self, adapter_id: AdapterId) { + profiling::scope!("Adapter::drop"); + api_log!("Adapter::drop {adapter_id:?}"); + + self.hub.adapters.remove(adapter_id); + } +} + +impl Global { + pub fn adapter_request_device( + &self, + adapter_id: AdapterId, + desc: &DeviceDescriptor, + device_id_in: Option, + queue_id_in: Option, + ) -> Result<(DeviceId, QueueId), RequestDeviceError> { + profiling::scope!("Adapter::request_device"); + api_log!("Adapter::request_device"); + + let device_fid = self.hub.devices.prepare(device_id_in); + let queue_fid = self.hub.queues.prepare(queue_id_in); + + let adapter = self.hub.adapters.get(adapter_id); + let (device, queue) = adapter.create_device_and_queue(desc, self.instance.flags)?; + + let device_id = device_fid.assign(device); + resource_log!("Created Device {:?}", device_id); + + let queue_id = queue_fid.assign(queue); + resource_log!("Created Queue {:?}", queue_id); + + Ok((device_id, queue_id)) + } + + /// # Safety + /// + /// - `hal_device` must be created from `adapter_id` or its internal handle. + /// - `desc` must be a subset of `hal_device` features and limits. + pub unsafe fn create_device_from_hal( + &self, + adapter_id: AdapterId, + hal_device: hal::DynOpenDevice, + desc: &DeviceDescriptor, + device_id_in: Option, + queue_id_in: Option, + ) -> Result<(DeviceId, QueueId), RequestDeviceError> { + profiling::scope!("Global::create_device_from_hal"); + + let devices_fid = self.hub.devices.prepare(device_id_in); + let queues_fid = self.hub.queues.prepare(queue_id_in); + + let adapter = self.hub.adapters.get(adapter_id); + let (device, queue) = + adapter.create_device_and_queue_from_hal(hal_device, desc, self.instance.flags)?; + + let device_id = devices_fid.assign(device); + resource_log!("Created Device {:?}", device_id); + + let queue_id = queues_fid.assign(queue); + resource_log!("Created Queue {:?}", queue_id); + + Ok((device_id, queue_id)) + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/lib.rs b/third_party/wgpu-core-29.0.4-patched/src/lib.rs new file mode 100644 index 00000000..cc01e084 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/lib.rs @@ -0,0 +1,249 @@ +//! This library safely implements WebGPU on native platforms. +//! It is designed for integration into browsers, as well as wrapping +//! into other language-specific user-friendly libraries. +//! +//! ## Feature flags +#![doc = document_features::document_features!()] +//! + +#![no_std] +// When we have no backends, we end up with a lot of dead or otherwise unreachable code. +#![cfg_attr( + all( + not(all(feature = "vulkan", not(target_arch = "wasm32"))), + not(all(feature = "metal", any(target_vendor = "apple"))), + not(all(feature = "dx12", windows)), + not(feature = "gles"), + ), + allow(unused, clippy::let_and_return) +)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![allow( + // It is much clearer to assert negative conditions with eq! false + clippy::bool_assert_comparison, + // We don't use syntax sugar where it's not necessary. + clippy::match_like_matches_macro, + // Redundant matching is more explicit. + clippy::redundant_pattern_matching, + // Explicit lifetimes are often easier to reason about. + clippy::needless_lifetimes, + // No need for defaults in the internal types. + clippy::new_without_default, + // Needless updates are more scalable, easier to play with features. + clippy::needless_update, + // Need many arguments for some core functions to be able to re-use code in many situations. + clippy::too_many_arguments, + // It gets in the way a lot and does not prevent bugs in practice. + clippy::pattern_type_mismatch, + // `wgpu-core` isn't entirely user-facing, so it's useful to document internal items. + rustdoc::private_intra_doc_links, +)] +#![warn( + clippy::alloc_instead_of_core, + clippy::ptr_as_ptr, + clippy::std_instead_of_alloc, + clippy::std_instead_of_core, + trivial_casts, + trivial_numeric_casts, + unsafe_op_in_unsafe_fn, + unused_extern_crates, + unused_qualifications +)] +// We use `Arc` in wgpu-core, but on wasm (unless opted out via `fragile-send-sync-non-atomic-wasm`) +// wgpu-hal resources are not Send/Sync, causing a clippy warning for unnecessary `Arc`s. +// We could use `Rc`s in this case as recommended, but unless atomics are enabled +// this doesn't make a difference. +// Therefore, this is only really a concern for users targeting WebGL +// (the only reason to use wgpu-core on the web in the first place) that have atomics enabled. +// +// NOTE: Keep this in sync with `wgpu`. +#![cfg_attr(not(send_sync), allow(clippy::arc_with_non_send_sync))] + +extern crate alloc; +#[cfg(feature = "std")] +extern crate std; +extern crate wgpu_hal as hal; +extern crate wgpu_types as wgt; + +mod as_hal; +pub mod binding_model; +pub mod command; +mod conv; +pub mod device; +pub mod error; +pub mod global; +mod hash_utils; +pub mod hub; +pub mod id; +pub mod identity; +mod indirect_validation; +mod init_tracker; +pub mod instance; +mod lock; +pub mod pipeline; +mod pipeline_cache; +mod pool; +pub mod present; +pub mod ray_tracing; +pub mod registry; +pub mod resource; +mod snatch; +pub mod storage; +mod timestamp_normalization; +mod track; +mod weak_vec; +// This is public for users who pre-compile shaders while still wanting to +// preserve all run-time checks that `wgpu-core` does. +// See , after which this can be +// made private again. +mod scratch; +pub mod validation; + +pub use validation::{map_storage_format_from_naga, map_storage_format_to_naga}; + +pub use hal::{api, MAX_BIND_GROUPS, MAX_COLOR_ATTACHMENTS, MAX_VERTEX_BUFFERS}; +pub use naga; + +use alloc::{ + borrow::{Cow, ToOwned as _}, + string::String, +}; + +pub(crate) use hash_utils::*; + +/// The index of a queue submission. +/// +/// These are the values stored in `Device::fence`. +pub type SubmissionIndex = hal::FenceValue; + +type Index = u32; +type Epoch = u32; + +pub type RawString = *const core::ffi::c_char; +pub type Label<'a> = Option>; + +trait LabelHelpers<'a> { + fn to_hal(&'a self, flags: wgt::InstanceFlags) -> Option<&'a str>; + fn to_string(&self) -> String; +} +impl<'a> LabelHelpers<'a> for Label<'a> { + fn to_hal(&'a self, flags: wgt::InstanceFlags) -> Option<&'a str> { + if flags.contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) { + return None; + } + + self.as_deref() + } + fn to_string(&self) -> String { + self.as_deref().map(str::to_owned).unwrap_or_default() + } +} + +pub fn hal_label>(opt: Option, flags: wgt::InstanceFlags) -> Option { + if flags.contains(wgt::InstanceFlags::DISCARD_HAL_LABELS) { + return None; + } + + opt +} + +const DOWNLEVEL_WARNING_MESSAGE: &str = concat!( + "The underlying API or device in use does not ", + "support enough features to be a fully compliant implementation of WebGPU. ", + "A subset of the features can still be used. ", + "If you are running this program on native and not in a browser and wish to limit ", + "the features you use to the supported subset, ", + "call Adapter::downlevel_properties or Device::downlevel_properties to get ", + "a listing of the features the current ", + "platform supports." +); + +const DOWNLEVEL_ERROR_MESSAGE: &str = concat!( + "This is not an invalid use of WebGPU: the underlying API or device does not ", + "support enough features to be a fully compliant implementation. ", + "A subset of the features can still be used. ", + "If you are running this program on native and not in a browser ", + "and wish to work around this issue, call ", + "Adapter::downlevel_properties or Device::downlevel_properties ", + "to get a listing of the features the current platform supports." +); + +#[cfg(feature = "api_log_info")] +macro_rules! api_log { + ($($arg:tt)+) => (log::info!($($arg)+)) +} +#[cfg(not(feature = "api_log_info"))] +macro_rules! api_log { + ($($arg:tt)+) => (log::trace!($($arg)+)) +} + +#[cfg(feature = "api_log_info")] +macro_rules! api_log_debug { + ($($arg:tt)+) => (log::info!($($arg)+)) +} +#[cfg(not(feature = "api_log_info"))] +macro_rules! api_log_debug { + ($($arg:tt)+) => (log::debug!($($arg)+)) +} + +pub(crate) use api_log; +pub(crate) use api_log_debug; + +#[cfg(feature = "resource_log_info")] +macro_rules! resource_log { + ($($arg:tt)+) => (log::info!($($arg)+)) +} +#[cfg(not(feature = "resource_log_info"))] +macro_rules! resource_log { + ($($arg:tt)+) => (log::trace!($($arg)+)) +} +pub(crate) use resource_log; + +#[inline] +pub(crate) fn get_lowest_common_denom(a: u32, b: u32) -> u32 { + let gcd = if a >= b { + get_greatest_common_divisor(a, b) + } else { + get_greatest_common_divisor(b, a) + }; + a * b / gcd +} + +#[inline] +pub(crate) fn get_greatest_common_divisor(mut a: u32, mut b: u32) -> u32 { + assert!(a >= b); + loop { + let c = a % b; + if c == 0 { + return b; + } else { + a = b; + b = c; + } + } +} + +#[cfg(not(feature = "std"))] +use core::cell::OnceCell as OnceCellOrLock; +#[cfg(feature = "std")] +use std::sync::OnceLock as OnceCellOrLock; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_lcd() { + assert_eq!(get_lowest_common_denom(2, 2), 2); + assert_eq!(get_lowest_common_denom(2, 3), 6); + assert_eq!(get_lowest_common_denom(6, 4), 12); + } + + #[test] + fn test_gcd() { + assert_eq!(get_greatest_common_divisor(5, 1), 1); + assert_eq!(get_greatest_common_divisor(4, 2), 2); + assert_eq!(get_greatest_common_divisor(6, 4), 2); + assert_eq!(get_greatest_common_divisor(7, 7), 7); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/lock/mod.rs b/third_party/wgpu-core-29.0.4-patched/src/lock/mod.rs new file mode 100644 index 00000000..2e83f5a3 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/lock/mod.rs @@ -0,0 +1,55 @@ +//! Instrumented lock types. +//! +//! This module defines a set of instrumented wrappers for the lock +//! types used in `wgpu-core` ([`Mutex`], [`RwLock`], and +//! [`SnatchLock`]) that help us understand and validate `wgpu-core` +//! synchronization. +//! +//! - The [`ranked`] module defines lock types that perform run-time +//! checks to ensure that each thread acquires locks only in a +//! specific order, to prevent deadlocks. +//! +//! - The [`observing`] module defines lock types that record +//! `wgpu-core`'s lock acquisition activity to disk, for later +//! analysis by the `lock-analyzer` binary. +//! +//! - The [`vanilla`] module defines lock types that are +//! uninstrumented, no-overhead wrappers around the standard lock +//! types. +//! +//! If the `wgpu_validate_locks` config is set (for example, with +//! `RUSTFLAGS='--cfg wgpu_validate_locks'`), `wgpu-core` uses the +//! [`ranked`] module's locks. We hope to make this the default for +//! debug builds soon. +//! +//! If the `observe_locks` feature is enabled, `wgpu-core` uses the +//! [`observing`] module's locks. +//! +//! Otherwise, `wgpu-core` uses the [`vanilla`] module's locks. +//! +//! [`Mutex`]: parking_lot::Mutex +//! [`RwLock`]: parking_lot::RwLock +//! [`SnatchLock`]: crate::snatch::SnatchLock + +pub mod rank; + +#[cfg(feature = "std")] // requires thread-locals to work +#[cfg_attr(not(wgpu_validate_locks), allow(dead_code))] +mod ranked; + +#[cfg(feature = "observe_locks")] +mod observing; + +#[cfg_attr(any(wgpu_validate_locks, feature = "observe_locks"), allow(dead_code))] +mod vanilla; + +#[cfg(wgpu_validate_locks)] +use ranked as chosen; + +#[cfg(feature = "observe_locks")] +use observing as chosen; + +#[cfg(not(any(wgpu_validate_locks, feature = "observe_locks")))] +use vanilla as chosen; + +pub use chosen::{Mutex, MutexGuard, RankData, RwLock, RwLockReadGuard, RwLockWriteGuard}; diff --git a/third_party/wgpu-core-29.0.4-patched/src/lock/observing.rs b/third_party/wgpu-core-29.0.4-patched/src/lock/observing.rs new file mode 100644 index 00000000..3c61d613 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/lock/observing.rs @@ -0,0 +1,503 @@ +//! Lock types that observe lock acquisition order. +//! +//! This module's [`Mutex`] type is instrumented to observe the +//! nesting of `wgpu-core` lock acquisitions. Whenever `wgpu-core` +//! acquires one lock while it is already holding another, we note +//! that nesting pair. This tells us what the [`LockRank::followers`] +//! set for each lock would need to include to accommodate +//! `wgpu-core`'s observed behavior. +//! +//! When `wgpu-core`'s `observe_locks` feature is enabled, if the +//! `WGPU_CORE_LOCK_OBSERVE_DIR` environment variable is set to the +//! path of an existing directory, then every thread that acquires a +//! lock in `wgpu-core` will write its own log file to that directory. +//! You can then run the `wgpu` workspace's `lock-analyzer` binary to +//! read those files and summarize the results. The output from +//! `lock-analyzer` has the same form as the lock ranks given in +//! [`lock/rank.rs`]. +//! +//! If the `WGPU_CORE_LOCK_OBSERVE_DIR` environment variable is not +//! set, then no instrumentation takes place, and the locks behave +//! normally. +//! +//! To make sure we capture all acquisitions regardless of when the +//! program exits, each thread writes events directly to its log file +//! as they occur. A `write` system call is generally just a copy from +//! userspace into the kernel's buffer, so hopefully this approach +//! will still have tolerable performance. +//! +//! [`lock/rank.rs`]: ../../../src/wgpu_core/lock/rank.rs.html + +use alloc::{format, string::String}; +use core::{cell::RefCell, panic::Location}; +use std::{ + fs::File, + path::{Path, PathBuf}, +}; + +use super::rank::{LockRank, LockRankSet}; +use crate::FastHashSet; + +pub type RankData = Option; + +/// A `Mutex` instrumented for lock acquisition order observation. +/// +/// This is just a wrapper around a [`parking_lot::Mutex`], along with +/// its rank in the `wgpu_core` lock ordering. +/// +/// For details, see [the module documentation][self]. +pub struct Mutex { + inner: parking_lot::Mutex, + rank: LockRank, +} + +/// A guard produced by locking [`Mutex`]. +/// +/// This is just a wrapper around a [`parking_lot::MutexGuard`], along +/// with the state needed to track lock acquisition. +/// +/// For details, see [the module documentation][self]. +pub struct MutexGuard<'a, T> { + inner: parking_lot::MutexGuard<'a, T>, + _state: LockStateGuard, +} + +impl Mutex { + pub fn new(rank: LockRank, value: T) -> Mutex { + Mutex { + inner: parking_lot::Mutex::new(value), + rank, + } + } + + #[track_caller] + pub fn lock(&self) -> MutexGuard<'_, T> { + let saved = acquire(self.rank, Location::caller()); + MutexGuard { + inner: self.inner.lock(), + _state: LockStateGuard { saved }, + } + } + + pub fn into_inner(self) -> T { + self.inner.into_inner() + } +} + +impl<'a, T> core::ops::Deref for MutexGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, T> core::ops::DerefMut for MutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.deref_mut() + } +} + +impl core::fmt::Debug for Mutex { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.inner.fmt(f) + } +} + +/// An `RwLock` instrumented for lock acquisition order observation. +/// +/// This is just a wrapper around a [`parking_lot::RwLock`], along with +/// its rank in the `wgpu_core` lock ordering. +/// +/// For details, see [the module documentation][self]. +pub struct RwLock { + inner: parking_lot::RwLock, + rank: LockRank, +} + +/// A read guard produced by locking [`RwLock`] for reading. +/// +/// This is just a wrapper around a [`parking_lot::RwLockReadGuard`], along with +/// the state needed to track lock acquisition. +/// +/// For details, see [the module documentation][self]. +pub struct RwLockReadGuard<'a, T> { + inner: parking_lot::RwLockReadGuard<'a, T>, + _state: LockStateGuard, +} + +/// A write guard produced by locking [`RwLock`] for writing. +/// +/// This is just a wrapper around a [`parking_lot::RwLockWriteGuard`], along +/// with the state needed to track lock acquisition. +/// +/// For details, see [the module documentation][self]. +pub struct RwLockWriteGuard<'a, T> { + inner: parking_lot::RwLockWriteGuard<'a, T>, + _state: LockStateGuard, +} + +impl RwLock { + pub fn new(rank: LockRank, value: T) -> RwLock { + RwLock { + inner: parking_lot::RwLock::new(value), + rank, + } + } + + #[track_caller] + pub fn read(&self) -> RwLockReadGuard<'_, T> { + let saved = acquire(self.rank, Location::caller()); + RwLockReadGuard { + inner: self.inner.read(), + _state: LockStateGuard { saved }, + } + } + + #[track_caller] + pub fn write(&self) -> RwLockWriteGuard<'_, T> { + let saved = acquire(self.rank, Location::caller()); + RwLockWriteGuard { + inner: self.inner.write(), + _state: LockStateGuard { saved }, + } + } + + /// Force an read-unlock operation on this lock. + /// + /// Safety: + /// - A read lock must be held which is not held by a guard. + pub unsafe fn force_unlock_read(&self, data: RankData) { + release(data); + unsafe { self.inner.force_unlock_read() }; + } +} + +impl<'a, T> RwLockReadGuard<'a, T> { + // Forget the read guard, leaving the lock in a locked state with no guard. + // + // Equivalent to std::mem::forget, but preserves the information about the lock + // rank. + pub fn forget(this: Self) -> RankData { + core::mem::forget(this.inner); + + this._state.saved + } +} + +impl<'a, T> RwLockWriteGuard<'a, T> { + pub fn downgrade(this: Self) -> RwLockReadGuard<'a, T> { + RwLockReadGuard { + inner: parking_lot::RwLockWriteGuard::downgrade(this.inner), + _state: this._state, + } + } +} + +impl core::fmt::Debug for RwLock { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.inner.fmt(f) + } +} + +impl<'a, T> core::ops::Deref for RwLockReadGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, T> core::ops::Deref for RwLockWriteGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, T> core::ops::DerefMut for RwLockWriteGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.deref_mut() + } +} + +/// A container that restores a prior per-thread lock state when dropped. +/// +/// This type serves two purposes: +/// +/// - Operations like `RwLockWriteGuard::downgrade` would like to be able to +/// destructure lock guards and reassemble their pieces into new guards, but +/// if the guard type itself implements `Drop`, we can't destructure it +/// without unsafe code or pointless `Option`s whose state is almost always +/// statically known. +/// +/// - We can just implement `Drop` for this type once, and then use it in lock +/// guards, rather than implementing `Drop` separately for each guard type. +struct LockStateGuard { + /// The youngest lock that was already held when we acquired this + /// one, if any. + saved: Option, +} + +impl Drop for LockStateGuard { + fn drop(&mut self) { + release(self.saved) + } +} + +/// Check and record the acquisition of a lock with `new_rank`. +/// +/// Log the acquisition of a lock with `new_rank`, and +/// update the per-thread state accordingly. +/// +/// Return the `Option` state that must be restored when this lock is +/// released. +fn acquire(new_rank: LockRank, location: &'static Location<'static>) -> Option { + LOCK_STATE.with_borrow_mut(|state| match *state { + ThreadState::Disabled => None, + ThreadState::Initial => { + let Ok(dir) = std::env::var("WGPU_CORE_LOCK_OBSERVE_DIR") else { + *state = ThreadState::Disabled; + return None; + }; + + // Create the observation log file. + let mut log = ObservationLog::create(dir) + .expect("Failed to open lock observation file (does the dir exist?)"); + + // Log the full set of lock ranks, so that the analysis can even see + // locks that are only acquired in isolation. + for rank in LockRankSet::all().iter() { + log.write_rank(rank); + } + + // Update our state to reflect that we are logging acquisitions, and + // that we have acquired this lock. + *state = ThreadState::Enabled { + held_lock: Some(HeldLock { + rank: new_rank, + location, + }), + log, + }; + + // Since this is the first acquisition on this thread, we know that + // there is no prior lock held, and thus nothing to log yet. + None + } + ThreadState::Enabled { + ref mut held_lock, + ref mut log, + } => { + if let Some(ref held_lock) = held_lock { + log.write_acquisition(held_lock, new_rank, location); + } + + held_lock.replace(HeldLock { + rank: new_rank, + location, + }) + } + }) +} + +/// Record the release of a lock whose saved state was `saved`. +fn release(saved: Option) { + LOCK_STATE.with_borrow_mut(|state| { + if let ThreadState::Enabled { + ref mut held_lock, .. + } = *state + { + *held_lock = saved; + } + }); +} + +std::thread_local! { + static LOCK_STATE: RefCell = const { RefCell::new(ThreadState::Initial) }; +} + +/// Thread-local state for lock observation. +enum ThreadState { + /// This thread hasn't yet checked the environment variable. + Initial, + + /// This thread checked the environment variable, and it was + /// unset, so this thread is not observing lock acquisitions. + Disabled, + + /// Lock observation is enabled for this thread. + Enabled { + held_lock: Option, + log: ObservationLog, + }, +} + +/// Information about a currently held lock. +#[derive(Debug, Copy, Clone)] +pub struct HeldLock { + /// The lock's rank. + rank: LockRank, + + /// Where we acquired the lock. + location: &'static Location<'static>, +} + +/// A log to which we can write observations of lock activity. +struct ObservationLog { + /// The file to which we are logging lock observations. + log_file: File, + + /// [`Location`]s we've seen so far. + /// + /// This is a hashset of raw pointers because raw pointers have + /// the [`Eq`] and [`Hash`] relations we want: the pointer value, not + /// the contents. There's no unsafe code in this module. + locations_seen: FastHashSet<*const Location<'static>>, + + /// Buffer for serializing events, retained for allocation reuse. + buffer: String, +} + +impl ObservationLog { + /// Create an observation log in `dir` for the current pid and thread. + fn create(dir: impl AsRef) -> Result { + let mut path = PathBuf::from(dir.as_ref()); + path.push(format!( + "locks-{}.{:?}.ron", + std::process::id(), + std::thread::current().id() + )); + let log_file = File::create(&path)?; + Ok(ObservationLog { + log_file, + locations_seen: FastHashSet::default(), + buffer: String::new(), + }) + } + + /// Record the acquisition of one lock while holding another. + /// + /// Log that we acquired a lock of `new_rank` at `new_location` while still + /// holding other locks, the most recently acquired of which has + /// `older_rank`. + fn write_acquisition( + &mut self, + older_lock: &HeldLock, + new_rank: LockRank, + new_location: &'static Location<'static>, + ) { + self.write_location(older_lock.location); + self.write_location(new_location); + self.write_action(&Action::Acquisition { + older_rank: older_lock.rank.bit.number(), + older_location: addr(older_lock.location), + newer_rank: new_rank.bit.number(), + newer_location: addr(new_location), + }); + } + + fn write_location(&mut self, location: &'static Location<'static>) { + if self.locations_seen.insert(location) { + self.write_action(&Action::Location { + address: addr(location), + file: location.file(), + line: location.line(), + column: location.column(), + }); + } + } + + fn write_rank(&mut self, rank: LockRankSet) { + self.write_action(&Action::Rank { + bit: rank.number(), + member_name: rank.member_name(), + const_name: rank.const_name(), + }); + } + + fn write_action(&mut self, action: &Action) { + use std::io::Write; + + self.buffer.clear(); + ron::ser::to_writer(&mut self.buffer, &action) + .expect("error serializing `lock::observing::Action`"); + self.buffer.push('\n'); + self.log_file + .write_all(self.buffer.as_bytes()) + .expect("error writing `lock::observing::Action`"); + } +} + +/// An action logged by a thread that is observing lock acquisition order. +/// +/// Each thread's log file is a sequence of these enums, serialized +/// using the [`ron`] crate, one action per line. +/// +/// Lock observation cannot assume that there will be any convenient +/// finalization point before the program exits, so in practice, +/// actions must be written immediately when they occur. This means we +/// can't, say, accumulate tables and write them out when they're +/// complete. The `lock-analyzer` binary is then responsible for +/// consolidating the data into a single table of observed transitions. +#[derive(serde::Serialize)] +enum Action { + /// A location that we will refer to in later actions. + /// + /// We write one of these events the first time we see a + /// particular `Location`. Treating this as a separate action + /// simply lets us avoid repeating the content over and over + /// again in every [`Acquisition`] action. + /// + /// [`Acquisition`]: Action::Acquisition + Location { + address: usize, + file: &'static str, + line: u32, + column: u32, + }, + + /// A lock rank that we will refer to in later actions. + /// + /// We write out one these events for every lock rank at the + /// beginning of each thread's log file. Treating this as a + /// separate action simply lets us avoid repeating the names over + /// and over again in every [`Acquisition`] action. + /// + /// [`Acquisition`]: Action::Acquisition + Rank { + bit: u32, + member_name: &'static str, + const_name: &'static str, + }, + + /// An attempt to acquire a lock while holding another lock. + Acquisition { + /// The number of the already acquired lock's rank. + older_rank: u32, + + /// The source position at which we acquired it. Specifically, + /// its `Location`'s address, as an integer. + older_location: usize, + + /// The number of the rank of the lock we are acquiring. + newer_rank: u32, + + /// The source position at which we are acquiring it. + /// Specifically, its `Location`'s address, as an integer. + newer_location: usize, + }, +} + +impl LockRankSet { + /// Return the number of this rank's first member. + fn number(self) -> u32 { + self.bits().trailing_zeros() + } +} + +/// Convenience for `core::ptr::from_ref(t) as usize`. +fn addr(t: &T) -> usize { + core::ptr::from_ref(t) as usize +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/lock/rank.rs b/third_party/wgpu-core-29.0.4-patched/src/lock/rank.rs new file mode 100644 index 00000000..527abf68 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/lock/rank.rs @@ -0,0 +1,163 @@ +//! Ranks for `wgpu-core` locks, restricting acquisition order. +//! +//! See [`LockRank`]. + +/// The rank of a lock. +/// +/// Each [`Mutex`], [`RwLock`], and [`SnatchLock`] in `wgpu-core` has been +/// assigned a *rank*: a node in the DAG defined at the bottom of +/// `wgpu-core/src/lock/rank.rs`. The rank of the most recently +/// acquired lock you are still holding determines which locks you may +/// attempt to acquire next. +/// +/// When you create a lock in `wgpu-core`, you must specify its rank +/// by passing in a [`LockRank`] value. This module declares a +/// pre-defined set of ranks to cover everything in `wgpu-core`, named +/// after the type in which they occur, and the name of the type's +/// field that is a lock. For example, [`CommandBuffer::data`] is a +/// `Mutex`, and its rank here is the constant +/// [`COMMAND_BUFFER_DATA`]. +/// +/// [`Mutex`]: parking_lot::Mutex +/// [`RwLock`]: parking_lot::RwLock +/// [`SnatchLock`]: crate::snatch::SnatchLock +/// [`CommandBuffer::data`]: crate::command::CommandBuffer::data +#[derive(Debug, Copy, Clone)] +pub struct LockRank { + /// The bit representing this lock. + /// + /// There should only be a single bit set in this value. + pub(super) bit: LockRankSet, + + /// A bitmask of permitted successor ranks. + /// + /// If `rank` is the rank of the most recently acquired lock we + /// are still holding, then `rank.followers` is the mask of + /// locks we are allowed to acquire next. + /// + /// The `define_lock_ranks!` macro ensures that there are no + /// cycles in the graph of lock ranks and their followers. + pub(super) followers: LockRankSet, +} + +/// Define a set of lock ranks, and each rank's permitted successors. +macro_rules! define_lock_ranks { + { + $( + $( #[ $attr:meta ] )* + rank $name:ident $member:literal followed by { $( $follower:ident ),* $(,)? } + )* + } => { + // An enum that assigns a unique number to each rank. + #[allow(non_camel_case_types, clippy::upper_case_acronyms)] + enum LockRankNumber { $( $name, )* } + + bitflags::bitflags! { + #[derive(Debug, Copy, Clone, Eq, PartialEq)] + /// A bitflags type representing a set of lock ranks. + pub struct LockRankSet: u64 { + $( + const $name = 1 << (LockRankNumber:: $name as u64); + )* + } + } + + impl LockRankSet { + pub fn member_name(self) -> &'static str { + match self { + $( + LockRankSet:: $name => $member, + )* + _ => "", + } + } + + #[cfg_attr(not(feature = "observe_locks"), allow(dead_code))] + pub fn const_name(self) -> &'static str { + match self { + $( + LockRankSet:: $name => stringify!($name), + )* + _ => "", + } + } + } + + $( + // If there is any cycle in the ranking, the initializers + // for `followers` will be cyclic, and rustc will give us + // an error message explaining the cycle. + $( #[ $attr ] )* + pub const $name: LockRank = LockRank { + bit: LockRankSet:: $name, + followers: LockRankSet::empty() $( .union($follower.bit) )*, + }; + )* + } +} + +define_lock_ranks! { + rank COMMAND_BUFFER_DATA "CommandBuffer::data" followed by { + DEVICE_SNATCHABLE_LOCK, + DEVICE_USAGE_SCOPES, + SHARED_TRACKER_INDEX_ALLOCATOR_INNER, + BUFFER_MAP_STATE, + } + rank DEVICE_SNATCHABLE_LOCK "Device::snatchable_lock" followed by { + SHARED_TRACKER_INDEX_ALLOCATOR_INNER, + DEVICE_TRACE, + BUFFER_MAP_STATE, + // Uncomment this to see an interesting cycle. + // COMMAND_BUFFER_DATA, + } + rank BUFFER_MAP_STATE "Buffer::map_state" followed by { + QUEUE_PENDING_WRITES, + SHARED_TRACKER_INDEX_ALLOCATOR_INNER, + DEVICE_TRACE, + } + rank QUEUE_PENDING_WRITES "Queue::pending_writes" followed by { + COMMAND_ALLOCATOR_FREE_ENCODERS, + SHARED_TRACKER_INDEX_ALLOCATOR_INNER, + QUEUE_LIFE_TRACKER, + } + rank QUEUE_LIFE_TRACKER "Queue::life_tracker" followed by { + COMMAND_ALLOCATOR_FREE_ENCODERS, + DEVICE_TRACE, + } + rank COMMAND_ALLOCATOR_FREE_ENCODERS "CommandAllocator::free_encoders" followed by { + SHARED_TRACKER_INDEX_ALLOCATOR_INNER, + } + + rank BUFFER_BIND_GROUPS "Buffer::bind_groups" followed by { } + rank BUFFER_INITIALIZATION_STATUS "Buffer::initialization_status" followed by { } + rank DEVICE_COMMAND_INDICES "Device::command_indices" followed by {} + rank DEVICE_DEFERRED_DESTROY "Device::deferred_destroy" followed by {} + rank DEVICE_FENCE "Device::fence" followed by { } + rank DEVICE_TRACE "Device::trace" followed by { } + rank DEVICE_TRACKERS "Device::trackers" followed by { } + rank DEVICE_LOST_CLOSURE "Device::device_lost_closure" followed by { } + rank DEVICE_USAGE_SCOPES "Device::usage_scopes" followed by { } + rank IDENTITY_MANAGER_VALUES "IdentityManager::values" followed by { } + rank REGISTRY_STORAGE "Registry::storage" followed by { } + rank RESOURCE_POOL_INNER "ResourcePool::inner" followed by { } + rank SHARED_TRACKER_INDEX_ALLOCATOR_INNER "SharedTrackerIndexAllocator::inner" followed by { } + rank SURFACE_PRESENTATION "Surface::presentation" followed by { } + rank TEXTURE_BIND_GROUPS "Texture::bind_groups" followed by { } + rank TEXTURE_INITIALIZATION_STATUS "Texture::initialization_status" followed by { } + rank TEXTURE_CLEAR_MODE "Texture::clear_mode" followed by { } + rank TEXTURE_VIEWS "Texture::views" followed by { } + rank BLAS_BUILT_INDEX "Blas::built_index" followed by { } + rank BLAS_COMPACTION_STATE "Blas::compaction_size" followed by { } + rank TLAS_BUILT_INDEX "Tlas::built_index" followed by { } + rank TLAS_DEPENDENCIES "Tlas::dependencies" followed by { } + rank BUFFER_POOL "BufferPool::buffers" followed by { } + + #[cfg(test)] + rank PAWN "pawn" followed by { ROOK, BISHOP } + #[cfg(test)] + rank ROOK "rook" followed by { KNIGHT } + #[cfg(test)] + rank KNIGHT "knight" followed by { } + #[cfg(test)] + rank BISHOP "bishop" followed by { } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/lock/ranked.rs b/third_party/wgpu-core-29.0.4-patched/src/lock/ranked.rs new file mode 100644 index 00000000..b633872a --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/lock/ranked.rs @@ -0,0 +1,411 @@ +//! Lock types that enforce well-ranked lock acquisition order. +//! +//! This module's [`Mutex`] and [`RwLock` types are instrumented to check that +//! `wgpu-core` acquires locks according to their rank, to prevent deadlocks. To +//! use it, put `--cfg wgpu_validate_locks` in `RUSTFLAGS`. +//! +//! The [`LockRank`] constants in the [`lock::rank`] module describe edges in a +//! directed graph of lock acquisitions: each lock's rank says, if this is the most +//! recently acquired lock that you are still holding, then these are the locks you +//! are allowed to acquire next. +//! +//! As long as this graph doesn't have cycles, any number of threads can acquire +//! locks along paths through the graph without deadlock: +//! +//! - Assume that if a thread is holding a lock, then it will either release it, +//! or block trying to acquire another one. No thread just sits on its locks +//! forever for unrelated reasons. If it did, then that would be a source of +//! deadlock "outside the system" that we can't do anything about. +//! +//! - This module asserts that threads acquire and release locks in a stack-like +//! order: a lock is dropped only when it is the *most recently acquired* lock +//! *still held* - call this the "youngest" lock. This stack-like ordering +//! isn't a Rust requirement; Rust lets you drop guards in any order you like. +//! This is a restriction we impose. +//! +//! - Consider the directed graph whose nodes are locks, and whose edges go from +//! each lock to its permitted followers, the locks in its [`LockRank::followers`] +//! set. The definition of the [`lock::rank`] module's [`LockRank`] constants +//! ensures that this graph has no cycles, including trivial cycles from a node to +//! itself. +//! +//! - This module then asserts that each thread attempts to acquire a lock only if +//! it is among its youngest lock's permitted followers. Thus, as a thread +//! acquires locks, it must be traversing a path through the graph along its +//! edges. +//! +//! - Because there are no cycles in the graph, whenever one thread is blocked +//! waiting to acquire a lock, that lock must be held by a different thread: if +//! you were allowed to acquire a lock you already hold, that would be a cycle in +//! the graph. +//! +//! - Furthermore, because the graph has no cycles, as we work our way from each +//! thread to the thread it is blocked waiting for, we must eventually reach an +//! end point: there must be some thread that is able to acquire its next lock, or +//! that is about to release a lock. +//! +//! Thus, the system as a whole is always able to make progress: it is free of +//! deadlocks. +//! +//! Note that this validation only monitors each thread's behavior in isolation: +//! there's only thread-local state, nothing communicated between threads. So we +//! don't detect deadlocks, per se, only the potential to cause deadlocks. This +//! means that the validation is conservative, but more reproducible, since it's not +//! dependent on any particular interleaving of execution. +//! +//! [`lock::rank`]: crate::lock::rank + +use core::{cell::Cell, fmt, ops, panic::Location}; + +use super::rank::LockRank; + +pub use LockState as RankData; + +/// A `Mutex` instrumented for deadlock prevention. +/// +/// This is just a wrapper around a [`parking_lot::Mutex`], along with +/// its rank in the `wgpu_core` lock ordering. +/// +/// For details, see [the module documentation][self]. +pub struct Mutex { + inner: parking_lot::Mutex, + rank: LockRank, +} + +/// A guard produced by locking [`Mutex`]. +/// +/// This is just a wrapper around a [`parking_lot::MutexGuard`], along +/// with the state needed to track lock acquisition. +/// +/// For details, see [the module documentation][self]. +pub struct MutexGuard<'a, T> { + inner: parking_lot::MutexGuard<'a, T>, + saved: LockStateGuard, +} + +std::thread_local! { + static LOCK_STATE: Cell = const { Cell::new(LockState::INITIAL) }; +} + +/// Per-thread state for the deadlock checker. +#[derive(Debug, Copy, Clone)] +pub struct LockState { + /// The last lock we acquired, and where. + last_acquired: Option<(LockRank, &'static Location<'static>)>, + + /// The number of locks currently held. + /// + /// This is used to enforce stack-like lock acquisition and release. + depth: u32, +} + +impl LockState { + const INITIAL: LockState = LockState { + last_acquired: None, + depth: 0, + }; +} + +/// A container that restores a [`LockState`] when dropped. +/// +/// This type serves two purposes: +/// +/// - Operations like `RwLockWriteGuard::downgrade` would like to be able to +/// destructure lock guards and reassemble their pieces into new guards, but +/// if the guard type itself implements `Drop`, we can't destructure it +/// without unsafe code or pointless `Option`s whose state is almost always +/// statically known. +/// +/// - We can just implement `Drop` for this type once, and then use it in lock +/// guards, rather than implementing `Drop` separately for each guard type. +struct LockStateGuard(LockState); + +impl Drop for LockStateGuard { + fn drop(&mut self) { + release(self.0) + } +} + +/// Check and record the acquisition of a lock with `new_rank`. +/// +/// Check that acquiring a lock with `new_rank` is permitted at this point, and +/// update the per-thread state accordingly. +/// +/// Return the `LockState` that must be restored when this thread is released. +fn acquire(new_rank: LockRank, location: &'static Location<'static>) -> LockState { + let state = LOCK_STATE.get(); + // Initially, it's fine to acquire any lock. So we only + // need to check when `last_acquired` is `Some`. + if let Some((ref last_rank, ref last_location)) = state.last_acquired { + assert!( + last_rank.followers.contains(new_rank.bit), + "Attempt to acquire nested mutexes in wrong order:\n\ + last locked {:<35} at {}\n\ + now locking {:<35} at {}\n\ + Locking {} after locking {} is not permitted.", + last_rank.bit.member_name(), + last_location, + new_rank.bit.member_name(), + location, + new_rank.bit.member_name(), + last_rank.bit.member_name(), + ); + } + LOCK_STATE.set(LockState { + last_acquired: Some((new_rank, location)), + depth: state.depth + 1, + }); + state +} + +/// Record the release of a lock whose saved state was `saved`. +/// +/// Check that locks are being acquired in stacking order, and update the +/// per-thread state accordingly. +fn release(saved: LockState) { + let prior = LOCK_STATE.replace(saved); + + // Although Rust allows mutex guards to be dropped in any + // order, this analysis requires that locks be acquired and + // released in stack order: the next lock to be released must be + // the most recently acquired lock still held. + assert_eq!( + prior.depth, + saved.depth + 1, + "Lock not released in stacking order" + ); +} + +impl Mutex { + pub fn new(rank: LockRank, value: T) -> Mutex { + Mutex { + inner: parking_lot::Mutex::new(value), + rank, + } + } + + #[track_caller] + pub fn lock(&self) -> MutexGuard<'_, T> { + let saved = acquire(self.rank, Location::caller()); + MutexGuard { + inner: self.inner.lock(), + saved: LockStateGuard(saved), + } + } +} + +impl<'a, T> ops::Deref for MutexGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, T> ops::DerefMut for MutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.deref_mut() + } +} + +impl fmt::Debug for Mutex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(f) + } +} + +/// An `RwLock` instrumented for deadlock prevention. +/// +/// This is just a wrapper around a [`parking_lot::RwLock`], along with +/// its rank in the `wgpu_core` lock ordering. +/// +/// For details, see [the module documentation][self]. +pub struct RwLock { + inner: parking_lot::RwLock, + rank: LockRank, +} + +/// A read guard produced by locking [`RwLock`] for reading. +/// +/// This is just a wrapper around a [`parking_lot::RwLockReadGuard`], along with +/// the state needed to track lock acquisition. +/// +/// For details, see [the module documentation][self]. +pub struct RwLockReadGuard<'a, T> { + inner: parking_lot::RwLockReadGuard<'a, T>, + saved: LockStateGuard, +} + +/// A write guard produced by locking [`RwLock`] for writing. +/// +/// This is just a wrapper around a [`parking_lot::RwLockWriteGuard`], along +/// with the state needed to track lock acquisition. +/// +/// For details, see [the module documentation][self]. +pub struct RwLockWriteGuard<'a, T> { + inner: parking_lot::RwLockWriteGuard<'a, T>, + saved: LockStateGuard, +} + +impl RwLock { + pub fn new(rank: LockRank, value: T) -> RwLock { + RwLock { + inner: parking_lot::RwLock::new(value), + rank, + } + } + + #[track_caller] + pub fn read(&self) -> RwLockReadGuard<'_, T> { + let saved = acquire(self.rank, Location::caller()); + RwLockReadGuard { + inner: self.inner.read(), + saved: LockStateGuard(saved), + } + } + + #[track_caller] + pub fn write(&self) -> RwLockWriteGuard<'_, T> { + let saved = acquire(self.rank, Location::caller()); + RwLockWriteGuard { + inner: self.inner.write(), + saved: LockStateGuard(saved), + } + } + + /// Force an read-unlock operation on this lock. + /// + /// Safety: + /// - A read lock must be held which is not held by a guard. + pub unsafe fn force_unlock_read(&self, data: RankData) { + release(data); + unsafe { self.inner.force_unlock_read() }; + } +} + +impl<'a, T> RwLockReadGuard<'a, T> { + // Forget the read guard, leaving the lock in a locked state with no guard. + // + // Equivalent to std::mem::forget, but preserves the information about the lock + // rank. + pub fn forget(this: Self) -> RankData { + core::mem::forget(this.inner); + + this.saved.0 + } +} + +impl<'a, T> RwLockWriteGuard<'a, T> { + pub fn downgrade(this: Self) -> RwLockReadGuard<'a, T> { + RwLockReadGuard { + inner: parking_lot::RwLockWriteGuard::downgrade(this.inner), + saved: this.saved, + } + } +} + +impl fmt::Debug for RwLock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(f) + } +} + +impl<'a, T> ops::Deref for RwLockReadGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, T> ops::Deref for RwLockWriteGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, T> ops::DerefMut for RwLockWriteGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.deref_mut() + } +} + +/// Locks can be acquired in the order indicated by their ranks. +#[test] +fn permitted() { + use super::rank; + + let lock1 = Mutex::new(rank::PAWN, ()); + let lock2 = Mutex::new(rank::ROOK, ()); + + let _guard1 = lock1.lock(); + let _guard2 = lock2.lock(); +} + +/// Locks can only be acquired in the order indicated by their ranks. +#[test] +#[should_panic(expected = "Locking pawn after locking rook")] +fn forbidden_unrelated() { + use super::rank; + + let lock1 = Mutex::new(rank::ROOK, ()); + let lock2 = Mutex::new(rank::PAWN, ()); + + let _guard1 = lock1.lock(); + let _guard2 = lock2.lock(); +} + +/// Lock acquisitions can't skip ranks. +/// +/// These two locks *could* be acquired in this order, but only if other locks +/// are acquired in between them. Skipping ranks isn't allowed. +#[test] +#[should_panic(expected = "Locking knight after locking pawn")] +fn forbidden_skip() { + use super::rank; + + let lock1 = Mutex::new(rank::PAWN, ()); + let lock2 = Mutex::new(rank::KNIGHT, ()); + + let _guard1 = lock1.lock(); + let _guard2 = lock2.lock(); +} + +/// Locks can be acquired and released in a stack-like order. +#[test] +fn stack_like() { + use super::rank; + + let lock1 = Mutex::new(rank::PAWN, ()); + let lock2 = Mutex::new(rank::ROOK, ()); + let lock3 = Mutex::new(rank::BISHOP, ()); + + let guard1 = lock1.lock(); + let guard2 = lock2.lock(); + drop(guard2); + + let guard3 = lock3.lock(); + drop(guard3); + drop(guard1); +} + +/// Locks can only be acquired and released in a stack-like order. +#[test] +#[should_panic(expected = "Lock not released in stacking order")] +fn non_stack_like() { + use super::rank; + + let lock1 = Mutex::new(rank::PAWN, ()); + let lock2 = Mutex::new(rank::ROOK, ()); + + let guard1 = lock1.lock(); + let guard2 = lock2.lock(); + + // Avoid a double panic from dropping this while unwinding due to the panic + // we're testing for. + core::mem::forget(guard2); + + drop(guard1); +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/lock/vanilla.rs b/third_party/wgpu-core-29.0.4-patched/src/lock/vanilla.rs new file mode 100644 index 00000000..4ed38c38 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/lock/vanilla.rs @@ -0,0 +1,151 @@ +//! Plain, uninstrumented wrappers around [`parking_lot`] lock types. +//! +//! These definitions are used when no particular lock instrumentation +//! Cargo feature is selected. + +use core::{fmt, ops}; + +use crate::lock::rank::LockRank; + +pub struct RankData; + +/// A plain wrapper around [`parking_lot::Mutex`]. +/// +/// This is just like [`parking_lot::Mutex`], except that our [`new`] +/// method takes a rank, indicating where the new mutex should sit in +/// `wgpu-core`'s lock ordering. The rank is ignored. +/// +/// See the [`lock`] module documentation for other wrappers. +/// +/// [`new`]: Mutex::new +/// [`lock`]: crate::lock +pub struct Mutex(parking_lot::Mutex); + +/// A guard produced by locking [`Mutex`]. +/// +/// This is just a wrapper around a [`parking_lot::MutexGuard`]. +pub struct MutexGuard<'a, T>(parking_lot::MutexGuard<'a, T>); + +impl Mutex { + pub fn new(_rank: LockRank, value: T) -> Mutex { + Mutex(parking_lot::Mutex::new(value)) + } + + pub fn lock(&self) -> MutexGuard<'_, T> { + MutexGuard(self.0.lock()) + } + + pub fn into_inner(self) -> T { + self.0.into_inner() + } +} + +impl<'a, T> ops::Deref for MutexGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl<'a, T> ops::DerefMut for MutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0.deref_mut() + } +} + +impl fmt::Debug for Mutex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// A plain wrapper around [`parking_lot::RwLock`]. +/// +/// This is just like [`parking_lot::RwLock`], except that our [`new`] +/// method takes a rank, indicating where the new mutex should sit in +/// `wgpu-core`'s lock ordering. The rank is ignored. +/// +/// See the [`lock`] module documentation for other wrappers. +/// +/// [`new`]: RwLock::new +/// [`lock`]: crate::lock +pub struct RwLock(parking_lot::RwLock); + +/// A read guard produced by locking [`RwLock`] as a reader. +/// +/// This is just a wrapper around a [`parking_lot::RwLockReadGuard`]. +pub struct RwLockReadGuard<'a, T>(parking_lot::RwLockReadGuard<'a, T>); + +/// A write guard produced by locking [`RwLock`] as a writer. +/// +/// This is just a wrapper around a [`parking_lot::RwLockWriteGuard`]. +pub struct RwLockWriteGuard<'a, T>(parking_lot::RwLockWriteGuard<'a, T>); + +impl RwLock { + pub fn new(_rank: LockRank, value: T) -> RwLock { + RwLock(parking_lot::RwLock::new(value)) + } + + pub fn read(&self) -> RwLockReadGuard<'_, T> { + RwLockReadGuard(self.0.read()) + } + + pub fn write(&self) -> RwLockWriteGuard<'_, T> { + RwLockWriteGuard(self.0.write()) + } + + /// Force an read-unlock operation on this lock. + /// + /// Safety: + /// - A read lock must be held which is not held by a guard. + pub unsafe fn force_unlock_read(&self, _data: RankData) { + unsafe { self.0.force_unlock_read() }; + } +} + +impl<'a, T> RwLockReadGuard<'a, T> { + // Forget the read guard, leaving the lock in a locked state with no guard. + // + // Equivalent to std::mem::forget, but preserves the information about the lock + // rank. + pub fn forget(this: Self) -> RankData { + core::mem::forget(this.0); + + RankData + } +} + +impl<'a, T> RwLockWriteGuard<'a, T> { + pub fn downgrade(this: Self) -> RwLockReadGuard<'a, T> { + RwLockReadGuard(parking_lot::RwLockWriteGuard::downgrade(this.0)) + } +} + +impl fmt::Debug for RwLock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl<'a, T> ops::Deref for RwLockReadGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl<'a, T> ops::Deref for RwLockWriteGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl<'a, T> ops::DerefMut for RwLockWriteGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0.deref_mut() + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/pipeline.rs b/third_party/wgpu-core-29.0.4-patched/src/pipeline.rs new file mode 100644 index 00000000..a9d3ad0a --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/pipeline.rs @@ -0,0 +1,853 @@ +use alloc::{ + borrow::Cow, + boxed::Box, + string::{String, ToString as _}, + sync::Arc, + vec::Vec, +}; +use core::{marker::PhantomData, mem::ManuallyDrop, num::NonZeroU32}; + +use arrayvec::ArrayVec; +use naga::error::ShaderError; +use thiserror::Error; +use wgt::error::{ErrorType, WebGpuError}; + +pub use crate::pipeline_cache::PipelineCacheValidationError; +use crate::{ + binding_model::{ + BindGroupLayout, CreateBindGroupLayoutError, CreatePipelineLayoutError, + GetBindGroupLayoutError, PipelineLayout, + }, + command::ColorAttachmentError, + device::{Device, DeviceError, MissingDownlevelFlags, MissingFeatures, RenderPassContext}, + id::{PipelineCacheId, PipelineLayoutId, ShaderModuleId}, + resource::{InvalidResourceError, Labeled, TrackingData}, + resource_log, validation, Label, +}; + +/// Information about buffer bindings, which +/// is validated against the shader (and pipeline) +/// at draw time as opposed to initialization time. +#[derive(Debug, Default)] +pub(crate) struct LateSizedBufferGroup { + // The order has to match `BindGroup::late_buffer_binding_sizes`. + pub(crate) shader_sizes: Vec, +} + +#[allow(clippy::large_enum_variant)] +pub enum ShaderModuleSource<'a> { + #[cfg(feature = "wgsl")] + Wgsl(Cow<'a, str>), + #[cfg(feature = "glsl")] + Glsl(Cow<'a, str>, naga::front::glsl::Options), + #[cfg(feature = "spirv")] + SpirV(Cow<'a, [u32]>, naga::front::spv::Options), + Naga(Cow<'static, naga::Module>), + /// Dummy variant because `Naga` doesn't have a lifetime and without enough active features it + /// could be the last one active. + #[doc(hidden)] + Dummy(PhantomData<&'a ()>), +} + +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ShaderModuleDescriptor<'a> { + pub label: Label<'a>, + #[cfg_attr(feature = "serde", serde(default))] + pub runtime_checks: wgt::ShaderRuntimeChecks, +} + +pub type ShaderModuleDescriptorPassthrough<'a> = + wgt::CreateShaderModuleDescriptorPassthrough<'a, Label<'a>>; + +#[derive(Debug)] +pub struct ShaderModule { + pub(crate) raw: ManuallyDrop>, + pub(crate) device: Arc, + pub(crate) interface: Option, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, +} + +impl Drop for ShaderModule { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { + self.device.raw().destroy_shader_module(raw); + } + } +} + +crate::impl_resource_type!(ShaderModule); +crate::impl_labeled!(ShaderModule); +crate::impl_parent_device!(ShaderModule); +crate::impl_storage_item!(ShaderModule); + +impl ShaderModule { + pub(crate) fn raw(&self) -> &dyn hal::DynShaderModule { + self.raw.as_ref() + } + + pub(crate) fn finalize_entry_point_name( + &self, + stage: naga::ShaderStage, + entry_point: Option<&str>, + ) -> Result { + match &self.interface { + Some(interface) => interface.finalize_entry_point_name(stage, entry_point), + None => entry_point + .map(|ep| ep.to_string()) + .ok_or(validation::StageError::NoEntryPointFound), + } + } +} + +//Note: `Clone` would require `WithSpan: Clone`. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateShaderModuleError { + #[cfg(feature = "wgsl")] + #[error(transparent)] + Parsing(#[from] ShaderError), + #[cfg(feature = "glsl")] + #[error(transparent)] + ParsingGlsl(#[from] ShaderError), + #[cfg(feature = "spirv")] + #[error(transparent)] + ParsingSpirV(#[from] ShaderError), + #[error("Failed to generate the backend-specific code")] + Generation, + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + Validation(#[from] ShaderError>), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error( + "Shader global {bind:?} uses a group index {group} that exceeds the max_bind_groups limit of {limit}." + )] + InvalidGroupIndex { + bind: naga::ResourceBinding, + group: u32, + limit: u32, + }, + #[error("Generic shader passthrough does not contain any code compatible with this backend.")] + NotCompiledForBackend, +} + +impl WebGpuError for CreateShaderModuleError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + + Self::Generation => ErrorType::Internal, + + Self::Validation(..) | Self::InvalidGroupIndex { .. } => ErrorType::Validation, + #[cfg(feature = "wgsl")] + Self::Parsing(..) => ErrorType::Validation, + #[cfg(feature = "glsl")] + Self::ParsingGlsl(..) => ErrorType::Validation, + #[cfg(feature = "spirv")] + Self::ParsingSpirV(..) => ErrorType::Validation, + Self::NotCompiledForBackend => ErrorType::Validation, + } + } +} + +/// Describes a programmable pipeline stage. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ProgrammableStageDescriptor<'a, SM = ShaderModuleId> { + /// The compiled shader module for this stage. + pub module: SM, + /// The name of the entry point in the compiled shader. The name is selected using the + /// following logic: + /// + /// * If `Some(name)` is specified, there must be a function with this name in the shader. + /// * If a single entry point associated with this stage must be in the shader, then proceed as + /// if `Some(…)` was specified with that entry point's name. + pub entry_point: Option>, + /// Specifies the values of pipeline-overridable constants in the shader module. + /// + /// If an `@id` attribute was specified on the declaration, + /// the key must be the pipeline constant ID as a decimal ASCII number; if not, + /// the key must be the constant's identifier name. + /// + /// The value may represent any of WGSL's concrete scalar types. + pub constants: naga::back::PipelineConstants, + /// Whether workgroup scoped memory will be initialized with zero values for this stage. + /// + /// This is required by the WebGPU spec, but may have overhead which can be avoided + /// for cross-platform applications + pub zero_initialize_workgroup_memory: bool, +} + +/// cbindgen:ignore +pub type ResolvedProgrammableStageDescriptor<'a> = + ProgrammableStageDescriptor<'a, Arc>; + +/// Number of implicit bind groups derived at pipeline creation. +pub type ImplicitBindGroupCount = u8; + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ImplicitLayoutError { + #[error("Unable to reflect the shader {0:?} interface")] + ReflectionError(wgt::ShaderStages), + #[error(transparent)] + BindGroup(#[from] CreateBindGroupLayoutError), + #[error(transparent)] + Pipeline(#[from] CreatePipelineLayoutError), + #[error("Unable to create implicit pipeline layout from passthrough shader stage: {0:?}")] + Passthrough(wgt::ShaderStages), +} + +impl WebGpuError for ImplicitLayoutError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::ReflectionError(_) => ErrorType::Validation, + Self::BindGroup(e) => e.webgpu_error_type(), + Self::Pipeline(e) => e.webgpu_error_type(), + Self::Passthrough(_) => ErrorType::Validation, + } + } +} + +/// Describes a compute pipeline. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ComputePipelineDescriptor< + 'a, + PLL = PipelineLayoutId, + SM = ShaderModuleId, + PLC = PipelineCacheId, +> { + pub label: Label<'a>, + /// The layout of bind groups for this pipeline. + pub layout: Option, + /// The compiled compute stage and its entry point. + pub stage: ProgrammableStageDescriptor<'a, SM>, + /// The pipeline cache to use when creating this pipeline. + pub cache: Option, +} + +/// cbindgen:ignore +pub type ResolvedComputePipelineDescriptor<'a> = + ComputePipelineDescriptor<'a, Arc, Arc, Arc>; + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateComputePipelineError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Unable to derive an implicit layout")] + Implicit(#[from] ImplicitLayoutError), + #[error("Error matching shader requirements against the pipeline")] + Stage(#[from] validation::StageError), + #[error("Internal error: {0}")] + Internal(String), + #[error("Pipeline constant error: {0}")] + PipelineConstants(String), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), +} + +impl WebGpuError for CreateComputePipelineError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::MissingDownlevelFlags(e) => e.webgpu_error_type(), + Self::Implicit(e) => e.webgpu_error_type(), + Self::Stage(e) => e.webgpu_error_type(), + Self::Internal(_) => ErrorType::Internal, + Self::PipelineConstants(_) => ErrorType::Validation, + } + } +} + +#[derive(Debug)] +pub struct ComputePipeline { + pub(crate) raw: ManuallyDrop>, + pub(crate) layout: Arc, + pub(crate) device: Arc, + pub(crate) _shader_module: Arc, + pub(crate) late_sized_buffer_groups: ArrayVec, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, +} + +impl Drop for ComputePipeline { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { + self.device.raw().destroy_compute_pipeline(raw); + } + } +} + +crate::impl_resource_type!(ComputePipeline); +crate::impl_labeled!(ComputePipeline); +crate::impl_parent_device!(ComputePipeline); +crate::impl_storage_item!(ComputePipeline); +crate::impl_trackable!(ComputePipeline); + +impl ComputePipeline { + pub(crate) fn raw(&self) -> &dyn hal::DynComputePipeline { + self.raw.as_ref() + } + + pub fn get_bind_group_layout( + self: &Arc, + index: u32, + ) -> Result, GetBindGroupLayoutError> { + self.layout.get_bind_group_layout(index) + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreatePipelineCacheError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Pipeline cache validation failed")] + Validation(#[from] PipelineCacheValidationError), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), +} + +impl WebGpuError for CreatePipelineCacheError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::Validation(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + } + } +} + +#[derive(Debug)] +pub struct PipelineCache { + pub(crate) raw: ManuallyDrop>, + pub(crate) device: Arc, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, +} + +impl Drop for PipelineCache { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { + self.device.raw().destroy_pipeline_cache(raw); + } + } +} + +crate::impl_resource_type!(PipelineCache); +crate::impl_labeled!(PipelineCache); +crate::impl_parent_device!(PipelineCache); +crate::impl_storage_item!(PipelineCache); + +impl PipelineCache { + pub(crate) fn raw(&self) -> &dyn hal::DynPipelineCache { + self.raw.as_ref() + } +} + +/// Describes how the vertex buffer is interpreted. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +pub struct VertexBufferLayout<'a> { + /// The stride, in bytes, between elements of this buffer. + pub array_stride: wgt::BufferAddress, + /// How often this vertex buffer is "stepped" forward. + pub step_mode: wgt::VertexStepMode, + /// The list of attributes which comprise a single vertex. + pub attributes: Cow<'a, [wgt::VertexAttribute]>, +} + +/// A null vertex buffer layout that may be placed in unused slots. +impl Default for VertexBufferLayout<'_> { + fn default() -> Self { + Self { + array_stride: Default::default(), + step_mode: Default::default(), + attributes: Cow::Borrowed(&[]), + } + } +} + +/// Describes the vertex process in a render pipeline. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct VertexState<'a, SM = ShaderModuleId> { + /// The compiled vertex stage and its entry point. + pub stage: ProgrammableStageDescriptor<'a, SM>, + /// The format of any vertex buffers used with this pipeline. + pub buffers: Cow<'a, [VertexBufferLayout<'a>]>, +} + +/// cbindgen:ignore +pub type ResolvedVertexState<'a> = VertexState<'a, Arc>; + +/// Describes fragment processing in a render pipeline. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct FragmentState<'a, SM = ShaderModuleId> { + /// The compiled fragment stage and its entry point. + pub stage: ProgrammableStageDescriptor<'a, SM>, + /// The effect of draw calls on the color aspect of the output target. + pub targets: Cow<'a, [Option]>, +} + +/// cbindgen:ignore +pub type ResolvedFragmentState<'a> = FragmentState<'a, Arc>; + +/// Describes the task shader in a mesh shader pipeline. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct TaskState<'a, SM = ShaderModuleId> { + /// The compiled task stage and its entry point. + pub stage: ProgrammableStageDescriptor<'a, SM>, +} + +pub type ResolvedTaskState<'a> = TaskState<'a, Arc>; + +/// Describes the mesh shader in a mesh shader pipeline. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct MeshState<'a, SM = ShaderModuleId> { + /// The compiled mesh stage and its entry point. + pub stage: ProgrammableStageDescriptor<'a, SM>, +} + +pub type ResolvedMeshState<'a> = MeshState<'a, Arc>; + +/// Describes a vertex processor for either a conventional or mesh shading +/// pipeline architecture. +/// +/// This is not a public API. It is for use by `player` only. The public APIs +/// are [`VertexState`], [`TaskState`], and [`MeshState`]. +/// +/// cbindgen:ignore +#[doc(hidden)] +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum RenderPipelineVertexProcessor<'a, SM = ShaderModuleId> { + Vertex(VertexState<'a, SM>), + Mesh(Option>, MeshState<'a, SM>), +} + +/// Describes a render (graphics) pipeline. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct RenderPipelineDescriptor< + 'a, + PLL = PipelineLayoutId, + SM = ShaderModuleId, + PLC = PipelineCacheId, +> { + pub label: Label<'a>, + /// The layout of bind groups for this pipeline. + pub layout: Option, + /// The vertex processing state for this pipeline. + pub vertex: VertexState<'a, SM>, + /// The properties of the pipeline at the primitive assembly and rasterization level. + #[cfg_attr(feature = "serde", serde(default))] + pub primitive: wgt::PrimitiveState, + /// The effect of draw calls on the depth and stencil aspects of the output target, if any. + #[cfg_attr(feature = "serde", serde(default))] + pub depth_stencil: Option, + /// The multi-sampling properties of the pipeline. + #[cfg_attr(feature = "serde", serde(default))] + pub multisample: wgt::MultisampleState, + /// The fragment processing state for this pipeline. + pub fragment: Option>, + /// If the pipeline will be used with a multiview render pass, this indicates how many array + /// layers the attachments will have. + pub multiview_mask: Option, + /// The pipeline cache to use when creating this pipeline. + pub cache: Option, +} +/// Describes a mesh shader pipeline. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct MeshPipelineDescriptor< + 'a, + PLL = PipelineLayoutId, + SM = ShaderModuleId, + PLC = PipelineCacheId, +> { + pub label: Label<'a>, + /// The layout of bind groups for this pipeline. + pub layout: Option, + /// The task processing state for this pipeline. + pub task: Option>, + /// The mesh processing state for this pipeline + pub mesh: MeshState<'a, SM>, + /// The properties of the pipeline at the primitive assembly and rasterization level. + #[cfg_attr(feature = "serde", serde(default))] + pub primitive: wgt::PrimitiveState, + /// The effect of draw calls on the depth and stencil aspects of the output target, if any. + #[cfg_attr(feature = "serde", serde(default))] + pub depth_stencil: Option, + /// The multi-sampling properties of the pipeline. + #[cfg_attr(feature = "serde", serde(default))] + pub multisample: wgt::MultisampleState, + /// The fragment processing state for this pipeline. + pub fragment: Option>, + /// If the pipeline will be used with a multiview render pass, this indicates how many array + /// layers the attachments will have. + pub multiview: Option, + /// The pipeline cache to use when creating this pipeline. + pub cache: Option, +} + +/// Describes a render (graphics) pipeline, with either conventional or mesh +/// shading architecture. +/// +/// This is not a public API. It is for use by `player` only. The public APIs +/// are [`RenderPipelineDescriptor`] and [`MeshPipelineDescriptor`]. +/// +/// cbindgen:ignore +#[doc(hidden)] +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct GeneralRenderPipelineDescriptor< + 'a, + PLL = PipelineLayoutId, + SM = ShaderModuleId, + PLC = PipelineCacheId, +> { + pub label: Label<'a>, + /// The layout of bind groups for this pipeline. + pub layout: Option, + /// The vertex processing state for this pipeline. + pub vertex: RenderPipelineVertexProcessor<'a, SM>, + /// The properties of the pipeline at the primitive assembly and rasterization level. + #[cfg_attr(feature = "serde", serde(default))] + pub primitive: wgt::PrimitiveState, + /// The effect of draw calls on the depth and stencil aspects of the output target, if any. + #[cfg_attr(feature = "serde", serde(default))] + pub depth_stencil: Option, + /// The multi-sampling properties of the pipeline. + #[cfg_attr(feature = "serde", serde(default))] + pub multisample: wgt::MultisampleState, + /// The fragment processing state for this pipeline. + pub fragment: Option>, + /// If the pipeline will be used with a multiview render pass, this indicates how many array + /// layers the attachments will have. + pub multiview_mask: Option, + /// The pipeline cache to use when creating this pipeline. + pub cache: Option, +} +impl<'a, PLL, SM, PLC> From> + for GeneralRenderPipelineDescriptor<'a, PLL, SM, PLC> +{ + fn from(value: RenderPipelineDescriptor<'a, PLL, SM, PLC>) -> Self { + Self { + label: value.label, + layout: value.layout, + vertex: RenderPipelineVertexProcessor::Vertex(value.vertex), + primitive: value.primitive, + depth_stencil: value.depth_stencil, + multisample: value.multisample, + fragment: value.fragment, + multiview_mask: value.multiview_mask, + cache: value.cache, + } + } +} +impl<'a, PLL, SM, PLC> From> + for GeneralRenderPipelineDescriptor<'a, PLL, SM, PLC> +{ + fn from(value: MeshPipelineDescriptor<'a, PLL, SM, PLC>) -> Self { + Self { + label: value.label, + layout: value.layout, + vertex: RenderPipelineVertexProcessor::Mesh(value.task, value.mesh), + primitive: value.primitive, + depth_stencil: value.depth_stencil, + multisample: value.multisample, + fragment: value.fragment, + multiview_mask: value.multiview, + cache: value.cache, + } + } +} + +/// Not a public API. For use by `player` only. +/// +/// cbindgen:ignore +pub type ResolvedGeneralRenderPipelineDescriptor<'a> = + GeneralRenderPipelineDescriptor<'a, Arc, Arc, Arc>; + +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct PipelineCacheDescriptor<'a> { + pub label: Label<'a>, + pub data: Option>, + pub fallback: bool, +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ColorStateError { + #[error("Format {0:?} is not renderable")] + FormatNotRenderable(wgt::TextureFormat), + #[error("Format {0:?} is not blendable")] + FormatNotBlendable(wgt::TextureFormat), + #[error("Format {0:?} does not have a color aspect")] + FormatNotColor(wgt::TextureFormat), + #[error("Sample count {0} is not supported by format {1:?} on this device. The WebGPU spec guarantees {2:?} samples are supported by this format. With the TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES feature your device supports {3:?}.")] + InvalidSampleCount(u32, wgt::TextureFormat, Vec, Vec), + #[error("Output format {pipeline} is incompatible with the shader {shader}")] + IncompatibleFormat { + pipeline: validation::NumericType, + shader: validation::NumericType, + }, + #[error("Invalid write mask {0:?}")] + InvalidWriteMask(wgt::ColorWrites), + #[error("Using the blend factor {factor:?} for render target {target} is not possible. Only the first render target may be used when dual-source blending.")] + BlendFactorOnUnsupportedTarget { + factor: wgt::BlendFactor, + target: u32, + }, + #[error( + "Blend factor {factor:?} for render target {target} is not valid. Blend factor must be `one` when using min/max blend operations." + )] + InvalidMinMaxBlendFactor { + factor: wgt::BlendFactor, + target: u32, + }, +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum DepthStencilStateError { + #[error("Format {0:?} is not renderable")] + FormatNotRenderable(wgt::TextureFormat), + #[error("Format {0:?} is not a depth/stencil format")] + FormatNotDepthOrStencil(wgt::TextureFormat), + #[error("Format {0:?} does not have a depth aspect, but depth test/write is enabled")] + FormatNotDepth(wgt::TextureFormat), + #[error("Format {0:?} does not have a stencil aspect, but stencil test/write is enabled")] + FormatNotStencil(wgt::TextureFormat), + #[error("Sample count {0} is not supported by format {1:?} on this device. The WebGPU spec guarantees {2:?} samples are supported by this format. With the TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES feature your device supports {3:?}.")] + InvalidSampleCount(u32, wgt::TextureFormat, Vec, Vec), + #[error("Depth bias is not compatible with non-triangle topology {0:?}")] + DepthBiasWithIncompatibleTopology(wgt::PrimitiveTopology), + #[error("Depth compare function must be specified for depth format {0:?}")] + MissingDepthCompare(wgt::TextureFormat), + #[error("Depth write enabled must be specified for depth format {0:?}")] + MissingDepthWriteEnabled(wgt::TextureFormat), +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateRenderPipelineError { + #[error(transparent)] + ColorAttachment(#[from] ColorAttachmentError), + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Unable to derive an implicit layout")] + Implicit(#[from] ImplicitLayoutError), + #[error("Color state [{0}] is invalid")] + ColorState(u8, #[source] ColorStateError), + #[error("Depth/stencil state is invalid")] + DepthStencilState(#[from] DepthStencilStateError), + #[error("Invalid sample count {0}")] + InvalidSampleCount(u32), + #[error("The number of vertex buffers {given} exceeds the limit {limit}")] + TooManyVertexBuffers { given: u32, limit: u32 }, + #[error("The total number of vertex attributes {given} exceeds the limit {limit}")] + TooManyVertexAttributes { given: u32, limit: u32 }, + #[error("Vertex attribute location {given} must be less than limit {limit}")] + VertexAttributeLocationTooLarge { given: u32, limit: u32 }, + #[error("Vertex buffer {index} stride {given} exceeds the limit {limit}")] + VertexStrideTooLarge { index: u32, given: u32, limit: u32 }, + #[error("Vertex attribute at location {location} stride {given} exceeds the limit {limit}")] + VertexAttributeStrideTooLarge { + location: wgt::ShaderLocation, + given: u32, + limit: u32, + }, + #[error("Vertex buffer {index} stride {stride} does not respect `VERTEX_ALIGNMENT`")] + UnalignedVertexStride { + index: u32, + stride: wgt::BufferAddress, + }, + #[error("Vertex attribute at location {location} has invalid offset {offset}")] + InvalidVertexAttributeOffset { + location: wgt::ShaderLocation, + offset: wgt::BufferAddress, + }, + #[error("Two or more vertex attributes were assigned to the same location in the shader: {0}")] + ShaderLocationClash(u32), + #[error("Strip index format was not set to None but to {strip_index_format:?} while using the non-strip topology {topology:?}")] + StripIndexFormatForNonStripTopology { + strip_index_format: Option, + topology: wgt::PrimitiveTopology, + }, + #[error("Conservative Rasterization is only supported for wgt::PolygonMode::Fill")] + ConservativeRasterizationNonFillPolygonMode, + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), + #[error("Error matching {stage:?} shader requirements against the pipeline")] + Stage { + stage: wgt::ShaderStages, + #[source] + error: validation::StageError, + }, + #[error("Internal error in {stage:?} shader: {error}")] + Internal { + stage: wgt::ShaderStages, + error: String, + }, + #[error("Pipeline constant error in {stage:?} shader: {error}")] + PipelineConstants { + stage: wgt::ShaderStages, + error: String, + }, + #[error("In the provided shader, the type given for group {group} binding {binding} has a size of {size}. As the device does not support `DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED`, the type must have a size that is a multiple of 16 bytes.")] + UnalignedShader { group: u32, binding: u32, size: u64 }, + #[error("Dual-source blending requires exactly one color target, but {count} color targets are present")] + DualSourceBlendingWithMultipleColorTargets { count: usize }, + #[error("{}", concat!( + "At least one color attachment or depth-stencil attachment was expected, ", + "but no render target for the pipeline was specified." + ))] + NoTargetSpecified, + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), +} + +impl WebGpuError for CreateRenderPipelineError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + Self::MissingDownlevelFlags(e) => e.webgpu_error_type(), + + Self::Internal { .. } => ErrorType::Internal, + + Self::ColorAttachment(_) + | Self::Implicit(_) + | Self::ColorState(_, _) + | Self::DepthStencilState(_) + | Self::InvalidSampleCount(_) + | Self::TooManyVertexBuffers { .. } + | Self::TooManyVertexAttributes { .. } + | Self::VertexAttributeLocationTooLarge { .. } + | Self::VertexStrideTooLarge { .. } + | Self::UnalignedVertexStride { .. } + | Self::InvalidVertexAttributeOffset { .. } + | Self::ShaderLocationClash(_) + | Self::StripIndexFormatForNonStripTopology { .. } + | Self::ConservativeRasterizationNonFillPolygonMode + | Self::Stage { .. } + | Self::UnalignedShader { .. } + | Self::DualSourceBlendingWithMultipleColorTargets { .. } + | Self::NoTargetSpecified + | Self::PipelineConstants { .. } + | Self::VertexAttributeStrideTooLarge { .. } => ErrorType::Validation, + } + } +} + +bitflags::bitflags! { + #[repr(transparent)] + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct PipelineFlags: u32 { + const BLEND_CONSTANT = 1 << 0; + const STENCIL_REFERENCE = 1 << 1; + const WRITES_DEPTH = 1 << 2; + const WRITES_STENCIL = 1 << 3; + } +} + +/// How a render pipeline will retrieve attributes from a particular vertex buffer. +#[derive(Clone, Copy, Debug)] +pub struct VertexStep { + /// The byte stride in the buffer between one attribute value and the next. + pub stride: wgt::BufferAddress, + + /// The byte size required to fit the last vertex in the stream. + pub last_stride: wgt::BufferAddress, + + /// Whether the buffer is indexed by vertex number or instance number. + pub mode: wgt::VertexStepMode, +} + +impl Default for VertexStep { + fn default() -> Self { + Self { + stride: 0, + last_stride: 0, + mode: wgt::VertexStepMode::Vertex, + } + } +} + +#[derive(Debug)] +pub struct RenderPipeline { + pub(crate) raw: ManuallyDrop>, + pub(crate) device: Arc, + pub(crate) layout: Arc, + pub(crate) _shader_modules: ArrayVec, { hal::MAX_CONCURRENT_SHADER_STAGES }>, + pub(crate) pass_context: RenderPassContext, + pub(crate) flags: PipelineFlags, + pub(crate) topology: wgt::PrimitiveTopology, + pub(crate) strip_index_format: Option, + pub(crate) vertex_steps: Vec, + pub(crate) late_sized_buffer_groups: ArrayVec, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, + /// Whether this is a mesh shader pipeline + pub(crate) is_mesh: bool, +} + +impl Drop for RenderPipeline { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { + self.device.raw().destroy_render_pipeline(raw); + } + } +} + +crate::impl_resource_type!(RenderPipeline); +crate::impl_labeled!(RenderPipeline); +crate::impl_parent_device!(RenderPipeline); +crate::impl_storage_item!(RenderPipeline); +crate::impl_trackable!(RenderPipeline); + +impl RenderPipeline { + pub(crate) fn raw(&self) -> &dyn hal::DynRenderPipeline { + self.raw.as_ref() + } + + pub fn get_bind_group_layout( + self: &Arc, + index: u32, + ) -> Result, GetBindGroupLayoutError> { + self.layout.get_bind_group_layout(index) + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/pipeline_cache.rs b/third_party/wgpu-core-29.0.4-patched/src/pipeline_cache.rs new file mode 100644 index 00000000..a0c36157 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/pipeline_cache.rs @@ -0,0 +1,541 @@ +use thiserror::Error; +use wgt::{ + error::{ErrorType, WebGpuError}, + AdapterInfo, +}; + +pub const HEADER_LENGTH: usize = size_of::(); + +#[derive(Debug, PartialEq, Eq, Clone, Error)] +#[non_exhaustive] +pub enum PipelineCacheValidationError { + #[error("The pipeline cache data was truncated")] + Truncated, + #[error("The pipeline cache data was longer than recorded")] + // TODO: Is it plausible that this would happen + Extended, + #[error("The pipeline cache data was corrupted (e.g. the hash didn't match)")] + Corrupted, + #[error("The pipeline cacha data was out of date and so cannot be safely used")] + Outdated, + #[error("The cache data was created for a different device")] + DeviceMismatch, + #[error("Pipeline cacha data was created for a future version of wgpu")] + Unsupported, +} + +impl PipelineCacheValidationError { + /// Could the error have been avoided? + /// That is, is there a mistake in user code interacting with the cache + pub fn was_avoidable(&self) -> bool { + match self { + PipelineCacheValidationError::DeviceMismatch => true, + PipelineCacheValidationError::Truncated + | PipelineCacheValidationError::Unsupported + | PipelineCacheValidationError::Extended + // It's unusual, but not implausible, to be downgrading wgpu + | PipelineCacheValidationError::Outdated + | PipelineCacheValidationError::Corrupted => false, + } + } +} + +impl WebGpuError for PipelineCacheValidationError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +/// Validate the data in a pipeline cache +pub fn validate_pipeline_cache<'d>( + cache_data: &'d [u8], + adapter: &AdapterInfo, + validation_key: [u8; 16], +) -> Result<&'d [u8], PipelineCacheValidationError> { + let adapter_key = adapter_key(adapter)?; + let Some((header, remaining_data)) = PipelineCacheHeader::read(cache_data) else { + return Err(PipelineCacheValidationError::Truncated); + }; + if header.magic != MAGIC { + return Err(PipelineCacheValidationError::Corrupted); + } + if header.header_version != HEADER_VERSION { + return Err(PipelineCacheValidationError::Outdated); + } + if header.cache_abi != ABI { + return Err(PipelineCacheValidationError::Outdated); + } + if header.backend != adapter.backend as u8 { + return Err(PipelineCacheValidationError::DeviceMismatch); + } + if header.adapter_key != adapter_key { + return Err(PipelineCacheValidationError::DeviceMismatch); + } + if header.validation_key != validation_key { + // If the validation key is wrong, that means that this device has changed + // in a way where the cache won't be compatible since the cache was made, + // so it is outdated + return Err(PipelineCacheValidationError::Outdated); + } + let data_size: usize = header + .data_size + .try_into() + // If the data was previously more than 4GiB, and we're still on a 32 bit system (ABI check, above) + // Then the data must be corrupted + .map_err(|_| PipelineCacheValidationError::Corrupted)?; + if remaining_data.len() < data_size { + return Err(PipelineCacheValidationError::Truncated); + } + if remaining_data.len() > data_size { + return Err(PipelineCacheValidationError::Extended); + } + if header.hash_space != HASH_SPACE_VALUE { + return Err(PipelineCacheValidationError::Corrupted); + } + Ok(remaining_data) +} + +pub fn add_cache_header( + in_region: &mut [u8], + data: &[u8], + adapter: &AdapterInfo, + validation_key: [u8; 16], +) { + assert_eq!(in_region.len(), HEADER_LENGTH); + let header = PipelineCacheHeader { + adapter_key: adapter_key(adapter) + .expect("Called add_cache_header for an adapter which doesn't support cache data. This is a wgpu internal bug"), + backend: adapter.backend as u8, + cache_abi: ABI, + magic: MAGIC, + header_version: HEADER_VERSION, + validation_key, + hash_space: HASH_SPACE_VALUE, + data_size: data + .len() + .try_into() + .expect("Cache larger than u64::MAX bytes"), + }; + header.write(in_region); +} + +const MAGIC: [u8; 8] = *b"WGPUPLCH"; +const HEADER_VERSION: u32 = 1; +const ABI: u32 = size_of::<*const ()>() as u32; + +/// The value used to fill [`PipelineCacheHeader::hash_space`] +/// +/// If we receive reports of pipeline cache data corruption which is not otherwise caught +/// on a real device, it would be worth modifying this +/// +/// Note that wgpu does not protect against malicious writes to e.g. a file used +/// to store a pipeline cache. +/// That is the responsibility of the end application, such as by using a +/// private space. +const HASH_SPACE_VALUE: u64 = 0xFEDCBA9_876543210; + +#[repr(C)] +#[derive(PartialEq, Eq)] +struct PipelineCacheHeader { + /// The magic header to ensure that we have the right file format + /// Has a value of MAGIC, as above + magic: [u8; 8], + // /// The total size of this header, in bytes + // header_size: u32, + /// The version of this wgpu header + /// Should be equal to HEADER_VERSION above + /// + /// This must always be the second item, after the value above + header_version: u32, + /// The number of bytes in the pointers of this ABI, because some drivers + /// have previously not distinguished between their 32 bit and 64 bit drivers + /// leading to Vulkan data corruption + cache_abi: u32, + /// The id for the backend in use, from [wgt::Backend] + backend: u8, + /// The key which identifiers the device/adapter. + /// This is used to validate that this pipeline cache (probably) was produced for + /// the expected device. + /// On Vulkan: it is a combination of vendor ID and device ID + adapter_key: [u8; 15], + /// A key used to validate that this device is still compatible with the cache + /// + /// This should e.g. contain driver version and/or intermediate compiler versions + validation_key: [u8; 16], + /// The length of the data which is sent to/recieved from the backend + data_size: u64, + /// Space reserved for a hash of the data in future + /// + /// We assume that your cache storage system will be relatively robust, and so + /// do not validate this hash + /// + /// Therefore, this will always have a value of [`HASH_SPACE_VALUE`] + hash_space: u64, +} + +impl PipelineCacheHeader { + fn read(data: &[u8]) -> Option<(PipelineCacheHeader, &[u8])> { + let mut reader = Reader { + data, + total_read: 0, + }; + let magic = reader.read_array()?; + let header_version = reader.read_u32()?; + let cache_abi = reader.read_u32()?; + let backend = reader.read_byte()?; + let adapter_key = reader.read_array()?; + let validation_key = reader.read_array()?; + let data_size = reader.read_u64()?; + let data_hash = reader.read_u64()?; + + assert_eq!(reader.total_read, size_of::()); + + Some(( + PipelineCacheHeader { + magic, + header_version, + cache_abi, + backend, + adapter_key, + validation_key, + data_size, + hash_space: data_hash, + }, + reader.data, + )) + } + + fn write(&self, into: &mut [u8]) -> Option<()> { + let mut writer = Writer { data: into }; + writer.write_array(&self.magic)?; + writer.write_u32(self.header_version)?; + writer.write_u32(self.cache_abi)?; + writer.write_byte(self.backend)?; + writer.write_array(&self.adapter_key)?; + writer.write_array(&self.validation_key)?; + writer.write_u64(self.data_size)?; + writer.write_u64(self.hash_space)?; + + assert_eq!(writer.data.len(), 0); + Some(()) + } +} + +fn adapter_key(adapter: &AdapterInfo) -> Result<[u8; 15], PipelineCacheValidationError> { + match adapter.backend { + wgt::Backend::Vulkan => { + // If these change size, the header format needs to change + // We set the type explicitly so this won't compile in that case + let v: [u8; 4] = adapter.vendor.to_be_bytes(); + let d: [u8; 4] = adapter.device.to_be_bytes(); + let adapter = [ + 255, 255, 255, v[0], v[1], v[2], v[3], d[0], d[1], d[2], d[3], 255, 255, 255, 255, + ]; + Ok(adapter) + } + _ => Err(PipelineCacheValidationError::Unsupported), + } +} + +struct Reader<'a> { + data: &'a [u8], + total_read: usize, +} + +impl<'a> Reader<'a> { + fn read_byte(&mut self) -> Option { + let res = *self.data.first()?; + self.total_read += 1; + self.data = &self.data[1..]; + Some(res) + } + fn read_array(&mut self) -> Option<[u8; N]> { + // Only greater than because we're indexing fenceposts, not items + if N > self.data.len() { + return None; + } + let (start, data) = self.data.split_at(N); + self.total_read += N; + self.data = data; + Some(start.try_into().expect("off-by-one-error in array size")) + } + + // fn read_u16(&mut self) -> Option { + // self.read_array().map(u16::from_be_bytes) + // } + fn read_u32(&mut self) -> Option { + self.read_array().map(u32::from_be_bytes) + } + fn read_u64(&mut self) -> Option { + self.read_array().map(u64::from_be_bytes) + } +} + +struct Writer<'a> { + data: &'a mut [u8], +} + +impl<'a> Writer<'a> { + fn write_byte(&mut self, byte: u8) -> Option<()> { + self.write_array(&[byte]) + } + fn write_array(&mut self, array: &[u8; N]) -> Option<()> { + // Only greater than because we're indexing fenceposts, not items + if N > self.data.len() { + return None; + } + let data = core::mem::take(&mut self.data); + let (start, data) = data.split_at_mut(N); + self.data = data; + start.copy_from_slice(array); + Some(()) + } + + // fn write_u16(&mut self, value: u16) -> Option<()> { + // self.write_array(&value.to_be_bytes()) + // } + fn write_u32(&mut self, value: u32) -> Option<()> { + self.write_array(&value.to_be_bytes()) + } + fn write_u64(&mut self, value: u64) -> Option<()> { + self.write_array(&value.to_be_bytes()) + } +} + +#[cfg(test)] +mod tests { + use alloc::{string::String, vec::Vec}; + use wgt::AdapterInfo; + + use crate::pipeline_cache::{PipelineCacheValidationError as E, HEADER_LENGTH}; + + use super::ABI; + + // Assert the correct size + const _: [(); HEADER_LENGTH] = [(); 64]; + + const ADAPTER: AdapterInfo = AdapterInfo { + name: String::new(), + vendor: 0x0002_FEED, + device: 0xFEFE_FEFE, + device_type: wgt::DeviceType::Other, + device_pci_bus_id: String::new(), + driver: String::new(), + driver_info: String::new(), + backend: wgt::Backend::Vulkan, + subgroup_min_size: 32, + subgroup_max_size: 32, + transient_saves_memory: true, + }; + + // IMPORTANT: If these tests fail, then you MUST increment HEADER_VERSION + const VALIDATION_KEY: [u8; 16] = u128::to_be_bytes(0xFFFFFFFF_FFFFFFFF_88888888_88888888); + #[test] + fn written_header() { + let mut result = [0; HEADER_LENGTH]; + super::add_cache_header(&mut result, &[], &ADAPTER, VALIDATION_KEY); + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x0u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let expected = cache.into_iter().flatten().collect::>(); + + assert_eq!(result.as_slice(), expected.as_slice()); + } + + #[test] + fn valid_data() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x0u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let cache = cache.into_iter().flatten().collect::>(); + let expected: &[u8] = &[]; + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Ok(expected)); + } + #[test] + fn invalid_magic() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"NOT_WGPU", // (Wrong) MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x0u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let cache = cache.into_iter().flatten().collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Err(E::Corrupted)); + } + + #[test] + fn wrong_version() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 2, 0, 0, 0, ABI as u8], // (wrong) Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x0u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let cache = cache.into_iter().flatten().collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Err(E::Outdated)); + } + #[test] + fn wrong_abi() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + // a 14 bit ABI is improbable + [0, 0, 0, 1, 0, 0, 0, 14], // Version and (wrong) ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x0u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Header + ]; + let cache = cache.into_iter().flatten().collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Err(E::Outdated)); + } + + #[test] + fn wrong_backend() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [2, 255, 255, 255, 0, 2, 0xFE, 0xED], // (wrong) Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x0u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let cache = cache.into_iter().flatten().collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Err(E::DeviceMismatch)); + } + #[test] + fn wrong_adapter() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0x00], // Backend and (wrong) Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x0u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let cache = cache.into_iter().flatten().collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Err(E::DeviceMismatch)); + } + #[test] + fn wrong_validation() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_00000000u64.to_be_bytes(), // (wrong) Validation key + 0x0u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let cache = cache.into_iter().flatten().collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Err(E::Outdated)); + } + #[test] + fn too_little_data() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x064u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let cache = cache.into_iter().flatten().collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Err(E::Truncated)); + } + #[test] + fn not_no_data() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 100u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let cache = cache + .into_iter() + .flatten() + .chain(core::iter::repeat_n(0u8, 100)) + .collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + let expected: &[u8] = &[0; 100]; + assert_eq!(validation_result, Ok(expected)); + } + #[test] + fn too_much_data() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x064u64.to_be_bytes(), // Data size + 0xFEDCBA9_876543210u64.to_be_bytes(), // Hash + ]; + let cache = cache + .into_iter() + .flatten() + .chain(core::iter::repeat_n(0u8, 200)) + .collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Err(E::Extended)); + } + #[test] + fn wrong_hash() { + let cache: [[u8; 8]; HEADER_LENGTH / 8] = [ + *b"WGPUPLCH", // MAGIC + [0, 0, 0, 1, 0, 0, 0, ABI as u8], // Version and ABI + [1, 255, 255, 255, 0, 2, 0xFE, 0xED], // Backend and Adapter key + [0xFE, 0xFE, 0xFE, 0xFE, 255, 255, 255, 255], // Backend and Adapter key + 0xFFFFFFFF_FFFFFFFFu64.to_be_bytes(), // Validation key + 0x88888888_88888888u64.to_be_bytes(), // Validation key + 0x0u64.to_be_bytes(), // Data size + 0x00000000_00000000u64.to_be_bytes(), // Hash + ]; + let cache = cache.into_iter().flatten().collect::>(); + let validation_result = super::validate_pipeline_cache(&cache, &ADAPTER, VALIDATION_KEY); + assert_eq!(validation_result, Err(E::Corrupted)); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/pool.rs b/third_party/wgpu-core-29.0.4-patched/src/pool.rs new file mode 100644 index 00000000..ae5272e0 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/pool.rs @@ -0,0 +1,309 @@ +use alloc::sync::{Arc, Weak}; +use core::hash::Hash; + +use hashbrown::{hash_map::Entry, HashMap}; +use once_cell::sync::OnceCell; + +use crate::lock::{rank, Mutex}; +use crate::FastHashMap; + +type SlotInner = Weak; +type ResourcePoolSlot = Arc>>; + +pub struct ResourcePool { + inner: Mutex>>, +} + +impl ResourcePool { + pub fn new() -> Self { + Self { + inner: Mutex::new(rank::RESOURCE_POOL_INNER, HashMap::default()), + } + } + + /// Get a resource from the pool with the given entry map, or create a new + /// one if it doesn't exist using the given constructor. + /// + /// Behaves such that only one resource will be created for each unique + /// entry map at any one time. + pub fn get_or_init(&self, key: K, constructor: F) -> Result, E> + where + F: FnOnce(K) -> Result, E>, + { + // We can't prove at compile time that these will only ever be consumed once, + // so we need to do the check at runtime. + let mut key = Some(key); + let mut constructor = Some(constructor); + + 'race: loop { + let mut map_guard = self.inner.lock(); + + let entry = match map_guard.entry(key.clone().unwrap()) { + // An entry exists for this resource. + // + // We know that either: + // - The resource is still alive, and Weak::upgrade will succeed. + // - The resource is in the process of being dropped, and Weak::upgrade will fail. + // + // The entry will never be empty while the BGL is still alive. + Entry::Occupied(entry) => Arc::clone(entry.get()), + // No entry exists for this resource. + // + // We know that the resource is not alive, so we can create a new entry. + Entry::Vacant(entry) => Arc::clone(entry.insert(Arc::new(OnceCell::new()))), + }; + + drop(map_guard); + + // Some other thread may beat us to initializing the entry, but OnceCell guarantees that only one thread + // will actually initialize the entry. + // + // We pass the strong reference outside of the closure to keep it alive while we're the only one keeping a reference to it. + let mut strong = None; + let weak = entry.get_or_try_init(|| { + let strong_inner = constructor.take().unwrap()(key.take().unwrap())?; + let weak = Arc::downgrade(&strong_inner); + strong = Some(strong_inner); + Ok(weak) + })?; + + // If strong is Some, that means we just initialized the entry, so we can just return it. + if let Some(strong) = strong { + return Ok(strong); + } + + // The entry was already initialized by someone else, so we need to try to upgrade it. + if let Some(strong) = weak.upgrade() { + // We succeed, the resource is still alive, just return that. + return Ok(strong); + } + + // The resource is in the process of being dropped, because upgrade failed. + // The entry still exists in the map, but it points to nothing. + // + // We're in a race with the drop implementation of the resource, + // so lets just go around again. When we go around again: + // - If the entry exists, we might need to go around a few more times. + // - If the entry doesn't exist, we'll create a new one. + continue 'race; + } + } + + /// Remove the given entry map from the pool. + /// + /// Must *only* be called in the Drop impl of [`BindGroupLayout`]. + /// + /// [`BindGroupLayout`]: crate::binding_model::BindGroupLayout + pub fn remove(&self, key: &K) { + let mut map_guard = self.inner.lock(); + + // Weak::upgrade will be failing long before this code is called. All threads trying to access the resource will be spinning, + // waiting for the entry to be removed. It is safe to remove the entry from the map. + map_guard.remove(key); + } +} + +#[cfg(test)] +mod tests { + use core::{ + sync::atomic::{AtomicU32, Ordering}, + time::Duration, + }; + use std::{eprintln, sync::Barrier, thread}; + + use super::*; + + #[test] + fn deduplication() { + let pool = ResourcePool::::new(); + + let mut counter = 0_u32; + + let arc1 = pool + .get_or_init::<_, ()>(0, |key| { + counter += 1; + Ok(Arc::new(key)) + }) + .unwrap(); + + assert_eq!(*arc1, 0); + assert_eq!(counter, 1); + + let arc2 = pool + .get_or_init::<_, ()>(0, |key| { + counter += 1; + Ok(Arc::new(key)) + }) + .unwrap(); + + assert!(Arc::ptr_eq(&arc1, &arc2)); + assert_eq!(*arc2, 0); + assert_eq!(counter, 1); + + drop(arc1); + drop(arc2); + pool.remove(&0); + + let arc3 = pool + .get_or_init::<_, ()>(0, |key| { + counter += 1; + Ok(Arc::new(key)) + }) + .unwrap(); + + assert_eq!(*arc3, 0); + assert_eq!(counter, 2); + } + + // Test name has "2_threads" in the name so nextest reserves two threads for it. + #[test] + fn concurrent_creation_2_threads() { + struct Resources { + pool: ResourcePool, + counter: AtomicU32, + barrier: Barrier, + } + + let resources = Arc::new(Resources { + pool: ResourcePool::::new(), + counter: AtomicU32::new(0), + barrier: Barrier::new(2), + }); + + // Like all races, this is not inherently guaranteed to work, but in practice it should work fine. + // + // To validate the expected order of events, we've put print statements in the code, indicating when each thread is at a certain point. + // The output will look something like this if the test is working as expected: + // + // ``` + // 0: prewait + // 1: prewait + // 1: postwait + // 0: postwait + // 1: init + // 1: postget + // 0: postget + // ``` + fn thread_inner(idx: u8, resources: &Resources) -> Arc { + eprintln!("{idx}: prewait"); + + // Once this returns, both threads should hit get_or_init at about the same time, + // allowing us to actually test concurrent creation. + // + // Like all races, this is not inherently guaranteed to work, but in practice it should work fine. + resources.barrier.wait(); + + eprintln!("{idx}: postwait"); + + let ret = resources + .pool + .get_or_init::<_, ()>(0, |key| { + eprintln!("{idx}: init"); + + // Simulate long running constructor, ensuring that both threads will be in get_or_init. + thread::sleep(Duration::from_millis(250)); + + resources.counter.fetch_add(1, Ordering::SeqCst); + + Ok(Arc::new(key)) + }) + .unwrap(); + + eprintln!("{idx}: postget"); + + ret + } + + let thread1 = thread::spawn({ + let resource_clone = Arc::clone(&resources); + move || thread_inner(1, &resource_clone) + }); + + let arc0 = thread_inner(0, &resources); + + assert_eq!(resources.counter.load(Ordering::Acquire), 1); + + let arc1 = thread1.join().unwrap(); + + assert!(Arc::ptr_eq(&arc0, &arc1)); + } + + // Test name has "2_threads" in the name so nextest reserves two threads for it. + #[test] + fn create_while_drop_2_threads() { + struct Resources { + pool: ResourcePool, + barrier: Barrier, + } + + let resources = Arc::new(Resources { + pool: ResourcePool::::new(), + barrier: Barrier::new(2), + }); + + // Like all races, this is not inherently guaranteed to work, but in practice it should work fine. + // + // To validate the expected order of events, we've put print statements in the code, indicating when each thread is at a certain point. + // The output will look something like this if the test is working as expected: + // + // ``` + // 0: prewait + // 1: prewait + // 1: postwait + // 0: postwait + // 1: postsleep + // 1: removal + // 0: postget + // ``` + // + // The last two _may_ be flipped. + + let existing_entry = resources + .pool + .get_or_init::<_, ()>(0, |key| Ok(Arc::new(key))) + .unwrap(); + + // Drop the entry, but do _not_ remove it from the pool. + // This simulates the situation where the resource arc has been dropped, but the Drop implementation + // has not yet run, which calls remove. + drop(existing_entry); + + fn thread0_inner(resources: &Resources) { + eprintln!("0: prewait"); + resources.barrier.wait(); + + eprintln!("0: postwait"); + // We try to create a new entry, but the entry already exists. + // + // As Arc::upgrade is failing, we will just keep spinning until remove is called. + resources + .pool + .get_or_init::<_, ()>(0, |key| Ok(Arc::new(key))) + .unwrap(); + eprintln!("0: postget"); + } + + fn thread1_inner(resources: &Resources) { + eprintln!("1: prewait"); + resources.barrier.wait(); + + eprintln!("1: postwait"); + // We wait a little bit, making sure that thread0_inner has started spinning. + thread::sleep(Duration::from_millis(250)); + eprintln!("1: postsleep"); + + // We remove the entry from the pool, allowing thread0_inner to re-create. + resources.pool.remove(&0); + eprintln!("1: removal"); + } + + let thread1 = thread::spawn({ + let resource_clone = Arc::clone(&resources); + move || thread1_inner(&resource_clone) + }); + + thread0_inner(&resources); + + thread1.join().unwrap(); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/present.rs b/third_party/wgpu-core-29.0.4-patched/src/present.rs new file mode 100644 index 00000000..524cc1f6 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/present.rs @@ -0,0 +1,423 @@ +/*! Presentation. + +## Lifecycle + +Whenever a submission detects the use of any surface texture, it adds it to the device +tracker for the duration of the submission (temporarily, while recording). +It's added with `UNINITIALIZED` state and transitioned into `empty()` state. +When this texture is presented, we remove it from the device tracker as well as +extract it from the hub. +!*/ + +use alloc::{sync::Arc, vec::Vec}; +use core::mem::ManuallyDrop; + +#[cfg(feature = "trace")] +use crate::device::trace::{Action, IntoTrace}; +use crate::{ + conv, + device::{Device, DeviceError, MissingDownlevelFlags, WaitIdleError}, + global::Global, + hal_label, id, + instance::Surface, + resource, +}; + +use thiserror::Error; +use wgt::{ + error::{ErrorType, WebGpuError}, + SurfaceStatus as Status, +}; + +const FRAME_TIMEOUT_MS: u32 = 1000; + +#[derive(Debug)] +pub(crate) struct Presentation { + pub(crate) device: Arc, + pub(crate) config: wgt::SurfaceConfiguration>, + pub(crate) acquired_texture: Option>, +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum SurfaceError { + #[error("Surface is invalid")] + Invalid, + #[error("Surface is not configured for presentation")] + NotConfigured, + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Surface image is already acquired")] + AlreadyAcquired, + #[error("Texture has been destroyed")] + TextureDestroyed, +} + +impl WebGpuError for SurfaceError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::Invalid + | Self::NotConfigured + | Self::AlreadyAcquired + | Self::TextureDestroyed => ErrorType::Validation, + } + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum ConfigureSurfaceError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Invalid surface")] + InvalidSurface, + #[error("The view format {0:?} is not compatible with texture format {1:?}, only changing srgb-ness is allowed.")] + InvalidViewFormat(wgt::TextureFormat, wgt::TextureFormat), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), + #[error("`SurfaceOutput` must be dropped before a new `Surface` is made")] + PreviousOutputExists, + #[error("Failed to wait for GPU to come idle before reconfiguring the Surface")] + GpuWaitTimeout, + #[error("Both `Surface` width and height must be non-zero. Wait to recreate the `Surface` until the window has non-zero area.")] + ZeroArea, + #[error("`Surface` width and height must be within the maximum supported texture size. Requested was ({width}, {height}), maximum extent for either dimension is {max_texture_dimension_2d}.")] + TooLarge { + width: u32, + height: u32, + max_texture_dimension_2d: u32, + }, + #[error("Surface does not support the adapter's queue family")] + UnsupportedQueueFamily, + #[error("Requested format {requested:?} is not in list of supported formats: {available:?}")] + UnsupportedFormat { + requested: wgt::TextureFormat, + available: Vec, + }, + #[error("Requested present mode {requested:?} is not in the list of supported present modes: {available:?}")] + UnsupportedPresentMode { + requested: wgt::PresentMode, + available: Vec, + }, + #[error("Requested alpha mode {requested:?} is not in the list of supported alpha modes: {available:?}")] + UnsupportedAlphaMode { + requested: wgt::CompositeAlphaMode, + available: Vec, + }, + #[error("Requested usage {requested:?} is not in the list of supported usages: {available:?}")] + UnsupportedUsage { + requested: wgt::TextureUses, + available: wgt::TextureUses, + }, +} + +impl From for ConfigureSurfaceError { + fn from(e: WaitIdleError) -> Self { + match e { + WaitIdleError::Device(d) => ConfigureSurfaceError::Device(d), + WaitIdleError::WrongSubmissionIndex(..) => unreachable!(), + WaitIdleError::Timeout => ConfigureSurfaceError::GpuWaitTimeout, + } + } +} + +impl WebGpuError for ConfigureSurfaceError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::MissingDownlevelFlags(e) => e.webgpu_error_type(), + Self::InvalidSurface + | Self::InvalidViewFormat(..) + | Self::PreviousOutputExists + | Self::GpuWaitTimeout + | Self::ZeroArea + | Self::TooLarge { .. } + | Self::UnsupportedQueueFamily + | Self::UnsupportedFormat { .. } + | Self::UnsupportedPresentMode { .. } + | Self::UnsupportedAlphaMode { .. } + | Self::UnsupportedUsage { .. } => ErrorType::Validation, + } + } +} + +pub type ResolvedSurfaceOutput = SurfaceOutput>; + +#[repr(C)] +#[derive(Debug)] +pub struct SurfaceOutput { + pub status: Status, + pub texture: Option, +} + +impl Surface { + pub fn get_current_texture(&self) -> Result { + profiling::scope!("Surface::get_current_texture"); + + let (device, config) = if let Some(ref present) = *self.presentation.lock() { + present.device.check_is_valid()?; + (present.device.clone(), present.config.clone()) + } else { + return Err(SurfaceError::NotConfigured); + }; + + let fence = device.fence.read(); + + let suf = self.raw(device.backend()).unwrap(); + let (texture, status) = match unsafe { + suf.acquire_texture( + Some(core::time::Duration::from_millis(FRAME_TIMEOUT_MS as u64)), + fence.as_ref(), + ) + } { + Ok(ast) => { + drop(fence); + + let texture_desc = wgt::TextureDescriptor { + label: hal_label( + Some(alloc::borrow::Cow::Borrowed("")), + device.instance_flags, + ), + size: wgt::Extent3d { + width: config.width, + height: config.height, + depth_or_array_layers: 1, + }, + sample_count: 1, + mip_level_count: 1, + format: config.format, + dimension: wgt::TextureDimension::D2, + usage: config.usage, + view_formats: config.view_formats, + }; + let format_features = wgt::TextureFormatFeatures { + allowed_usages: wgt::TextureUsages::RENDER_ATTACHMENT, + flags: wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4 + | wgt::TextureFormatFeatureFlags::MULTISAMPLE_RESOLVE, + }; + let hal_usage = conv::map_texture_usage( + config.usage, + config.format.into(), + format_features.flags, + ); + let clear_view_desc = hal::TextureViewDescriptor { + label: hal_label( + Some("(wgpu internal) clear surface texture view"), + device.instance_flags, + ), + format: config.format, + dimension: wgt::TextureViewDimension::D2, + usage: wgt::TextureUses::COLOR_TARGET, + range: wgt::ImageSubresourceRange::default(), + }; + let clear_view = unsafe { + device + .raw() + .create_texture_view(ast.texture.as_ref().borrow(), &clear_view_desc) + } + .map_err(|e| device.handle_hal_error(e))?; + + let mut presentation = self.presentation.lock(); + let present = presentation.as_mut().unwrap(); + let texture = resource::Texture::new( + &device, + resource::TextureInner::Surface { raw: ast.texture }, + hal_usage, + &texture_desc, + format_features, + resource::TextureClearMode::Surface { + clear_view: ManuallyDrop::new(clear_view), + }, + true, + ); + + let texture = Arc::new(texture); + + device + .trackers + .lock() + .textures + .insert_single(&texture, wgt::TextureUses::UNINITIALIZED); + + if present.acquired_texture.is_some() { + return Err(SurfaceError::AlreadyAcquired); + } + present.acquired_texture = Some(texture.clone()); + + let status = if ast.suboptimal { + Status::Suboptimal + } else { + Status::Good + }; + (Some(texture), status) + } + Err(err) => ( + None, + match err { + hal::SurfaceError::Timeout => Status::Timeout, + hal::SurfaceError::Occluded => Status::Occluded, + hal::SurfaceError::Lost => Status::Lost, + hal::SurfaceError::Device(err) => { + return Err(device.handle_hal_error(err).into()); + } + hal::SurfaceError::Outdated => Status::Outdated, + hal::SurfaceError::Other(msg) => { + log::error!("acquire error: {msg}"); + Status::Lost + } + }, + ), + }; + + Ok(ResolvedSurfaceOutput { status, texture }) + } + + pub fn present(&self) -> Result { + profiling::scope!("Surface::present"); + + let mut presentation = self.presentation.lock(); + let present = match presentation.as_mut() { + Some(present) => present, + None => return Err(SurfaceError::NotConfigured), + }; + + let device = &present.device; + + device.check_is_valid()?; + let queue = device.get_queue().unwrap(); + + let texture = present + .acquired_texture + .take() + .ok_or(SurfaceError::AlreadyAcquired)?; + + let mut exclusive_snatch_guard = device.snatchable_lock.write(); + let inner = texture.inner.snatch(&mut exclusive_snatch_guard); + drop(exclusive_snatch_guard); + + let result = match inner { + None => return Err(SurfaceError::TextureDestroyed), + Some(resource::TextureInner::Surface { raw }) => { + let raw_surface = self.raw(device.backend()).unwrap(); + let raw_queue = queue.raw(); + let _fence_lock = device.fence.write(); + unsafe { raw_queue.present(raw_surface, raw) } + } + _ => unreachable!(), + }; + + match result { + Ok(()) => Ok(Status::Good), + Err(err) => match err { + hal::SurfaceError::Timeout => Ok(Status::Timeout), + hal::SurfaceError::Occluded => Ok(Status::Occluded), + hal::SurfaceError::Lost => Ok(Status::Lost), + hal::SurfaceError::Device(err) => { + Err(SurfaceError::from(device.handle_hal_error(err))) + } + hal::SurfaceError::Outdated => Ok(Status::Outdated), + hal::SurfaceError::Other(msg) => { + log::error!("present error: {msg}"); + Err(SurfaceError::Invalid) + } + }, + } + } + + pub fn discard(&self) -> Result<(), SurfaceError> { + profiling::scope!("Surface::discard"); + + let mut presentation = self.presentation.lock(); + let present = match presentation.as_mut() { + Some(present) => present, + None => return Err(SurfaceError::NotConfigured), + }; + + let device = &present.device; + + device.check_is_valid()?; + + let texture = present + .acquired_texture + .take() + .ok_or(SurfaceError::AlreadyAcquired)?; + + let mut exclusive_snatch_guard = device.snatchable_lock.write(); + let inner = texture.inner.snatch(&mut exclusive_snatch_guard); + drop(exclusive_snatch_guard); + + match inner { + None => return Err(SurfaceError::TextureDestroyed), + Some(resource::TextureInner::Surface { raw }) => { + let raw_surface = self.raw(device.backend()).unwrap(); + unsafe { raw_surface.discard_texture(raw) }; + } + _ => unreachable!(), + } + + Ok(()) + } +} + +impl Global { + pub fn surface_get_current_texture( + &self, + surface_id: id::SurfaceId, + texture_id_in: Option, + ) -> Result { + let surface = self.surfaces.get(surface_id); + + let fid = self.hub.textures.prepare(texture_id_in); + + let output = surface.get_current_texture()?; + + #[cfg(feature = "trace")] + if let Some(present) = surface.presentation.lock().as_ref() { + if let Some(ref mut trace) = *present.device.trace.lock() { + if let Some(texture) = present.acquired_texture.as_ref() { + trace.add(Action::GetSurfaceTexture { + id: texture.to_trace(), + parent: surface.to_trace(), + }); + } + } + } + + let status = output.status; + let texture_id = output + .texture + .map(|texture| fid.assign(resource::Fallible::Valid(texture))); + + Ok(SurfaceOutput { + status, + texture: texture_id, + }) + } + + pub fn surface_present(&self, surface_id: id::SurfaceId) -> Result { + let surface = self.surfaces.get(surface_id); + + #[cfg(feature = "trace")] + if let Some(present) = surface.presentation.lock().as_ref() { + if let Some(ref mut trace) = *present.device.trace.lock() { + trace.add(Action::Present(surface.to_trace())); + } + } + + surface.present() + } + + pub fn surface_texture_discard(&self, surface_id: id::SurfaceId) -> Result<(), SurfaceError> { + let surface = self.surfaces.get(surface_id); + + #[cfg(feature = "trace")] + if let Some(present) = surface.presentation.lock().as_ref() { + if let Some(ref mut trace) = *present.device.trace.lock() { + trace.add(Action::DiscardSurfaceTexture(surface.to_trace())); + } + } + + surface.discard() + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/ray_tracing.rs b/third_party/wgpu-core-29.0.4-patched/src/ray_tracing.rs new file mode 100644 index 00000000..279884d6 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/ray_tracing.rs @@ -0,0 +1,448 @@ +// Ray tracing +// Major missing optimizations (no api surface changes needed): +// - use custom tracker to track build state +// - no forced rebuilt (build mode deduction) +// - lazy instance buffer allocation +// - maybe share scratch and instance staging buffer allocation +// - partial instance buffer uploads (api surface already designed with this in mind) +// - Batch BLAS read-backs (if it shows up in performance). +// - ([non performance] extract function in build (rust function extraction with guards is a pain)) + +use alloc::{boxed::Box, sync::Arc, vec::Vec}; + +#[cfg(feature = "serde")] +use macro_rules_attribute::apply; +use thiserror::Error; +use wgt::{ + error::{ErrorType, WebGpuError}, + AccelerationStructureGeometryFlags, BufferAddress, IndexFormat, VertexFormat, +}; + +#[cfg(feature = "serde")] +use crate::command::serde_object_reference_struct; +use crate::{ + command::{ArcReferences, EncoderStateError, IdReferences, ReferenceType}, + device::{DeviceError, MissingFeatures}, + id::{BlasId, BufferId, TlasId}, + resource::{ + Blas, BlasCompactCallback, BlasPrepareCompactResult, DestroyedResourceError, + InvalidResourceError, MissingBufferUsageError, ResourceErrorIdent, Tlas, + }, +}; + +#[derive(Clone, Debug, Error)] +pub enum CreateBlasError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error( + "Only one of 'index_count' and 'index_format' was provided (either provide both or none)" + )] + MissingIndexData, + #[error("Provided format was not within allowed formats. Provided format: {0:?}. Allowed formats: {1:?}")] + InvalidVertexFormat(VertexFormat, Vec), + #[error("Limit `max_blas_geometry_count` is {0}, but the BLAS had {1} geometries")] + TooManyGeometries(u32, u32), + #[error( + "Limit `max_blas_primitive_count` is {0}, but the BLAS had a maximum of {1} primitives" + )] + TooManyPrimitives(u32, u32), +} + +impl WebGpuError for CreateBlasError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + Self::MissingIndexData + | Self::InvalidVertexFormat(..) + | Self::TooManyGeometries(..) + | Self::TooManyPrimitives(..) => ErrorType::Validation, + } + } +} + +#[derive(Clone, Debug, Error)] +pub enum CreateTlasError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error("Flag {0:?} is not allowed on a TLAS")] + DisallowedFlag(wgt::AccelerationStructureFlags), + #[error("Limit `max_tlas_instance_count` is {0}, but the TLAS had a maximum of {1} instances")] + TooManyInstances(u32, u32), +} + +impl WebGpuError for CreateTlasError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + Self::DisallowedFlag(..) | Self::TooManyInstances(..) => ErrorType::Validation, + } + } +} + +/// Error encountered while attempting to do a copy on a command encoder. +#[derive(Clone, Debug, Error)] +pub enum BuildAccelerationStructureError { + #[error(transparent)] + EncoderState(#[from] EncoderStateError), + + #[error(transparent)] + Device(#[from] DeviceError), + + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), + + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + + #[error( + "Buffer {0:?} size is insufficient for provided size information (size: {1}, required: {2}" + )] + InsufficientBufferSize(ResourceErrorIdent, u64, u64), + + #[error("Buffer {0:?} associated offset doesn't align with the index type")] + UnalignedIndexBufferOffset(ResourceErrorIdent), + + #[error("Buffer {0:?} associated offset is unaligned")] + UnalignedTransformBufferOffset(ResourceErrorIdent), + + #[error("Buffer {0:?} associated index count not divisible by 3 (count: {1}")] + InvalidIndexCount(ResourceErrorIdent, u32), + + #[error("Buffer {0:?} associated data contains None")] + MissingAssociatedData(ResourceErrorIdent), + + #[error( + "Blas {0:?} build sizes to may be greater than the descriptor at build time specified" + )] + IncompatibleBlasBuildSizes(ResourceErrorIdent), + + #[error("Blas {0:?} flags are different, creation flags: {1:?}, provided: {2:?}")] + IncompatibleBlasFlags( + ResourceErrorIdent, + AccelerationStructureGeometryFlags, + AccelerationStructureGeometryFlags, + ), + + #[error("Blas {0:?} build vertex count is greater than creation count (needs to be less than or equal to), creation: {1:?}, build: {2:?}")] + IncompatibleBlasVertexCount(ResourceErrorIdent, u32, u32), + + #[error("Blas {0:?} vertex formats are different, creation format: {1:?}, provided: {2:?}")] + DifferentBlasVertexFormats(ResourceErrorIdent, VertexFormat, VertexFormat), + + #[error("Blas {0:?} stride was required to be at least {1} but stride given was {2}")] + VertexStrideTooSmall(ResourceErrorIdent, u64, u64), + + #[error("Blas {0:?} stride was required to be a multiple of {1} but stride given was {2}")] + VertexStrideUnaligned(ResourceErrorIdent, u64, u64), + + #[error("Blas {0:?} index count was provided at creation or building, but not the other")] + BlasIndexCountProvidedMismatch(ResourceErrorIdent), + + #[error("Blas {0:?} build index count is greater than creation count (needs to be less than or equal to), creation: {1:?}, build: {2:?}")] + IncompatibleBlasIndexCount(ResourceErrorIdent, u32, u32), + + #[error("Blas {0:?} index formats are different, creation format: {1:?}, provided: {2:?}")] + DifferentBlasIndexFormats(ResourceErrorIdent, Option, Option), + + #[error("Blas {0:?} is compacted and so cannot be built")] + CompactedBlas(ResourceErrorIdent), + + #[error("Blas {0:?} build sizes require index buffer but none was provided")] + MissingIndexBuffer(ResourceErrorIdent), + + #[error( + "Tlas {0:?} an associated instances contains an invalid custom index (more than 24bits)" + )] + TlasInvalidCustomIndex(ResourceErrorIdent), + + #[error( + "Tlas {0:?} has {1} active instances but only {2} are allowed as specified by the descriptor at creation" + )] + TlasInstanceCountExceeded(ResourceErrorIdent, u32, u32), + + #[error("Blas {0:?} has flag USE_TRANSFORM but the transform buffer is missing")] + TransformMissing(ResourceErrorIdent), + + #[error("Blas {0:?} is missing the flag USE_TRANSFORM but the transform buffer is set")] + UseTransformMissing(ResourceErrorIdent), + #[error( + "Tlas {0:?} dependent {1:?} is missing AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN" + )] + TlasDependentMissingVertexReturn(ResourceErrorIdent, ResourceErrorIdent), +} + +impl WebGpuError for BuildAccelerationStructureError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::EncoderState(e) => e.webgpu_error_type(), + Self::Device(e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::DestroyedResource(e) => e.webgpu_error_type(), + Self::MissingBufferUsage(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + Self::InsufficientBufferSize(..) + | Self::UnalignedIndexBufferOffset(..) + | Self::UnalignedTransformBufferOffset(..) + | Self::InvalidIndexCount(..) + | Self::MissingAssociatedData(..) + | Self::IncompatibleBlasBuildSizes(..) + | Self::IncompatibleBlasFlags(..) + | Self::IncompatibleBlasVertexCount(..) + | Self::DifferentBlasVertexFormats(..) + | Self::VertexStrideTooSmall(..) + | Self::VertexStrideUnaligned(..) + | Self::BlasIndexCountProvidedMismatch(..) + | Self::IncompatibleBlasIndexCount(..) + | Self::DifferentBlasIndexFormats(..) + | Self::CompactedBlas(..) + | Self::MissingIndexBuffer(..) + | Self::TlasInvalidCustomIndex(..) + | Self::TlasInstanceCountExceeded(..) + | Self::TransformMissing(..) + | Self::UseTransformMissing(..) + | Self::TlasDependentMissingVertexReturn(..) => ErrorType::Validation, + } + } +} + +#[derive(Clone, Debug, Error)] +pub enum ValidateAsActionsError { + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + + #[error("Tlas {0:?} is used before it is built")] + UsedUnbuiltTlas(ResourceErrorIdent), + + #[error("Blas {0:?} is used before it is built (in Tlas {1:?})")] + UsedUnbuiltBlas(ResourceErrorIdent, ResourceErrorIdent), + + #[error("Blas {0:?} is newer than the containing Tlas {1:?}")] + BlasNewerThenTlas(ResourceErrorIdent, ResourceErrorIdent), +} + +impl WebGpuError for ValidateAsActionsError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::DestroyedResource(e) => e.webgpu_error_type(), + Self::UsedUnbuiltTlas(..) | Self::UsedUnbuiltBlas(..) | Self::BlasNewerThenTlas(..) => { + ErrorType::Validation + } + } + } +} + +#[derive(Debug)] +pub struct BlasTriangleGeometry<'a> { + pub size: &'a wgt::BlasTriangleGeometrySizeDescriptor, + pub vertex_buffer: BufferId, + pub index_buffer: Option, + pub transform_buffer: Option, + pub first_vertex: u32, + pub vertex_stride: BufferAddress, + pub first_index: Option, + pub transform_buffer_offset: Option, +} + +pub enum BlasGeometries<'a> { + TriangleGeometries(Box> + 'a>), +} + +pub struct BlasBuildEntry<'a> { + pub blas_id: BlasId, + pub geometries: BlasGeometries<'a>, +} + +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct TlasBuildEntry { + pub tlas_id: TlasId, + pub instance_buffer_id: BufferId, + pub instance_count: u32, +} + +#[derive(Debug)] +pub struct TlasInstance<'a> { + pub blas_id: BlasId, + pub transform: &'a [f32; 12], + pub custom_data: u32, + pub mask: u8, +} + +pub struct TlasPackage<'a> { + pub tlas_id: TlasId, + pub instances: Box>> + 'a>, + pub lowest_unmodified: u32, +} + +#[derive(Debug, Clone)] +pub(crate) struct TlasBuild { + pub tlas: Arc, + pub dependencies: Vec>, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct AsBuild { + pub blas_s_built: Vec>, + pub tlas_s_built: Vec, +} + +impl AsBuild { + pub(crate) fn with_capacity(blas: usize, tlas: usize) -> Self { + Self { + blas_s_built: Vec::with_capacity(blas), + tlas_s_built: Vec::with_capacity(tlas), + } + } +} + +#[derive(Debug, Clone)] +pub(crate) enum AsAction { + Build(AsBuild), + UseTlas(Arc), +} + +/// Like [`BlasTriangleGeometry`], but with owned data. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", apply(serde_object_reference_struct))] +pub struct OwnedBlasTriangleGeometry { + pub size: wgt::BlasTriangleGeometrySizeDescriptor, + pub vertex_buffer: R::Buffer, + pub index_buffer: Option, + pub transform_buffer: Option, + pub first_vertex: u32, + pub vertex_stride: BufferAddress, + pub first_index: Option, + pub transform_buffer_offset: Option, +} + +pub type ArcBlasTriangleGeometry = OwnedBlasTriangleGeometry; +pub type TraceBlasTriangleGeometry = OwnedBlasTriangleGeometry; + +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", apply(serde_object_reference_struct))] +pub enum OwnedBlasGeometries { + TriangleGeometries(Vec>), +} + +pub type ArcBlasGeometries = OwnedBlasGeometries; +pub type TraceBlasGeometries = OwnedBlasGeometries; + +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", apply(serde_object_reference_struct))] +pub struct OwnedBlasBuildEntry { + pub blas: R::Blas, + pub geometries: OwnedBlasGeometries, +} + +pub type ArcBlasBuildEntry = OwnedBlasBuildEntry; +pub type TraceBlasBuildEntry = OwnedBlasBuildEntry; + +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", apply(serde_object_reference_struct))] +pub struct OwnedTlasInstance { + pub blas: R::Blas, + pub transform: [f32; 12], + pub custom_data: u32, + pub mask: u8, +} + +pub type ArcTlasInstance = OwnedTlasInstance; +pub type TraceTlasInstance = OwnedTlasInstance; + +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", apply(serde_object_reference_struct))] +pub struct OwnedTlasPackage { + pub tlas: R::Tlas, + pub instances: Vec>>, + pub lowest_unmodified: u32, +} + +pub type TraceTlasPackage = OwnedTlasPackage; +pub type ArcTlasPackage = OwnedTlasPackage; + +/// [`BlasTriangleGeometry`], without the resources. +#[derive(Debug, Clone)] +pub struct BlasTriangleGeometryInfo { + pub size: wgt::BlasTriangleGeometrySizeDescriptor, + pub first_vertex: u32, + pub vertex_stride: BufferAddress, + pub first_index: Option, + pub transform_buffer_offset: Option, +} + +#[derive(Clone, Debug, Error)] +pub enum BlasPrepareCompactError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), + #[error("Compaction is already being prepared")] + CompactionPreparingAlready, + #[error("Cannot compact an already compacted BLAS")] + DoubleCompaction, + #[error("BLAS is not yet built")] + NotBuilt, + #[error("BLAS does not support compaction (is AccelerationStructureFlags::ALLOW_COMPACTION missing?)")] + CompactionUnsupported, +} + +impl WebGpuError for BlasPrepareCompactError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::CompactionPreparingAlready + | Self::DoubleCompaction + | Self::NotBuilt + | Self::CompactionUnsupported => ErrorType::Validation, + } + } +} + +#[derive(Clone, Debug, Error)] +pub enum CompactBlasError { + #[error(transparent)] + Encoder(#[from] EncoderStateError), + + #[error(transparent)] + Device(#[from] DeviceError), + + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), + + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + + #[error("BLAS was not prepared for compaction")] + BlasNotReady, +} + +impl WebGpuError for CompactBlasError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Encoder(e) => e.webgpu_error_type(), + Self::Device(e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::DestroyedResource(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + Self::BlasNotReady => ErrorType::Validation, + } + } +} + +pub type BlasCompactReadyPendingClosure = (Option, BlasPrepareCompactResult); diff --git a/third_party/wgpu-core-29.0.4-patched/src/registry.rs b/third_party/wgpu-core-29.0.4-patched/src/registry.rs new file mode 100644 index 00000000..2cdaf3f1 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/registry.rs @@ -0,0 +1,152 @@ +use alloc::sync::Arc; + +use crate::{ + id::Id, + identity::IdentityManager, + lock::{rank, RwLock, RwLockReadGuard}, + storage::{Element, Storage, StorageItem}, +}; + +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct RegistryReport { + pub num_allocated: usize, + pub num_kept_from_user: usize, + pub num_released_from_user: usize, + pub element_size: usize, +} + +impl RegistryReport { + pub fn is_empty(&self) -> bool { + self.num_allocated + self.num_kept_from_user == 0 + } +} + +/// Registry is the primary holder of each resource type +/// Every resource is now arcanized so the last arc released +/// will in the end free the memory and release the inner raw resource +/// +/// Registry act as the main entry point to keep resource alive +/// when created and released from user land code +/// +/// A resource may still be alive when released from user land code +/// if it's used in active submission or anyway kept alive from +/// any other dependent resource +/// +#[derive(Debug)] +pub(crate) struct Registry { + // Must only contain an id which has either never been used or has been released from `storage` + identity: Arc>, + storage: RwLock>, +} + +impl Registry { + pub(crate) fn new() -> Self { + Self { + identity: Arc::new(IdentityManager::new()), + storage: RwLock::new(rank::REGISTRY_STORAGE, Storage::new()), + } + } +} + +#[must_use] +pub(crate) struct FutureId<'a, T: StorageItem> { + id: Id, + data: &'a RwLock>, +} + +impl FutureId<'_, T> { + /// Assign a new resource to this ID. + /// + /// Registers it with the registry. + pub fn assign(self, value: T) -> Id { + let mut data = self.data.write(); + data.insert(self.id, value); + self.id + } +} + +impl Registry { + pub(crate) fn prepare(&self, id_in: Option>) -> FutureId<'_, T> { + FutureId { + id: match id_in { + Some(id_in) => { + self.identity.mark_as_used(id_in); + id_in + } + None => self.identity.process(), + }, + data: &self.storage, + } + } + + #[track_caller] + pub(crate) fn read<'a>(&'a self) -> RwLockReadGuard<'a, Storage> { + self.storage.read() + } + pub(crate) fn remove(&self, id: Id) -> T { + let value = self.storage.write().remove(id); + // This needs to happen *after* removing it from the storage, to maintain the + // invariant that `self.identity` only contains ids which are actually available + // See https://github.com/gfx-rs/wgpu/issues/5372 + self.identity.free(id); + //Returning None is legal if it's an error ID + value + } + + pub(crate) fn generate_report(&self) -> RegistryReport { + let storage = self.storage.read(); + let mut report = RegistryReport { + element_size: size_of::(), + ..Default::default() + }; + report.num_allocated = self.identity.values.lock().count(); + for element in storage.map.iter() { + match *element { + Element::Occupied(..) => report.num_kept_from_user += 1, + Element::Vacant => report.num_released_from_user += 1, + } + } + report + } +} + +impl Registry { + pub(crate) fn get(&self, id: Id) -> T { + self.read().get(id) + } +} + +#[cfg(test)] +mod tests { + use super::Registry; + use crate::{id::Marker, resource::ResourceType, storage::StorageItem}; + use alloc::sync::Arc; + + struct TestData; + struct TestDataId; + impl Marker for TestDataId {} + + impl ResourceType for TestData { + const TYPE: &'static str = "TestData"; + } + impl StorageItem for TestData { + type Marker = TestDataId; + } + + #[test] + fn simultaneous_registration() { + let registry = Registry::new(); + std::thread::scope(|s| { + for _ in 0..5 { + s.spawn(|| { + for _ in 0..1000 { + let value = Arc::new(TestData); + let new_id = registry.prepare(None); + let id = new_id.assign(value); + registry.remove(id); + } + }); + } + }) + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/resource.rs b/third_party/wgpu-core-29.0.4-patched/src/resource.rs new file mode 100644 index 00000000..e609b968 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/resource.rs @@ -0,0 +1,2427 @@ +use alloc::{borrow::Cow, borrow::ToOwned as _, boxed::Box, string::String, sync::Arc, vec::Vec}; +use core::{ + borrow::Borrow, + fmt, + mem::{self, size_of, ManuallyDrop}, + num::NonZeroU64, + ops::Range, + ptr::NonNull, +}; +use smallvec::SmallVec; +use thiserror::Error; +use wgt::{ + error::{ErrorType, WebGpuError}, + TextureSelector, +}; + +#[cfg(feature = "trace")] +use crate::device::trace; +use crate::{ + binding_model::{BindGroup, BindingError}, + device::{ + queue, resource::DeferredDestroy, BufferMapPendingClosure, Device, DeviceError, + DeviceMismatch, HostMap, MissingDownlevelFlags, MissingFeatures, + }, + hal_label, + init_tracker::{BufferInitTracker, TextureInitTracker}, + lock::{rank, Mutex, RwLock}, + ray_tracing::{BlasCompactReadyPendingClosure, BlasPrepareCompactError}, + resource_log, + snatch::{SnatchGuard, Snatchable}, + timestamp_normalization::TimestampNormalizationBindGroup, + track::{SharedTrackerIndexAllocator, TrackerIndex}, + weak_vec::WeakVec, + Label, LabelHelpers, SubmissionIndex, +}; + +/// Information about the wgpu-core resource. +/// +/// Each type representing a `wgpu-core` resource, like [`Device`], +/// [`Buffer`], etc., contains a `ResourceInfo` which contains +/// its latest submission index and label. +/// +/// A resource may need to be retained for any of several reasons: +/// and any lifetime logic will be handled by `Arc` refcount +/// +/// - The user may hold a reference to it (via a `wgpu::Buffer`, say). +/// +/// - Other resources may depend on it (a texture view's backing +/// texture, for example). +/// +/// - It may be used by commands sent to the GPU that have not yet +/// finished execution. +/// +/// [`Device`]: crate::device::resource::Device +/// [`Buffer`]: crate::resource::Buffer +#[derive(Debug)] +pub(crate) struct TrackingData { + tracker_index: TrackerIndex, + tracker_indices: Arc, +} + +impl Drop for TrackingData { + fn drop(&mut self) { + self.tracker_indices.free(self.tracker_index); + } +} + +impl TrackingData { + pub(crate) fn new(tracker_indices: Arc) -> Self { + Self { + tracker_index: tracker_indices.alloc(), + tracker_indices, + } + } + + pub(crate) fn tracker_index(&self) -> TrackerIndex { + self.tracker_index + } +} + +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ResourceErrorIdent { + r#type: Cow<'static, str>, + label: String, +} + +impl fmt::Display for ResourceErrorIdent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + write!(f, "{} with '{}' label", self.r#type, self.label) + } +} + +pub trait ParentDevice: Labeled { + fn device(&self) -> &Arc; + + fn is_equal(self: &Arc, other: &Arc) -> bool { + Arc::ptr_eq(self, other) + } + + fn same_device_as(&self, other: &O) -> Result<(), DeviceError> { + if Arc::ptr_eq(self.device(), other.device()) { + Ok(()) + } else { + Err(DeviceError::DeviceMismatch(Box::new(DeviceMismatch { + res: self.error_ident(), + res_device: self.device().error_ident(), + target: Some(other.error_ident()), + target_device: other.device().error_ident(), + }))) + } + } + + fn same_device(&self, device: &Device) -> Result<(), DeviceError> { + if core::ptr::eq(&**self.device(), device) { + Ok(()) + } else { + Err(DeviceError::DeviceMismatch(Box::new(DeviceMismatch { + res: self.error_ident(), + res_device: self.device().error_ident(), + target: None, + target_device: device.error_ident(), + }))) + } + } +} + +#[macro_export] +macro_rules! impl_parent_device { + ($ty:ident) => { + impl $crate::resource::ParentDevice for $ty { + fn device(&self) -> &Arc { + &self.device + } + } + }; +} + +/// Allow access to the hal resource as guarded by the `SnatchGuard`. +pub trait RawResourceAccess: ParentDevice { + type DynResource: hal::DynResource + ?Sized; + + /// Get access to the raw resource if it is not destroyed. + /// + /// Returns `None` if the resource has been destroyed. This method + /// does not allocate in either case. + fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource>; + + /// Get access to the raw resource if it is not destroyed. + /// + /// Returns a full error if the resource has been destroyed. This + /// method allocates a label in the error case. + fn try_raw<'a>( + &'a self, + guard: &'a SnatchGuard, + ) -> Result<&'a Self::DynResource, DestroyedResourceError> { + self.raw(guard) + .ok_or_else(|| DestroyedResourceError(self.error_ident())) + } +} + +pub trait ResourceType { + const TYPE: &'static str; +} + +#[macro_export] +macro_rules! impl_resource_type { + ($ty:ident) => { + impl $crate::resource::ResourceType for $ty { + const TYPE: &'static str = stringify!($ty); + } + }; +} + +pub trait Labeled: ResourceType { + /// Returns a string identifying this resource for logging and errors. + /// + /// It may be a user-provided string or it may be a placeholder from wgpu. + /// + /// It is non-empty unless the user-provided string was empty. + fn label(&self) -> &str; + + fn error_ident(&self) -> ResourceErrorIdent { + ResourceErrorIdent { + r#type: Cow::Borrowed(Self::TYPE), + label: self.label().to_owned(), + } + } +} + +#[macro_export] +macro_rules! impl_labeled { + ($ty:ident) => { + impl $crate::resource::Labeled for $ty { + fn label(&self) -> &str { + &self.label + } + } + }; +} + +pub(crate) trait Trackable { + fn tracker_index(&self) -> TrackerIndex; +} + +#[macro_export] +macro_rules! impl_trackable { + ($ty:ident) => { + impl $crate::resource::Trackable for $ty { + fn tracker_index(&self) -> $crate::track::TrackerIndex { + self.tracking_data.tracker_index() + } + } + }; +} + +#[derive(Debug)] +pub(crate) enum BufferMapState { + /// Mapped at creation. + Init { staging_buffer: StagingBuffer }, + /// Waiting for GPU to be done before mapping + Waiting(BufferPendingMapping), + /// Mapped + Active { + mapping: hal::BufferMapping, + range: hal::MemoryRange, + host: HostMap, + }, + /// Not mapped + Idle, +} + +#[cfg(send_sync)] +unsafe impl Send for BufferMapState {} +#[cfg(send_sync)] +unsafe impl Sync for BufferMapState {} + +#[cfg(send_sync)] +pub type BufferMapCallback = Box; +#[cfg(not(send_sync))] +pub type BufferMapCallback = Box; + +pub struct BufferMapOperation { + pub host: HostMap, + pub callback: Option, +} + +impl fmt::Debug for BufferMapOperation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BufferMapOperation") + .field("host", &self.host) + .field("callback", &self.callback.as_ref().map(|_| "?")) + .finish() + } +} + +#[derive(Clone, Debug, Error)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[non_exhaustive] +pub enum BufferAccessError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Buffer map failed")] + Failed, + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + #[error("Buffer is already mapped")] + AlreadyMapped, + #[error("Buffer map is pending")] + MapAlreadyPending, + #[error(transparent)] + MissingBufferUsage(#[from] MissingBufferUsageError), + #[error("Buffer is not mapped")] + NotMapped, + #[error( + "Buffer map range must start aligned to `MAP_ALIGNMENT` and end to `COPY_BUFFER_ALIGNMENT`" + )] + UnalignedRange, + #[error("Buffer offset invalid: offset {offset} must be multiple of 8")] + UnalignedOffset { offset: wgt::BufferAddress }, + #[error("Buffer range size invalid: range_size {range_size} must be multiple of 4")] + UnalignedRangeSize { range_size: wgt::BufferAddress }, + #[error("Buffer access out of bounds: index {index} would underrun the buffer (limit: {min})")] + OutOfBoundsStartOffsetUnderrun { + index: wgt::BufferAddress, + min: wgt::BufferAddress, + }, + #[error( + "Buffer access out of bounds: start offset {index} would overrun the buffer (limit: {max})" + )] + OutOfBoundsStartOffsetOverrun { + index: wgt::BufferAddress, + max: wgt::BufferAddress, + }, + #[error( + "Buffer access out of bounds: start offset {index} + size {size} would overrun the buffer (limit: {max})" + )] + OutOfBoundsEndOffsetOverrun { + index: wgt::BufferAddress, + size: wgt::BufferAddress, + max: wgt::BufferAddress, + }, + #[error("Buffer map aborted")] + MapAborted, + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), + #[error("Map start offset ({offset}) is out-of-bounds for buffer of size {buffer_size}")] + MapStartOffsetOverrun { + offset: wgt::BufferAddress, + buffer_size: wgt::BufferAddress, + }, + #[error( + "Map end offset (start at {} + size of {}) is out-of-bounds for buffer of size {}", + offset, + size, + buffer_size + )] + MapEndOffsetOverrun { + offset: wgt::BufferAddress, + size: wgt::BufferAddress, + buffer_size: wgt::BufferAddress, + }, +} + +impl WebGpuError for BufferAccessError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::DestroyedResource(e) => e.webgpu_error_type(), + + Self::Failed + | Self::AlreadyMapped + | Self::MapAlreadyPending + | Self::MissingBufferUsage(_) + | Self::NotMapped + | Self::UnalignedRange + | Self::UnalignedOffset { .. } + | Self::UnalignedRangeSize { .. } + | Self::OutOfBoundsStartOffsetUnderrun { .. } + | Self::OutOfBoundsStartOffsetOverrun { .. } + | Self::OutOfBoundsEndOffsetOverrun { .. } + | Self::MapAborted + | Self::MapStartOffsetOverrun { .. } + | Self::MapEndOffsetOverrun { .. } => ErrorType::Validation, + } + } +} + +#[derive(Clone, Debug, Error)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[error("Usage flags {actual:?} of {res} do not contain required usage flags {expected:?}")] +pub struct MissingBufferUsageError { + pub(crate) res: ResourceErrorIdent, + pub(crate) actual: wgt::BufferUsages, + pub(crate) expected: wgt::BufferUsages, +} + +impl WebGpuError for MissingBufferUsageError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +#[derive(Clone, Debug, Error)] +#[error("Usage flags {actual:?} of {res} do not contain required usage flags {expected:?}")] +pub struct MissingTextureUsageError { + pub(crate) res: ResourceErrorIdent, + pub(crate) actual: wgt::TextureUsages, + pub(crate) expected: wgt::TextureUsages, +} + +impl WebGpuError for MissingTextureUsageError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +#[derive(Clone, Debug, Error)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[error("{0} has been destroyed")] +pub struct DestroyedResourceError(pub ResourceErrorIdent); + +impl WebGpuError for DestroyedResourceError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +#[derive(Clone, Debug, Error)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[error("{0} is invalid")] +pub struct InvalidResourceError(pub ResourceErrorIdent); + +impl WebGpuError for InvalidResourceError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +pub enum Fallible { + Valid(Arc), + Invalid(Arc), +} + +impl Fallible { + pub fn get(self) -> Result, InvalidResourceError> { + match self { + Fallible::Valid(v) => Ok(v), + Fallible::Invalid(label) => Err(InvalidResourceError(ResourceErrorIdent { + r#type: Cow::Borrowed(T::TYPE), + label: (*label).clone(), + })), + } + } +} + +impl Clone for Fallible { + fn clone(&self) -> Self { + match self { + Self::Valid(v) => Self::Valid(v.clone()), + Self::Invalid(l) => Self::Invalid(l.clone()), + } + } +} + +impl ResourceType for Fallible { + const TYPE: &'static str = T::TYPE; +} + +impl crate::storage::StorageItem for Fallible { + type Marker = T::Marker; +} + +pub type BufferAccessResult = Result<(), BufferAccessError>; + +#[derive(Debug)] +pub(crate) struct BufferPendingMapping { + pub(crate) range: Range, + pub(crate) op: BufferMapOperation, + // hold the parent alive while the mapping is active + pub(crate) _parent_buffer: Arc, +} + +pub type BufferDescriptor<'a> = wgt::BufferDescriptor>; + +#[derive(Debug)] +pub struct Buffer { + pub(crate) raw: Snatchable>, + pub(crate) device: Arc, + pub(crate) usage: wgt::BufferUsages, + pub(crate) size: wgt::BufferAddress, + pub(crate) initialization_status: RwLock, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, + pub(crate) map_state: Mutex, + pub(crate) bind_groups: Mutex>, + pub(crate) timestamp_normalization_bind_group: Snatchable, + pub(crate) indirect_validation_bind_groups: Snatchable, +} + +impl Drop for Buffer { + fn drop(&mut self) { + if let Some(raw) = self.timestamp_normalization_bind_group.take() { + raw.dispose(self.device.raw()); + } + + if let Some(raw) = self.indirect_validation_bind_groups.take() { + raw.dispose(self.device.raw()); + } + + if let Some(raw) = self.raw.take() { + resource_log!("Destroy raw {}", self.error_ident()); + unsafe { + self.device.raw().destroy_buffer(raw); + } + } + } +} + +impl RawResourceAccess for Buffer { + type DynResource = dyn hal::DynBuffer; + + fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> { + self.raw.get(guard).map(|b| b.as_ref()) + } +} + +impl Buffer { + pub(crate) fn check_destroyed( + &self, + guard: &SnatchGuard, + ) -> Result<(), DestroyedResourceError> { + self.raw + .get(guard) + .map(|_| ()) + .ok_or_else(|| DestroyedResourceError(self.error_ident())) + } + + /// Checks that the given buffer usage contains the required buffer usage, + /// returns an error otherwise. + pub(crate) fn check_usage( + &self, + expected: wgt::BufferUsages, + ) -> Result<(), MissingBufferUsageError> { + if self.usage.contains(expected) { + Ok(()) + } else { + Err(MissingBufferUsageError { + res: self.error_ident(), + actual: self.usage, + expected, + }) + } + } + + /// Resolve the size of a binding for buffer with `offset` and `size`. + /// + /// If `size` is `None`, then the remainder of the buffer starting from + /// `offset` is used. + /// + /// If the binding would overflow the buffer, then an error is returned. + /// + /// Zero-size bindings are permitted here for historical reasons. Although + /// zero-size bindings are permitted by WebGPU, they are not permitted by + /// some backends. See [`Buffer::binding`] and + /// [#3170](https://github.com/gfx-rs/wgpu/issues/3170). + pub fn resolve_binding_size( + &self, + offset: wgt::BufferAddress, + binding_size: Option, + ) -> Result { + let buffer_size = self.size; + + match binding_size { + Some(binding_size) => match offset.checked_add(binding_size.get()) { + Some(end) if end <= buffer_size => Ok(binding_size.get()), + _ => Err(BindingError::BindingRangeTooLarge { + buffer: self.error_ident(), + offset, + binding_size: binding_size.get(), + buffer_size, + }), + }, + None => { + buffer_size + .checked_sub(offset) + .ok_or_else(|| BindingError::BindingOffsetTooLarge { + buffer: self.error_ident(), + offset, + buffer_size, + }) + } + } + } + + /// Create a new [`hal::BufferBinding`] for the buffer with `offset` and + /// `binding_size`. + /// + /// If `binding_size` is `None`, then the remainder of the buffer starting + /// from `offset` is used. + /// + /// If the binding would overflow the buffer, then an error is returned. + /// + /// A zero-size binding at the end of the buffer is permitted here for historical reasons. Although + /// zero-size bindings are permitted by WebGPU, they are not permitted by + /// some backends. The zero-size binding need to be quashed or remapped to a + /// non-zero size, either universally in wgpu-core, or in specific backends + /// that do not support them. See + /// [#3170](https://github.com/gfx-rs/wgpu/issues/3170). + /// + /// Although it seems like it would be simpler and safer to use the resolved + /// size in the returned [`hal::BufferBinding`], doing this (and removing + /// redundant logic in backends to resolve the implicit size) was observed + /// to cause problems in certain CTS tests, so an implicit size + /// specification is preserved in the output. + pub fn binding<'a>( + &'a self, + offset: wgt::BufferAddress, + binding_size: Option, + snatch_guard: &'a SnatchGuard, + ) -> Result<(hal::BufferBinding<'a, dyn hal::DynBuffer>, u64), BindingError> { + let buf_raw = self.try_raw(snatch_guard)?; + let resolved_size = self.resolve_binding_size(offset, binding_size)?; + // SAFETY: The offset and size passed to hal::BufferBinding::new_unchecked must + // define a binding contained within the buffer. + Ok(( + hal::BufferBinding::new_unchecked(buf_raw, offset, binding_size), + resolved_size, + )) + } + + /// Returns the mapping callback in case of error so that the callback can be fired outside + /// of the locks that are held in this function. + pub fn map_async( + self: &Arc, + offset: wgt::BufferAddress, + size: Option, + op: BufferMapOperation, + ) -> Result { + let range_size = if let Some(size) = size { + size + } else { + self.size.saturating_sub(offset) + }; + + if !offset.is_multiple_of(wgt::MAP_ALIGNMENT) { + return Err((op, BufferAccessError::UnalignedOffset { offset })); + } + if !range_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) { + return Err((op, BufferAccessError::UnalignedRangeSize { range_size })); + } + + if offset > self.size { + return Err(( + op, + BufferAccessError::MapStartOffsetOverrun { + offset, + buffer_size: self.size, + }, + )); + } + // NOTE: Should never underflow because of our earlier check. + if range_size > self.size - offset { + return Err(( + op, + BufferAccessError::MapEndOffsetOverrun { + offset, + size: range_size, + buffer_size: self.size, + }, + )); + } + let end_offset = offset + range_size; + + if !offset.is_multiple_of(wgt::MAP_ALIGNMENT) + || !end_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) + { + return Err((op, BufferAccessError::UnalignedRange)); + } + + let (pub_usage, internal_use) = match op.host { + HostMap::Read => (wgt::BufferUsages::MAP_READ, wgt::BufferUses::MAP_READ), + HostMap::Write => (wgt::BufferUsages::MAP_WRITE, wgt::BufferUses::MAP_WRITE), + }; + + if let Err(e) = self.check_usage(pub_usage) { + return Err((op, e.into())); + } + + let device = &self.device; + if let Err(e) = device.check_is_valid() { + return Err((op, e.into())); + } + + { + let snatch_guard = device.snatchable_lock.read(); + if let Err(e) = self.check_destroyed(&snatch_guard) { + return Err((op, e.into())); + } + } + + { + let map_state = &mut *self.map_state.lock(); + *map_state = match *map_state { + BufferMapState::Init { .. } | BufferMapState::Active { .. } => { + return Err((op, BufferAccessError::AlreadyMapped)); + } + BufferMapState::Waiting(_) => { + return Err((op, BufferAccessError::MapAlreadyPending)); + } + BufferMapState::Idle => BufferMapState::Waiting(BufferPendingMapping { + range: offset..end_offset, + op, + _parent_buffer: self.clone(), + }), + }; + } + + // TODO: we are ignoring the transition here, I think we need to add a barrier + // at the end of the submission + device + .trackers + .lock() + .buffers + .set_single(self, internal_use); + + let submit_index = if let Some(queue) = device.get_queue() { + queue.lock_life().map(self).unwrap_or(0) // '0' means no wait is necessary + } else { + // We can safely unwrap below since we just set the `map_state` to `BufferMapState::Waiting`. + let (mut operation, status) = self.map(&device.snatchable_lock.read()).unwrap(); + if let Some(callback) = operation.callback.take() { + callback(status); + } + 0 + }; + + Ok(submit_index) + } + + pub fn get_mapped_range( + self: &Arc, + offset: wgt::BufferAddress, + size: Option, + ) -> Result<(NonNull, u64), BufferAccessError> { + { + let snatch_guard = self.device.snatchable_lock.read(); + self.check_destroyed(&snatch_guard)?; + } + + let range_size = if let Some(size) = size { + size + } else { + self.size.saturating_sub(offset) + }; + + if !offset.is_multiple_of(wgt::MAP_ALIGNMENT) { + return Err(BufferAccessError::UnalignedOffset { offset }); + } + if !range_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) { + return Err(BufferAccessError::UnalignedRangeSize { range_size }); + } + let map_state = &*self.map_state.lock(); + match *map_state { + BufferMapState::Init { ref staging_buffer } => { + if offset > self.size { + return Err(BufferAccessError::MapStartOffsetOverrun { + offset, + buffer_size: self.size, + }); + } + // NOTE: Should never underflow because of our earlier check. + if range_size > self.size - offset { + return Err(BufferAccessError::MapEndOffsetOverrun { + offset, + size: range_size, + buffer_size: self.size, + }); + } + let ptr = unsafe { staging_buffer.ptr() }; + let ptr = unsafe { NonNull::new_unchecked(ptr.as_ptr().offset(offset as isize)) }; + Ok((ptr, range_size)) + } + BufferMapState::Active { + ref mapping, + ref range, + .. + } => { + if offset > range.end { + return Err(BufferAccessError::OutOfBoundsStartOffsetOverrun { + index: offset, + max: range.end, + }); + } + if offset < range.start { + return Err(BufferAccessError::OutOfBoundsStartOffsetUnderrun { + index: offset, + min: range.start, + }); + } + if range_size > range.end - offset { + return Err(BufferAccessError::OutOfBoundsEndOffsetOverrun { + index: offset, + size: range_size, + max: range.end, + }); + } + // ptr points to the beginning of the range we mapped in map_async + // rather than the beginning of the buffer. + let relative_offset = (offset - range.start) as isize; + unsafe { + Ok(( + NonNull::new_unchecked(mapping.ptr.as_ptr().offset(relative_offset)), + range_size, + )) + } + } + BufferMapState::Idle | BufferMapState::Waiting(_) => Err(BufferAccessError::NotMapped), + } + } + /// This function returns [`None`] only if [`Self::map_state`] is not [`BufferMapState::Waiting`]. + #[must_use] + pub(crate) fn map(&self, snatch_guard: &SnatchGuard) -> Option { + // This _cannot_ be inlined into the match. If it is, the lock will be held + // open through the whole match, resulting in a deadlock when we try to re-lock + // the buffer back to active. + let mapping = mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle); + let pending_mapping = match mapping { + BufferMapState::Waiting(pending_mapping) => pending_mapping, + // Mapping cancelled + BufferMapState::Idle => return None, + // Mapping queued at least twice by map -> unmap -> map + // and was already successfully mapped below + BufferMapState::Active { .. } => { + *self.map_state.lock() = mapping; + return None; + } + _ => panic!("No pending mapping."), + }; + let status = if pending_mapping.range.start != pending_mapping.range.end { + let host = pending_mapping.op.host; + let size = pending_mapping.range.end - pending_mapping.range.start; + match crate::device::map_buffer( + self, + pending_mapping.range.start, + size, + host, + snatch_guard, + ) { + Ok(mapping) => { + *self.map_state.lock() = BufferMapState::Active { + mapping, + range: pending_mapping.range.clone(), + host, + }; + Ok(()) + } + Err(e) => Err(e), + } + } else { + *self.map_state.lock() = BufferMapState::Active { + mapping: hal::BufferMapping { + ptr: NonNull::dangling(), + is_coherent: true, + }, + range: pending_mapping.range, + host: pending_mapping.op.host, + }; + Ok(()) + }; + Some((pending_mapping.op, status)) + } + + // Note: This must not be called while holding a lock. + pub fn unmap(self: &Arc) -> Result<(), BufferAccessError> { + if let Some((mut operation, status)) = self.unmap_inner()? { + if let Some(callback) = operation.callback.take() { + callback(status); + } + } + + Ok(()) + } + + fn unmap_inner(self: &Arc) -> Result, BufferAccessError> { + let device = &self.device; + let snatch_guard = device.snatchable_lock.read(); + let raw_buf = self.try_raw(&snatch_guard)?; + match mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle) { + BufferMapState::Init { staging_buffer } => { + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + use crate::device::trace::{DataKind, IntoTrace}; + + let data = trace.make_binary(DataKind::Bin, staging_buffer.get_data()); + trace.add(trace::Action::WriteBuffer { + id: self.to_trace(), + data, + // NOTE: `self.size` here corresponds to `data`'s actual length. + offset: 0, + size: self.size, + queued: true, + }); + } + + let staging_buffer = staging_buffer.flush(); + + if let Some(queue) = device.get_queue() { + let region = wgt::BufferSize::new(self.size).map(|size| hal::BufferCopy { + src_offset: 0, + dst_offset: 0, + size, + }); + let transition_src = hal::BufferBarrier { + buffer: staging_buffer.raw(), + usage: hal::StateTransition { + from: wgt::BufferUses::MAP_WRITE, + to: wgt::BufferUses::COPY_SRC, + }, + }; + let transition_dst = hal::BufferBarrier:: { + buffer: raw_buf, + usage: hal::StateTransition { + from: wgt::BufferUses::empty(), + to: wgt::BufferUses::COPY_DST, + }, + }; + let mut pending_writes = queue.pending_writes.lock(); + let encoder = pending_writes.activate(); + unsafe { + encoder.transition_buffers(&[transition_src, transition_dst]); + if self.size > 0 { + encoder.copy_buffer_to_buffer( + staging_buffer.raw(), + raw_buf, + region.as_slice(), + ); + } + } + pending_writes.consume(staging_buffer); + pending_writes.insert_buffer(self); + } + } + BufferMapState::Idle => { + return Err(BufferAccessError::NotMapped); + } + BufferMapState::Waiting(pending) => { + return Ok(Some((pending.op, Err(BufferAccessError::MapAborted)))); + } + BufferMapState::Active { + mapping, + range, + host, + } => { + if host == HostMap::Write { + #[cfg(feature = "trace")] + if let Some(ref mut trace) = *device.trace.lock() { + use crate::device::trace::{DataKind, IntoTrace}; + + let size = range.end - range.start; + let data = trace.make_binary(DataKind::Bin, unsafe { + core::slice::from_raw_parts(mapping.ptr.as_ptr(), size as usize) + }); + trace.add(trace::Action::WriteBuffer { + id: self.to_trace(), + data, + offset: range.start, + size, + queued: false, + }); + } + if !mapping.is_coherent { + unsafe { device.raw().flush_mapped_ranges(raw_buf, &[range]) }; + } + } + unsafe { device.raw().unmap_buffer(raw_buf) }; + } + } + Ok(None) + } + + pub fn destroy(self: &Arc) { + let device = &self.device; + + let temp = { + let mut snatch_guard = device.snatchable_lock.write(); + + let raw = match self.raw.snatch(&mut snatch_guard) { + Some(raw) => raw, + None => { + // Per spec, it is valid to call `destroy` multiple times. + return; + } + }; + + let timestamp_normalization_bind_group = self + .timestamp_normalization_bind_group + .snatch(&mut snatch_guard); + + let indirect_validation_bind_groups = self + .indirect_validation_bind_groups + .snatch(&mut snatch_guard); + + drop(snatch_guard); + + let bind_groups = { + let mut guard = self.bind_groups.lock(); + mem::take(&mut *guard) + }; + + queue::TempResource::DestroyedBuffer(DestroyedBuffer { + raw: ManuallyDrop::new(raw), + device: Arc::clone(&self.device), + label: self.label().to_owned(), + bind_groups, + timestamp_normalization_bind_group, + indirect_validation_bind_groups, + }) + }; + + if let Some(queue) = device.get_queue() { + let mut pending_writes = queue.pending_writes.lock(); + if pending_writes.contains_buffer(self) { + pending_writes.consume_temp(temp); + } else { + let mut life_lock = queue.lock_life(); + let last_submit_index = life_lock.get_buffer_latest_submission_index(self); + if let Some(last_submit_index) = last_submit_index { + life_lock.schedule_resource_destruction(temp, last_submit_index); + } + } + } + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateBufferError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Failed to map buffer while creating: {0}")] + AccessError(#[from] BufferAccessError), + #[error("Buffers that are mapped at creation have to be aligned to `COPY_BUFFER_ALIGNMENT`")] + UnalignedSize, + #[error("Invalid usage flags {0:?}")] + InvalidUsage(wgt::BufferUsages), + #[error("`MAP` usage can only be combined with the opposite `COPY`, requested {0:?}")] + UsageMismatch(wgt::BufferUsages), + #[error("Buffer size {requested} is greater than the maximum buffer size ({maximum})")] + MaxBufferSize { requested: u64, maximum: u64 }, + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error("Failed to create bind group for indirect buffer validation: {0}")] + IndirectValidationBindGroup(DeviceError), +} + +crate::impl_resource_type!(Buffer); +crate::impl_labeled!(Buffer); +crate::impl_parent_device!(Buffer); +crate::impl_storage_item!(Buffer); +crate::impl_trackable!(Buffer); + +impl WebGpuError for CreateBufferError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::AccessError(e) => e.webgpu_error_type(), + Self::MissingDownlevelFlags(e) => e.webgpu_error_type(), + Self::IndirectValidationBindGroup(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + + Self::UnalignedSize + | Self::InvalidUsage(_) + | Self::UsageMismatch(_) + | Self::MaxBufferSize { .. } => ErrorType::Validation, + } + } +} + +/// A buffer that has been marked as destroyed and is staged for actual deletion soon. +#[derive(Debug)] +pub struct DestroyedBuffer { + raw: ManuallyDrop>, + device: Arc, + label: String, + bind_groups: WeakVec, + timestamp_normalization_bind_group: Option, + indirect_validation_bind_groups: Option, +} + +impl DestroyedBuffer { + pub fn label(&self) -> &dyn fmt::Debug { + &self.label + } +} + +impl Drop for DestroyedBuffer { + fn drop(&mut self) { + let mut deferred = self.device.deferred_destroy.lock(); + deferred.push(DeferredDestroy::BindGroups(mem::take( + &mut self.bind_groups, + ))); + drop(deferred); + + if let Some(raw) = self.timestamp_normalization_bind_group.take() { + raw.dispose(self.device.raw()); + } + + if let Some(raw) = self.indirect_validation_bind_groups.take() { + raw.dispose(self.device.raw()); + } + + resource_log!("Destroy raw Buffer (destroyed) {:?}", self.label()); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { + hal::DynDevice::destroy_buffer(self.device.raw(), raw); + } + } +} + +#[cfg(send_sync)] +unsafe impl Send for StagingBuffer {} +#[cfg(send_sync)] +unsafe impl Sync for StagingBuffer {} + +/// A temporary buffer, consumed by the command that uses it. +/// +/// A [`StagingBuffer`] is designed for one-shot uploads of data to the GPU. It +/// is always created mapped, and the command that uses it destroys the buffer +/// when it is done. +/// +/// [`StagingBuffer`]s can be created with [`queue_create_staging_buffer`] and +/// used with [`queue_write_staging_buffer`]. They are also used internally by +/// operations like [`queue_write_texture`] that need to upload data to the GPU, +/// but that don't belong to any particular wgpu command buffer. +/// +/// Used `StagingBuffer`s are accumulated in [`Device::pending_writes`], to be +/// freed once their associated operation's queue submission has finished +/// execution. +/// +/// [`queue_create_staging_buffer`]: crate::global::Global::queue_create_staging_buffer +/// [`queue_write_staging_buffer`]: crate::global::Global::queue_write_staging_buffer +/// [`queue_write_texture`]: crate::global::Global::queue_write_texture +/// [`Device::pending_writes`]: crate::device::Device +#[derive(Debug)] +pub struct StagingBuffer { + raw: Box, + device: Arc, + pub(crate) size: wgt::BufferSize, + is_coherent: bool, + ptr: NonNull, +} + +impl StagingBuffer { + pub(crate) fn new(device: &Arc, size: wgt::BufferSize) -> Result { + profiling::scope!("StagingBuffer::new"); + let stage_desc = hal::BufferDescriptor { + label: hal_label(Some("(wgpu internal) Staging"), device.instance_flags), + size: size.get(), + usage: wgt::BufferUses::MAP_WRITE | wgt::BufferUses::COPY_SRC, + memory_flags: hal::MemoryFlags::TRANSIENT, + }; + + let raw = unsafe { device.raw().create_buffer(&stage_desc) } + .map_err(|e| device.handle_hal_error(e))?; + let mapping = unsafe { device.raw().map_buffer(raw.as_ref(), 0..size.get()) } + .map_err(|e| device.handle_hal_error(e))?; + + let staging_buffer = StagingBuffer { + raw, + device: device.clone(), + size, + is_coherent: mapping.is_coherent, + ptr: mapping.ptr, + }; + + Ok(staging_buffer) + } + + /// SAFETY: You must not call any functions of `self` + /// until you stopped using the returned pointer. + pub(crate) unsafe fn ptr(&self) -> NonNull { + self.ptr + } + + #[cfg(feature = "trace")] + pub(crate) fn get_data(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.size.get() as usize) } + } + + pub(crate) fn write_zeros(&mut self) { + unsafe { core::ptr::write_bytes(self.ptr.as_ptr(), 0, self.size.get() as usize) }; + } + + pub(crate) fn write(&mut self, data: &[u8]) { + assert!(data.len() >= self.size.get() as usize); + // SAFETY: With the assert above, all of `copy_nonoverlapping`'s + // requirements are satisfied. + unsafe { + core::ptr::copy_nonoverlapping( + data.as_ptr(), + self.ptr.as_ptr(), + self.size.get() as usize, + ); + } + } + + /// SAFETY: The offsets and size must be in-bounds. + pub(crate) unsafe fn write_with_offset( + &mut self, + data: &[u8], + src_offset: isize, + dst_offset: isize, + size: usize, + ) { + unsafe { + debug_assert!( + (src_offset + size as isize) as usize <= data.len(), + "src_offset + size must be in-bounds: src_offset = {}, size = {}, data.len() = {}", + src_offset, + size, + data.len() + ); + core::ptr::copy_nonoverlapping( + data.as_ptr().offset(src_offset), + self.ptr.as_ptr().offset(dst_offset), + size, + ); + } + } + + pub(crate) fn flush(self) -> FlushedStagingBuffer { + let device = self.device.raw(); + if !self.is_coherent { + #[allow(clippy::single_range_in_vec_init)] + unsafe { + device.flush_mapped_ranges(self.raw.as_ref(), &[0..self.size.get()]) + }; + } + unsafe { device.unmap_buffer(self.raw.as_ref()) }; + + let StagingBuffer { + raw, device, size, .. + } = self; + + FlushedStagingBuffer { + raw: ManuallyDrop::new(raw), + device, + size, + } + } +} + +crate::impl_resource_type!(StagingBuffer); +crate::impl_storage_item!(StagingBuffer); + +#[derive(Debug)] +pub struct FlushedStagingBuffer { + raw: ManuallyDrop>, + device: Arc, + pub(crate) size: wgt::BufferSize, +} + +impl FlushedStagingBuffer { + pub(crate) fn raw(&self) -> &dyn hal::DynBuffer { + self.raw.as_ref() + } +} + +impl Drop for FlushedStagingBuffer { + fn drop(&mut self) { + resource_log!("Destroy raw StagingBuffer"); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { self.device.raw().destroy_buffer(raw) }; + } +} + +pub type TextureDescriptor<'a> = wgt::TextureDescriptor, Vec>; + +#[derive(Debug)] +pub(crate) enum TextureInner { + Native { + raw: Box, + }, + Surface { + raw: Box, + }, +} + +impl TextureInner { + pub(crate) fn raw(&self) -> &dyn hal::DynTexture { + match self { + Self::Native { raw } => raw.as_ref(), + Self::Surface { raw, .. } => raw.as_ref().borrow(), + } + } +} + +#[derive(Debug)] +pub enum TextureClearMode { + BufferCopy, + // View for clear via RenderPass for every subsurface (mip/layer/slice) + RenderPass { + clear_views: SmallVec<[ManuallyDrop>; 1]>, + is_color: bool, + }, + Surface { + clear_view: ManuallyDrop>, + }, + // Texture can't be cleared, attempting to do so will cause panic. + // (either because it is impossible for the type of texture or it is being destroyed) + None, +} + +#[derive(Debug)] +pub struct Texture { + pub(crate) inner: Snatchable, + pub(crate) device: Arc, + pub(crate) desc: wgt::TextureDescriptor<(), Vec>, + pub(crate) _hal_usage: wgt::TextureUses, + pub(crate) format_features: wgt::TextureFormatFeatures, + pub(crate) initialization_status: RwLock, + pub(crate) full_range: TextureSelector, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, + pub(crate) clear_mode: RwLock, + pub(crate) views: Mutex>, + pub(crate) bind_groups: Mutex>, +} + +impl Texture { + pub(crate) fn new( + device: &Arc, + inner: TextureInner, + hal_usage: wgt::TextureUses, + desc: &TextureDescriptor, + format_features: wgt::TextureFormatFeatures, + clear_mode: TextureClearMode, + init: bool, + ) -> Self { + Texture { + inner: Snatchable::new(inner), + device: device.clone(), + desc: desc.map_label(|_| ()), + _hal_usage: hal_usage, + format_features, + initialization_status: RwLock::new( + rank::TEXTURE_INITIALIZATION_STATUS, + if init { + TextureInitTracker::new(desc.mip_level_count, desc.array_layer_count()) + } else { + TextureInitTracker::new(desc.mip_level_count, 0) + }, + ), + full_range: TextureSelector { + mips: 0..desc.mip_level_count, + layers: 0..desc.array_layer_count(), + }, + label: desc.label.to_string(), + tracking_data: TrackingData::new(device.tracker_indices.textures.clone()), + clear_mode: RwLock::new(rank::TEXTURE_CLEAR_MODE, clear_mode), + views: Mutex::new(rank::TEXTURE_VIEWS, WeakVec::new()), + bind_groups: Mutex::new(rank::TEXTURE_BIND_GROUPS, WeakVec::new()), + } + } + + /// Checks that the given texture usage contains the required texture usage, + /// returns an error otherwise. + pub(crate) fn check_usage( + &self, + expected: wgt::TextureUsages, + ) -> Result<(), MissingTextureUsageError> { + if self.desc.usage.contains(expected) { + Ok(()) + } else { + Err(MissingTextureUsageError { + res: self.error_ident(), + actual: self.desc.usage, + expected, + }) + } + } +} + +impl Drop for Texture { + fn drop(&mut self) { + match *self.clear_mode.write() { + TextureClearMode::Surface { + ref mut clear_view, .. + } => { + // SAFETY: We are in the Drop impl and we don't use clear_view anymore after this point. + let raw = unsafe { ManuallyDrop::take(clear_view) }; + unsafe { + self.device.raw().destroy_texture_view(raw); + } + } + TextureClearMode::RenderPass { + ref mut clear_views, + .. + } => { + clear_views.iter_mut().for_each(|clear_view| { + // SAFETY: We are in the Drop impl and we don't use clear_view anymore after this point. + let raw = unsafe { ManuallyDrop::take(clear_view) }; + unsafe { + self.device.raw().destroy_texture_view(raw); + } + }); + } + _ => {} + }; + + if let Some(TextureInner::Native { raw }) = self.inner.take() { + resource_log!("Destroy raw {}", self.error_ident()); + unsafe { + self.device.raw().destroy_texture(raw); + } + } + } +} + +impl RawResourceAccess for Texture { + type DynResource = dyn hal::DynTexture; + + fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> { + self.inner.get(guard).map(|t| t.raw()) + } +} + +impl Texture { + pub(crate) fn try_inner<'a>( + &'a self, + guard: &'a SnatchGuard, + ) -> Result<&'a TextureInner, DestroyedResourceError> { + self.inner + .get(guard) + .ok_or_else(|| DestroyedResourceError(self.error_ident())) + } + + pub(crate) fn check_destroyed( + &self, + guard: &SnatchGuard, + ) -> Result<(), DestroyedResourceError> { + self.inner + .get(guard) + .map(|_| ()) + .ok_or_else(|| DestroyedResourceError(self.error_ident())) + } + + pub(crate) fn get_clear_view<'a>( + clear_mode: &'a TextureClearMode, + desc: &'a wgt::TextureDescriptor<(), Vec>, + mip_level: u32, + depth_or_layer: u32, + ) -> &'a dyn hal::DynTextureView { + match *clear_mode { + TextureClearMode::BufferCopy => { + panic!("Given texture is cleared with buffer copies, not render passes") + } + TextureClearMode::None => { + panic!("Given texture can't be cleared") + } + TextureClearMode::Surface { ref clear_view, .. } => clear_view.as_ref(), + TextureClearMode::RenderPass { + ref clear_views, .. + } => { + let index = if desc.dimension == wgt::TextureDimension::D3 { + (0..mip_level).fold(0, |acc, mip| { + acc + (desc.size.depth_or_array_layers >> mip).max(1) + }) + } else { + mip_level * desc.size.depth_or_array_layers + } + depth_or_layer; + clear_views[index as usize].as_ref() + } + } + } + + pub fn destroy(self: &Arc) { + let device = &self.device; + + let temp = { + let raw = match self.inner.snatch(&mut device.snatchable_lock.write()) { + Some(TextureInner::Native { raw }) => raw, + Some(TextureInner::Surface { .. }) => { + return; + } + None => { + // Per spec, it is valid to call `destroy` multiple times. + return; + } + }; + + let views = { + let mut guard = self.views.lock(); + mem::take(&mut *guard) + }; + + let bind_groups = { + let mut guard = self.bind_groups.lock(); + mem::take(&mut *guard) + }; + + queue::TempResource::DestroyedTexture(DestroyedTexture { + raw: ManuallyDrop::new(raw), + views, + clear_mode: mem::replace(&mut *self.clear_mode.write(), TextureClearMode::None), + bind_groups, + device: Arc::clone(&self.device), + label: self.label().to_owned(), + }) + }; + + if let Some(queue) = device.get_queue() { + let mut pending_writes = queue.pending_writes.lock(); + if pending_writes.contains_texture(self) { + pending_writes.consume_temp(temp); + } else { + let mut life_lock = queue.lock_life(); + let last_submit_index = life_lock.get_texture_latest_submission_index(self); + if let Some(last_submit_index) = last_submit_index { + life_lock.schedule_resource_destruction(temp, last_submit_index); + } + } + } + } +} + +/// A texture that has been marked as destroyed and is staged for actual deletion soon. +#[derive(Debug)] +pub struct DestroyedTexture { + raw: ManuallyDrop>, + views: WeakVec, + clear_mode: TextureClearMode, + bind_groups: WeakVec, + device: Arc, + label: String, +} + +impl DestroyedTexture { + pub fn label(&self) -> &dyn fmt::Debug { + &self.label + } +} + +impl Drop for DestroyedTexture { + fn drop(&mut self) { + let device = &self.device; + + let mut deferred = device.deferred_destroy.lock(); + deferred.push(DeferredDestroy::TextureViews(mem::take(&mut self.views))); + deferred.push(DeferredDestroy::BindGroups(mem::take( + &mut self.bind_groups, + ))); + drop(deferred); + + match mem::replace(&mut self.clear_mode, TextureClearMode::None) { + TextureClearMode::RenderPass { clear_views, .. } => { + for clear_view in clear_views { + let raw = ManuallyDrop::into_inner(clear_view); + unsafe { self.device.raw().destroy_texture_view(raw) }; + } + } + TextureClearMode::Surface { clear_view } => { + let raw = ManuallyDrop::into_inner(clear_view); + unsafe { self.device.raw().destroy_texture_view(raw) }; + } + _ => (), + } + + resource_log!("Destroy raw Texture (destroyed) {:?}", self.label()); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { + self.device.raw().destroy_texture(raw); + } + } +} + +#[derive(Clone, Copy, Debug)] +pub enum TextureErrorDimension { + X, + Y, + Z, +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum TextureDimensionError { + #[error("Dimension {0:?} is zero")] + Zero(TextureErrorDimension), + #[error("Dimension {dim:?} value {given} exceeds the limit of {limit}")] + LimitExceeded { + dim: TextureErrorDimension, + given: u32, + limit: u32, + }, + #[error("Sample count {0} is invalid")] + InvalidSampleCount(u32), + #[error("Width {width} is not a multiple of {format:?}'s block width ({block_width})")] + NotMultipleOfBlockWidth { + width: u32, + block_width: u32, + format: wgt::TextureFormat, + }, + #[error("Height {height} is not a multiple of {format:?}'s block height ({block_height})")] + NotMultipleOfBlockHeight { + height: u32, + block_height: u32, + format: wgt::TextureFormat, + }, + #[error( + "Width {width} is not a multiple of {format:?}'s width multiple requirement ({multiple})" + )] + WidthNotMultipleOf { + width: u32, + multiple: u32, + format: wgt::TextureFormat, + }, + #[error("Height {height} is not a multiple of {format:?}'s height multiple requirement ({multiple})")] + HeightNotMultipleOf { + height: u32, + multiple: u32, + format: wgt::TextureFormat, + }, + #[error("Multisampled texture depth or array layers must be 1, got {0}")] + MultisampledDepthOrArrayLayer(u32), +} + +impl WebGpuError for TextureDimensionError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateTextureError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + CreateTextureView(#[from] CreateTextureViewError), + #[error("Invalid usage flags {0:?}")] + InvalidUsage(wgt::TextureUsages), + #[error("Texture usage {0:?} is not compatible with texture usage {1:?}")] + IncompatibleUsage(wgt::TextureUsages, wgt::TextureUsages), + #[error(transparent)] + InvalidDimension(#[from] TextureDimensionError), + #[error("Depth texture ({1:?}) can't be created as {0:?}")] + InvalidDepthDimension(wgt::TextureDimension, wgt::TextureFormat), + #[error("Compressed texture ({1:?}) can't be created as {0:?}")] + InvalidCompressedDimension(wgt::TextureDimension, wgt::TextureFormat), + #[error( + "Texture descriptor mip level count {requested} is invalid, maximum allowed is {maximum}" + )] + InvalidMipLevelCount { requested: u32, maximum: u32 }, + #[error( + "Texture usages {0:?} are not allowed on a texture of type {1:?}{downlevel_suffix}", + downlevel_suffix = if *.2 { " due to downlevel restrictions" } else { "" } + )] + InvalidFormatUsages(wgt::TextureUsages, wgt::TextureFormat, bool), + #[error("The view format {0:?} is not compatible with texture format {1:?}, only changing srgb-ness is allowed.")] + InvalidViewFormat(wgt::TextureFormat, wgt::TextureFormat), + #[error("Texture usages {0:?} are not allowed on a texture of dimensions {1:?}")] + InvalidDimensionUsages(wgt::TextureUsages, wgt::TextureDimension), + #[error("Texture usage STORAGE_BINDING is not allowed for multisampled textures")] + InvalidMultisampledStorageBinding, + #[error("Format {0:?} does not support multisampling")] + InvalidMultisampledFormat(wgt::TextureFormat), + #[error("Sample count {0} is not supported by format {1:?} on this device. The WebGPU spec guarantees {2:?} samples are supported by this format. With the TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES feature your device supports {3:?}.")] + InvalidSampleCount(u32, wgt::TextureFormat, Vec, Vec), + #[error("Multisampled textures must have RENDER_ATTACHMENT usage")] + MultisampledNotRenderAttachment, + #[error("Texture format {0:?} can't be used due to missing features")] + MissingFeatures(wgt::TextureFormat, #[source] MissingFeatures), + #[error(transparent)] + MissingDownlevelFlags(#[from] MissingDownlevelFlags), +} + +crate::impl_resource_type!(Texture); +crate::impl_labeled!(Texture); +crate::impl_parent_device!(Texture); +crate::impl_storage_item!(Texture); +crate::impl_trackable!(Texture); + +impl Borrow for Texture { + fn borrow(&self) -> &TextureSelector { + &self.full_range + } +} + +impl WebGpuError for CreateTextureError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::CreateTextureView(e) => e.webgpu_error_type(), + Self::InvalidDimension(e) => e.webgpu_error_type(), + Self::MissingFeatures(_, e) => e.webgpu_error_type(), + Self::MissingDownlevelFlags(e) => e.webgpu_error_type(), + + Self::InvalidUsage(_) + | Self::IncompatibleUsage(_, _) + | Self::InvalidDepthDimension(_, _) + | Self::InvalidCompressedDimension(_, _) + | Self::InvalidMipLevelCount { .. } + | Self::InvalidFormatUsages(_, _, _) + | Self::InvalidViewFormat(_, _) + | Self::InvalidDimensionUsages(_, _) + | Self::InvalidMultisampledStorageBinding + | Self::InvalidMultisampledFormat(_) + | Self::InvalidSampleCount(..) + | Self::MultisampledNotRenderAttachment => ErrorType::Validation, + } + } +} + +/// Describes a [`TextureView`]. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(default))] +pub struct TextureViewDescriptor<'a> { + /// Debug label of the texture view. + /// + /// This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// Format of the texture view, or `None` for the same format as the texture + /// itself. + /// + /// At this time, it must be the same the underlying format of the texture. + pub format: Option, + /// The dimension of the texture view. + /// + /// - For 1D textures, this must be `D1`. + /// - For 2D textures it must be one of `D2`, `D2Array`, `Cube`, or `CubeArray`. + /// - For 3D textures it must be `D3`. + pub dimension: Option, + /// The allowed usage(s) for the texture view. Must be a subset of the usage flags of the texture. + /// If not provided, defaults to the full set of usage flags of the texture. + pub usage: Option, + /// Range within the texture that is accessible via this view. + pub range: wgt::ImageSubresourceRange, +} + +#[derive(Debug)] +pub(crate) struct HalTextureViewDescriptor { + pub texture_format: wgt::TextureFormat, + pub format: wgt::TextureFormat, + pub usage: wgt::TextureUsages, + pub dimension: wgt::TextureViewDimension, + pub range: wgt::ImageSubresourceRange, +} + +impl HalTextureViewDescriptor { + pub fn aspects(&self) -> hal::FormatAspects { + hal::FormatAspects::new(self.texture_format, self.range.aspect) + } +} + +#[derive(Debug, Copy, Clone, Error)] +pub enum TextureViewNotRenderableReason { + #[error("The texture this view references doesn't include the RENDER_ATTACHMENT usage. Provided usages: {0:?}")] + Usage(wgt::TextureUsages), + #[error("The dimension of this texture view is not 2D. View dimension: {0:?}")] + Dimension(wgt::TextureViewDimension), + #[error("This texture view has more than one mipmap level. View mipmap levels: {0:?}")] + MipLevelCount(u32), + #[error("This texture view has more than one array layer. View array layers: {0:?}")] + ArrayLayerCount(u32), + #[error( + "The aspects of this texture view are a subset of the aspects in the original texture. Aspects: {0:?}" + )] + Aspects(hal::FormatAspects), +} + +#[derive(Debug)] +pub struct TextureView { + pub(crate) raw: Snatchable>, + // if it's a surface texture - it's none + pub(crate) parent: Arc, + pub(crate) device: Arc, + pub(crate) desc: HalTextureViewDescriptor, + pub(crate) format_features: wgt::TextureFormatFeatures, + /// This is `Err` only if the texture view is not renderable + pub(crate) render_extent: Result, + pub(crate) samples: u32, + pub(crate) selector: TextureSelector, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, +} + +impl Drop for TextureView { + fn drop(&mut self) { + if let Some(raw) = self.raw.take() { + resource_log!("Destroy raw {}", self.error_ident()); + unsafe { + self.device.raw().destroy_texture_view(raw); + } + } + } +} + +impl RawResourceAccess for TextureView { + type DynResource = dyn hal::DynTextureView; + + fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> { + self.raw.get(guard).map(|it| it.as_ref()) + } + + fn try_raw<'a>( + &'a self, + guard: &'a SnatchGuard, + ) -> Result<&'a Self::DynResource, DestroyedResourceError> { + self.parent.check_destroyed(guard)?; + + self.raw(guard) + .ok_or_else(|| DestroyedResourceError(self.error_ident())) + } +} + +impl TextureView { + /// Checks that the given texture usage contains the required texture usage, + /// returns an error otherwise. + pub(crate) fn check_usage( + &self, + expected: wgt::TextureUsages, + ) -> Result<(), MissingTextureUsageError> { + if self.desc.usage.contains(expected) { + Ok(()) + } else { + Err(MissingTextureUsageError { + res: self.error_ident(), + actual: self.desc.usage, + expected, + }) + } + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateTextureViewError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + DestroyedResource(#[from] DestroyedResourceError), + #[error("Invalid texture view dimension `{view:?}` with texture of dimension `{texture:?}`")] + InvalidTextureViewDimension { + view: wgt::TextureViewDimension, + texture: wgt::TextureDimension, + }, + #[error("Texture view format `{0:?}` cannot be used as a render attachment. Make sure the format supports RENDER_ATTACHMENT usage and required device features are enabled.")] + TextureViewFormatNotRenderable(wgt::TextureFormat), + #[error("Texture view format `{0:?}` cannot be used as a storage binding. Make sure the format supports STORAGE usage and required device features are enabled.")] + TextureViewFormatNotStorage(wgt::TextureFormat), + #[error("Texture view usages (`{view:?}`) must be a subset of the texture's original usages (`{texture:?}`)")] + InvalidTextureViewUsage { + view: wgt::TextureUsages, + texture: wgt::TextureUsages, + }, + #[error("Texture view dimension `{0:?}` cannot be used with a multisampled texture")] + InvalidMultisampledTextureViewDimension(wgt::TextureViewDimension), + #[error( + "TextureView has an arrayLayerCount of {depth}. Views of type `Cube` must have arrayLayerCount of 6." + )] + InvalidCubemapTextureDepth { depth: u32 }, + #[error("TextureView has an arrayLayerCount of {depth}. Views of type `CubeArray` must have an arrayLayerCount that is a multiple of 6.")] + InvalidCubemapArrayTextureDepth { depth: u32 }, + #[error("Source texture width and height must be equal for a texture view of dimension `Cube`/`CubeArray`")] + InvalidCubeTextureViewSize, + #[error("Mip level count is 0")] + ZeroMipLevelCount, + #[error("Array layer count is 0")] + ZeroArrayLayerCount, + #[error( + "`TextureView` starts at mip level {base_mip_level} and spans {mip_level_count} mip \ + levels, but the texture view only has {total} total mip level(s)" + )] + TooManyMipLevels { + base_mip_level: u32, + mip_level_count: u32, + total: u32, + }, + #[error( + "`TextureView` starts at array layer {base_array_layer} and spans {array_layer_count}) \ + array layers, but the texture view only has {total} total layer(s)" + )] + TooManyArrayLayers { + base_array_layer: u32, + array_layer_count: u32, + total: u32, + }, + #[error("Requested array layer count {requested} is not valid for the target view dimension {dim:?}")] + InvalidArrayLayerCount { + requested: u32, + dim: wgt::TextureViewDimension, + }, + #[error( + "Aspect {requested_aspect:?} is not a valid aspect of the source texture format {texture_format:?}" + )] + InvalidAspect { + texture_format: wgt::TextureFormat, + requested_aspect: wgt::TextureAspect, + }, + #[error( + "Trying to create a view of format {view:?} of a texture with format {texture:?}, \ + but this view format is not present in the texture's viewFormat array" + )] + FormatReinterpretation { + texture: wgt::TextureFormat, + view: wgt::TextureFormat, + }, + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), +} + +impl WebGpuError for CreateTextureViewError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + + Self::InvalidTextureViewDimension { .. } + | Self::InvalidResource(_) + | Self::InvalidMultisampledTextureViewDimension(_) + | Self::InvalidCubemapTextureDepth { .. } + | Self::InvalidCubemapArrayTextureDepth { .. } + | Self::InvalidCubeTextureViewSize + | Self::ZeroMipLevelCount + | Self::ZeroArrayLayerCount + | Self::TooManyMipLevels { .. } + | Self::TooManyArrayLayers { .. } + | Self::InvalidArrayLayerCount { .. } + | Self::InvalidAspect { .. } + | Self::FormatReinterpretation { .. } + | Self::DestroyedResource(_) + | Self::TextureViewFormatNotRenderable(_) + | Self::TextureViewFormatNotStorage(_) + | Self::InvalidTextureViewUsage { .. } + | Self::MissingFeatures(_) => ErrorType::Validation, + } + } +} + +crate::impl_resource_type!(TextureView); +crate::impl_labeled!(TextureView); +crate::impl_parent_device!(TextureView); +crate::impl_storage_item!(TextureView); + +pub type ExternalTextureDescriptor<'a> = wgt::ExternalTextureDescriptor>; + +#[derive(Debug)] +pub struct ExternalTexture { + pub(crate) device: Arc, + /// Between 1 and 3 (inclusive) planes of texture data. + pub(crate) planes: arrayvec::ArrayVec, 3>, + /// Buffer containing a [`crate::device::resource::ExternalTextureParams`] + /// describing the external texture. + pub(crate) params: Arc, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, +} + +impl Drop for ExternalTexture { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + } +} + +impl ExternalTexture { + pub fn destroy(self: &Arc) { + self.params.destroy(); + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateExternalTextureError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), + #[error(transparent)] + CreateBuffer(#[from] CreateBufferError), + #[error(transparent)] + QueueWrite(#[from] queue::QueueWriteError), + #[error("External texture format {format:?} expects {expected} planes, but given {provided}")] + IncorrectPlaneCount { + format: wgt::ExternalTextureFormat, + expected: usize, + provided: usize, + }, + #[error("External texture planes cannot be multisampled, but given view with samples = {0}")] + InvalidPlaneMultisample(u32), + #[error("External texture planes expect a filterable float sample type, but given view with format {format:?} (sample type {sample_type:?})")] + InvalidPlaneSampleType { + format: wgt::TextureFormat, + sample_type: wgt::TextureSampleType, + }, + #[error("External texture planes expect 2D dimension, but given view with dimension = {0:?}")] + InvalidPlaneDimension(wgt::TextureViewDimension), + #[error(transparent)] + MissingTextureUsage(#[from] MissingTextureUsageError), + #[error("External texture format {format:?} plane {plane} expects format with {expected} components but given view with format {provided:?} ({} components)", + provided.components())] + InvalidPlaneFormat { + format: wgt::ExternalTextureFormat, + plane: usize, + expected: u8, + provided: wgt::TextureFormat, + }, +} + +impl WebGpuError for CreateExternalTextureError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + CreateExternalTextureError::Device(e) => e.webgpu_error_type(), + CreateExternalTextureError::MissingFeatures(e) => e.webgpu_error_type(), + CreateExternalTextureError::InvalidResource(e) => e.webgpu_error_type(), + CreateExternalTextureError::CreateBuffer(e) => e.webgpu_error_type(), + CreateExternalTextureError::QueueWrite(e) => e.webgpu_error_type(), + CreateExternalTextureError::MissingTextureUsage(e) => e.webgpu_error_type(), + CreateExternalTextureError::IncorrectPlaneCount { .. } + | CreateExternalTextureError::InvalidPlaneMultisample(_) + | CreateExternalTextureError::InvalidPlaneSampleType { .. } + | CreateExternalTextureError::InvalidPlaneDimension(_) + | CreateExternalTextureError::InvalidPlaneFormat { .. } => ErrorType::Validation, + } + } +} + +crate::impl_resource_type!(ExternalTexture); +crate::impl_labeled!(ExternalTexture); +crate::impl_parent_device!(ExternalTexture); +crate::impl_storage_item!(ExternalTexture); +crate::impl_trackable!(ExternalTexture); + +/// Describes a [`Sampler`] +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct SamplerDescriptor<'a> { + /// Debug label of the sampler. + /// + /// This will show up in graphics debuggers for easy identification. + pub label: Label<'a>, + /// How to deal with out of bounds accesses in the u (i.e. x) direction + pub address_modes: [wgt::AddressMode; 3], + /// How to filter the texture when it needs to be magnified (made larger) + pub mag_filter: wgt::FilterMode, + /// How to filter the texture when it needs to be minified (made smaller) + pub min_filter: wgt::FilterMode, + /// How to filter between mip map levels + pub mipmap_filter: wgt::MipmapFilterMode, + /// Minimum level of detail (i.e. mip level) to use + pub lod_min_clamp: f32, + /// Maximum level of detail (i.e. mip level) to use + pub lod_max_clamp: f32, + /// If this is enabled, this is a comparison sampler using the given comparison function. + pub compare: Option, + /// Must be at least 1. If this is not 1, all filter modes must be linear. + pub anisotropy_clamp: u16, + /// Border color to use when address_mode is + /// [`AddressMode::ClampToBorder`](wgt::AddressMode::ClampToBorder) + pub border_color: Option, +} + +#[derive(Debug)] +pub struct Sampler { + pub(crate) raw: ManuallyDrop>, + pub(crate) device: Arc, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, + /// `true` if this is a comparison sampler + pub(crate) comparison: bool, + /// `true` if this is a filtering sampler + pub(crate) filtering: bool, +} + +impl Drop for Sampler { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { + self.device.raw().destroy_sampler(raw); + } + } +} + +impl Sampler { + pub(crate) fn raw(&self) -> &dyn hal::DynSampler { + self.raw.as_ref() + } +} + +#[derive(Copy, Clone)] +pub enum SamplerFilterErrorType { + MagFilter, + MinFilter, + MipmapFilter, +} + +impl fmt::Debug for SamplerFilterErrorType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + SamplerFilterErrorType::MagFilter => write!(f, "magFilter"), + SamplerFilterErrorType::MinFilter => write!(f, "minFilter"), + SamplerFilterErrorType::MipmapFilter => write!(f, "mipmapFilter"), + } + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateSamplerError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Invalid lodMinClamp: {0}. Must be greater or equal to 0.0")] + InvalidLodMinClamp(f32), + #[error("Invalid lodMaxClamp: {lod_max_clamp}. Must be greater or equal to lodMinClamp (which is {lod_min_clamp}).")] + InvalidLodMaxClamp { + lod_min_clamp: f32, + lod_max_clamp: f32, + }, + #[error("Invalid anisotropic clamp: {0}. Must be at least 1.")] + InvalidAnisotropy(u16), + #[error("Invalid filter mode for {filter_type:?}: {filter_mode:?}. When anistropic clamp is not 1 (it is {anisotropic_clamp}), all filter modes must be linear.")] + InvalidFilterModeWithAnisotropy { + filter_type: SamplerFilterErrorType, + filter_mode: wgt::FilterMode, + anisotropic_clamp: u16, + }, + #[error("Invalid filter mode for {filter_type:?}: {filter_mode:?}. When anistropic clamp is not 1 (it is {anisotropic_clamp}), all filter modes must be linear.")] + InvalidMipmapFilterModeWithAnisotropy { + filter_type: SamplerFilterErrorType, + filter_mode: wgt::MipmapFilterMode, + anisotropic_clamp: u16, + }, + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), +} + +crate::impl_resource_type!(Sampler); +crate::impl_labeled!(Sampler); +crate::impl_parent_device!(Sampler); +crate::impl_storage_item!(Sampler); +crate::impl_trackable!(Sampler); + +impl WebGpuError for CreateSamplerError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + + Self::InvalidLodMinClamp(_) + | Self::InvalidLodMaxClamp { .. } + | Self::InvalidAnisotropy(_) + | Self::InvalidFilterModeWithAnisotropy { .. } + | Self::InvalidMipmapFilterModeWithAnisotropy { .. } => ErrorType::Validation, + } + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum CreateQuerySetError { + #[error(transparent)] + Device(#[from] DeviceError), + #[error("QuerySets cannot be made with zero queries")] + ZeroCount, + #[error("{count} is too many queries for a single QuerySet. QuerySets cannot be made more than {maximum} queries.")] + TooManyQueries { count: u32, maximum: u32 }, + #[error(transparent)] + MissingFeatures(#[from] MissingFeatures), +} + +impl WebGpuError for CreateQuerySetError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Device(e) => e.webgpu_error_type(), + Self::MissingFeatures(e) => e.webgpu_error_type(), + + Self::TooManyQueries { .. } | Self::ZeroCount => ErrorType::Validation, + } + } +} + +pub type QuerySetDescriptor<'a> = wgt::QuerySetDescriptor>; + +#[derive(Debug)] +pub struct QuerySet { + pub(crate) raw: ManuallyDrop>, + pub(crate) device: Arc, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, + pub(crate) desc: wgt::QuerySetDescriptor<()>, +} + +impl Drop for QuerySet { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { + self.device.raw().destroy_query_set(raw); + } + } +} + +crate::impl_resource_type!(QuerySet); +crate::impl_labeled!(QuerySet); +crate::impl_parent_device!(QuerySet); +crate::impl_storage_item!(QuerySet); +crate::impl_trackable!(QuerySet); + +impl QuerySet { + pub(crate) fn raw(&self) -> &dyn hal::DynQuerySet { + self.raw.as_ref() + } +} + +pub type BlasDescriptor<'a> = wgt::CreateBlasDescriptor>; +pub type TlasDescriptor<'a> = wgt::CreateTlasDescriptor>; + +pub type BlasPrepareCompactResult = Result<(), BlasPrepareCompactError>; + +#[cfg(send_sync)] +pub type BlasCompactCallback = Box; +#[cfg(not(send_sync))] +pub type BlasCompactCallback = Box; + +pub(crate) struct BlasPendingCompact { + pub(crate) op: Option, + // hold the parent alive while the mapping is active + pub(crate) _parent_blas: Arc, +} + +impl fmt::Debug for BlasPendingCompact { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BlasPendingCompact") + .field("op", &()) + .field("_parent_blas", &self._parent_blas) + .finish() + } +} + +#[derive(Debug)] +pub(crate) enum BlasCompactState { + /// Created from a compact operation. + Compacted, + /// Waiting for GPU to be done before mapping to get compacted size + Waiting(BlasPendingCompact), + /// Ready to be compacted + Ready { size: wgt::BufferAddress }, + /// Ready to prepare to compact. + Idle, +} + +#[cfg(send_sync)] +unsafe impl Send for BlasCompactState {} +#[cfg(send_sync)] +unsafe impl Sync for BlasCompactState {} + +#[derive(Debug)] +pub struct Blas { + pub(crate) raw: Snatchable>, + pub(crate) device: Arc, + pub(crate) size_info: hal::AccelerationStructureBuildSizes, + pub(crate) sizes: wgt::BlasGeometrySizeDescriptors, + pub(crate) flags: wgt::AccelerationStructureFlags, + pub(crate) update_mode: wgt::AccelerationStructureUpdateMode, + pub(crate) built_index: RwLock>, + pub(crate) handle: u64, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, + pub(crate) compaction_buffer: Option>>, + pub(crate) compacted_state: Mutex, +} + +impl Drop for Blas { + fn drop(&mut self) { + resource_log!("Destroy raw {}", self.error_ident()); + // SAFETY: We are in the Drop impl, and we don't use self.raw or self.compaction_buffer anymore after this point. + if let Some(raw) = self.raw.take() { + unsafe { + self.device.raw().destroy_acceleration_structure(raw); + } + } + if let Some(mut raw) = self.compaction_buffer.take() { + unsafe { + self.device + .raw() + .destroy_buffer(ManuallyDrop::take(&mut raw)) + } + } + } +} + +impl RawResourceAccess for Blas { + type DynResource = dyn hal::DynAccelerationStructure; + + fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> { + self.raw.get(guard).map(|it| it.as_ref()) + } +} + +impl Blas { + pub(crate) fn prepare_compact_async( + self: &Arc, + op: Option, + ) -> Result, BlasPrepareCompactError)> { + let device = &self.device; + if let Err(e) = device.check_is_valid() { + return Err((op, e.into())); + } + + if self.built_index.read().is_none() { + return Err((op, BlasPrepareCompactError::NotBuilt)); + } + + if !self + .flags + .contains(wgt::AccelerationStructureFlags::ALLOW_COMPACTION) + { + return Err((op, BlasPrepareCompactError::CompactionUnsupported)); + } + + let mut state = self.compacted_state.lock(); + *state = match *state { + BlasCompactState::Compacted => { + return Err((op, BlasPrepareCompactError::DoubleCompaction)) + } + BlasCompactState::Waiting(_) => { + return Err((op, BlasPrepareCompactError::CompactionPreparingAlready)) + } + BlasCompactState::Ready { .. } => { + return Err((op, BlasPrepareCompactError::CompactionPreparingAlready)) + } + BlasCompactState::Idle => BlasCompactState::Waiting(BlasPendingCompact { + op, + _parent_blas: self.clone(), + }), + }; + + let submit_index = if let Some(queue) = device.get_queue() { + queue.lock_life().prepare_compact(self).unwrap_or(0) // '0' means no wait is necessary + } else { + // We can safely unwrap below since we just set the `compacted_state` to `BlasCompactState::Waiting`. + let (mut callback, status) = self.read_back_compact_size().unwrap(); + if let Some(callback) = callback.take() { + callback(status); + } + 0 + }; + + Ok(submit_index) + } + + /// This function returns [`None`] only if [`Self::compacted_state`] is not [`BlasCompactState::Waiting`]. + #[must_use] + pub(crate) fn read_back_compact_size(&self) -> Option { + let mut state = self.compacted_state.lock(); + let pending_compact = match mem::replace(&mut *state, BlasCompactState::Idle) { + BlasCompactState::Waiting(pending_mapping) => pending_mapping, + // Compaction cancelled e.g. by rebuild + BlasCompactState::Idle => return None, + BlasCompactState::Ready { .. } => { + unreachable!("This should be validated out by `prepare_for_compaction`") + } + _ => panic!("No pending mapping."), + }; + let status = { + let compaction_buffer = self.compaction_buffer.as_ref().unwrap().as_ref(); + unsafe { + let map_res = self.device.raw().map_buffer( + compaction_buffer, + 0..size_of::() as wgt::BufferAddress, + ); + match map_res { + Ok(mapping) => { + if !mapping.is_coherent { + // Clippy complains about this because it might not be intended, but + // this is intentional. + #[expect(clippy::single_range_in_vec_init)] + self.device.raw().invalidate_mapped_ranges( + compaction_buffer, + &[0..size_of::() as wgt::BufferAddress], + ); + } + let size = core::ptr::read_unaligned( + mapping.ptr.as_ptr().cast::(), + ); + self.device.raw().unmap_buffer(compaction_buffer); + if self.size_info.acceleration_structure_size != 0 { + debug_assert_ne!(size, 0); + } + *state = BlasCompactState::Ready { size }; + Ok(()) + } + Err(err) => Err(BlasPrepareCompactError::from(DeviceError::from_hal(err))), + } + } + }; + Some((pending_compact.op, status)) + } +} + +crate::impl_resource_type!(Blas); +crate::impl_labeled!(Blas); +crate::impl_parent_device!(Blas); +crate::impl_storage_item!(Blas); +crate::impl_trackable!(Blas); + +#[derive(Debug)] +pub struct Tlas { + pub(crate) raw: Snatchable>, + pub(crate) device: Arc, + pub(crate) size_info: hal::AccelerationStructureBuildSizes, + pub(crate) max_instance_count: u32, + pub(crate) flags: wgt::AccelerationStructureFlags, + pub(crate) update_mode: wgt::AccelerationStructureUpdateMode, + pub(crate) built_index: RwLock>, + pub(crate) dependencies: RwLock>>, + pub(crate) instance_buffer: ManuallyDrop>, + /// The `label` from the descriptor used to create the resource. + pub(crate) label: String, + pub(crate) tracking_data: TrackingData, +} + +impl Drop for Tlas { + fn drop(&mut self) { + unsafe { + resource_log!("Destroy raw {}", self.error_ident()); + if let Some(structure) = self.raw.take() { + self.device.raw().destroy_acceleration_structure(structure); + } + let buffer = ManuallyDrop::take(&mut self.instance_buffer); + self.device.raw().destroy_buffer(buffer); + } + } +} + +impl RawResourceAccess for Tlas { + type DynResource = dyn hal::DynAccelerationStructure; + + fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> { + self.raw.get(guard).map(|raw| raw.as_ref()) + } +} + +crate::impl_resource_type!(Tlas); +crate::impl_labeled!(Tlas); +crate::impl_parent_device!(Tlas); +crate::impl_storage_item!(Tlas); +crate::impl_trackable!(Tlas); diff --git a/third_party/wgpu-core-29.0.4-patched/src/scratch.rs b/third_party/wgpu-core-29.0.4-patched/src/scratch.rs new file mode 100644 index 00000000..797f9db1 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/scratch.rs @@ -0,0 +1,45 @@ +use alloc::{boxed::Box, sync::Arc}; +use core::mem::ManuallyDrop; + +use wgt::BufferUses; + +use crate::device::{Device, DeviceError}; +use crate::{hal_label, resource_log}; + +#[derive(Debug)] +pub struct ScratchBuffer { + raw: ManuallyDrop>, + device: Arc, +} + +impl ScratchBuffer { + pub(crate) fn new(device: &Arc, size: wgt::BufferSize) -> Result { + let raw = unsafe { + device + .raw() + .create_buffer(&hal::BufferDescriptor { + label: hal_label(Some("(wgpu) scratch buffer"), device.instance_flags), + size: size.get(), + usage: BufferUses::ACCELERATION_STRUCTURE_SCRATCH, + memory_flags: hal::MemoryFlags::empty(), + }) + .map_err(DeviceError::from_hal)? + }; + Ok(Self { + raw: ManuallyDrop::new(raw), + device: device.clone(), + }) + } + pub(crate) fn raw(&self) -> &dyn hal::DynBuffer { + self.raw.as_ref() + } +} + +impl Drop for ScratchBuffer { + fn drop(&mut self) { + resource_log!("Destroy raw ScratchBuffer"); + // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point. + let raw = unsafe { ManuallyDrop::take(&mut self.raw) }; + unsafe { self.device.raw().destroy_buffer(raw) }; + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/snatch.rs b/third_party/wgpu-core-29.0.4-patched/src/snatch.rs new file mode 100644 index 00000000..a79cbcef --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/snatch.rs @@ -0,0 +1,200 @@ +use core::{cell::UnsafeCell, fmt, mem::ManuallyDrop}; + +use crate::lock::{rank, RankData, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +/// A guard that provides read access to snatchable data. +pub struct SnatchGuard<'a>(RwLockReadGuard<'a, ()>); +/// A guard that allows snatching the snatchable data. +pub struct ExclusiveSnatchGuard<'a>(#[expect(dead_code)] RwLockWriteGuard<'a, ()>); + +/// A value that is mostly immutable but can be "snatched" if we need to destroy +/// it early. +/// +/// In order to safely access the underlying data, the device's global snatchable +/// lock must be taken. To guarantee it, methods take a read or write guard of that +/// special lock. +pub struct Snatchable { + value: UnsafeCell>, +} + +impl Snatchable { + pub fn new(val: T) -> Self { + Snatchable { + value: UnsafeCell::new(Some(val)), + } + } + + #[allow(dead_code)] + pub fn empty() -> Self { + Snatchable { + value: UnsafeCell::new(None), + } + } + + /// Get read access to the value. Requires a the snatchable lock's read guard. + pub fn get<'a>(&'a self, _guard: &'a SnatchGuard) -> Option<&'a T> { + unsafe { (*self.value.get()).as_ref() } + } + + /// Take the value. Requires a the snatchable lock's write guard. + pub fn snatch(&self, _guard: &mut ExclusiveSnatchGuard) -> Option { + unsafe { (*self.value.get()).take() } + } + + /// Take the value without a guard. This can only be used with exclusive access + /// to self, so it does not require locking. + /// + /// Typically useful in a drop implementation. + pub fn take(&mut self) -> Option { + self.value.get_mut().take() + } +} + +// Can't safely print the contents of a snatchable object without holding +// the lock. +impl fmt::Debug for Snatchable { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "") + } +} + +unsafe impl Sync for Snatchable {} + +use trace::LockTrace; +#[cfg(all(debug_assertions, feature = "std"))] +mod trace { + use core::{cell::Cell, fmt, panic::Location}; + use std::{backtrace::Backtrace, thread}; + + pub(super) struct LockTrace { + purpose: &'static str, + caller: &'static Location<'static>, + backtrace: Backtrace, + } + + impl fmt::Display for LockTrace { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "a {} lock at {}\n{}", + self.purpose, self.caller, self.backtrace + ) + } + } + + impl LockTrace { + #[track_caller] + pub(super) fn enter(purpose: &'static str) { + let new = LockTrace { + purpose, + caller: Location::caller(), + backtrace: Backtrace::capture(), + }; + + if let Some(prev) = SNATCH_LOCK_TRACE.take() { + let current = thread::current(); + let name = current.name().unwrap_or(""); + panic!( + "thread '{name}' attempted to acquire a snatch lock recursively.\n\ + - Currently trying to acquire {new}\n\ + - Previously acquired {prev}", + ); + } else { + SNATCH_LOCK_TRACE.set(Some(new)); + } + } + + pub(super) fn exit() { + SNATCH_LOCK_TRACE.take(); + } + } + + std::thread_local! { + static SNATCH_LOCK_TRACE: Cell> = const { Cell::new(None) }; + } +} +#[cfg(not(all(debug_assertions, feature = "std")))] +mod trace { + pub(super) struct LockTrace { + _private: (), + } + + impl LockTrace { + pub(super) fn enter(_purpose: &'static str) {} + pub(super) fn exit() {} + } +} + +/// A Device-global lock for all snatchable data. +pub struct SnatchLock { + lock: RwLock<()>, +} + +impl SnatchLock { + /// The safety of `Snatchable::get` and `Snatchable::snatch` rely on their using of the + /// right SnatchLock (the one associated to the same device). This method is unsafe + /// to force force sers to think twice about creating a SnatchLock. The only place this + /// method should be called is when creating the device. + pub unsafe fn new(rank: rank::LockRank) -> Self { + SnatchLock { + lock: RwLock::new(rank, ()), + } + } + + /// Request read access to snatchable resources. + #[track_caller] + pub fn read(&self) -> SnatchGuard<'_> { + LockTrace::enter("read"); + SnatchGuard(self.lock.read()) + } + + /// Request write access to snatchable resources. + /// + /// This should only be called when a resource needs to be snatched. This has + /// a high risk of causing lock contention if called concurrently with other + /// wgpu work. + #[track_caller] + pub fn write(&self) -> ExclusiveSnatchGuard<'_> { + LockTrace::enter("write"); + ExclusiveSnatchGuard(self.lock.write()) + } + + #[track_caller] + pub unsafe fn force_unlock_read(&self, data: RankData) { + // This is unsafe because it can cause deadlocks if the lock is held. + // It should only be used in very specific cases, like when a resource + // needs to be snatched in a panic handler. + LockTrace::exit(); + unsafe { self.lock.force_unlock_read(data) }; + } +} + +impl SnatchGuard<'_> { + /// Forget the guard, leaving the lock in a locked state with no guard. + /// + /// This is equivalent to `std::mem::forget`, but preserves the information about the lock + /// rank. + pub fn forget(this: Self) -> RankData { + // Cancel the drop implementation of the current guard. + let manually_drop = ManuallyDrop::new(this); + + // As we are unable to destructure out of this guard due to the drop implementation, + // so we manually read the inner value. + // SAFETY: This is safe because we never access the original guard again. + let inner_guard = unsafe { core::ptr::read(&manually_drop.0) }; + + RwLockReadGuard::forget(inner_guard) + } +} + +impl Drop for SnatchGuard<'_> { + fn drop(&mut self) { + LockTrace::exit(); + } +} + +impl Drop for ExclusiveSnatchGuard<'_> { + fn drop(&mut self) { + LockTrace::exit(); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/storage.rs b/third_party/wgpu-core-29.0.4-patched/src/storage.rs new file mode 100644 index 00000000..e053cb8f --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/storage.rs @@ -0,0 +1,144 @@ +use alloc::{sync::Arc, vec::Vec}; +use core::mem; + +use crate::id::{Id, Marker}; +use crate::resource::ResourceType; +use crate::{Epoch, Index}; + +/// An entry in a `Storage::map` table. +#[derive(Debug)] +pub(crate) enum Element +where + T: StorageItem, +{ + /// There are no live ids with this index. + Vacant, + + /// There is one live id with this index, allocated at the given + /// epoch. + Occupied(T, Epoch), +} + +/// Not a public API. For use only by `player`. +#[doc(hidden)] +pub trait StorageItem: ResourceType { + type Marker: Marker; +} + +impl ResourceType for Arc { + const TYPE: &'static str = T::TYPE; +} + +impl StorageItem for Arc { + type Marker = T::Marker; +} + +#[macro_export] +macro_rules! impl_storage_item { + ($ty:ident) => { + impl $crate::storage::StorageItem for $ty { + type Marker = $crate::id::markers::$ty; + } + }; +} + +/// A table of `T` values indexed by the id type `I`. +/// +/// `Storage` implements [`core::ops::Index`], accepting `Id` values as +/// indices. +/// +/// The table is represented as a vector indexed by the ids' index +/// values, so you should use an id allocator like `IdentityManager` +/// that keeps the index values dense and close to zero. +#[derive(Debug)] +pub(crate) struct Storage +where + T: StorageItem, +{ + pub(crate) map: Vec>, + kind: &'static str, +} + +impl Storage +where + T: StorageItem, +{ + pub(crate) fn new() -> Self { + Self { + map: Vec::new(), + kind: T::TYPE, + } + } +} + +impl Storage +where + T: StorageItem, +{ + pub(crate) fn insert(&mut self, id: Id, value: T) { + let (index, epoch) = id.unzip(); + let index = index as usize; + if index >= self.map.len() { + self.map.resize_with(index + 1, || Element::Vacant); + } + match mem::replace(&mut self.map[index], Element::Occupied(value, epoch)) { + Element::Vacant => {} + Element::Occupied(_, storage_epoch) => { + assert_ne!( + epoch, + storage_epoch, + "Index {index:?} of {} is already occupied", + T::TYPE + ); + } + } + } + + pub(crate) fn remove(&mut self, id: Id) -> T { + let (index, epoch) = id.unzip(); + let stored = self + .map + .get_mut(index as usize) + .unwrap_or_else(|| panic!("{}[{:?}] does not exist", self.kind, id)); + match mem::replace(stored, Element::Vacant) { + Element::Occupied(value, storage_epoch) => { + assert_eq!(epoch, storage_epoch, "id epoch mismatch"); + value + } + Element::Vacant => panic!("Cannot remove a vacant resource"), + } + } + + pub(crate) fn iter(&self) -> impl Iterator, &T)> { + self.map + .iter() + .enumerate() + .filter_map(move |(index, x)| match *x { + Element::Occupied(ref value, storage_epoch) => { + Some((Id::zip(index as Index, storage_epoch), value)) + } + _ => None, + }) + } +} + +impl Storage +where + T: StorageItem + Clone, +{ + /// Get an owned reference to an item. + /// Panics if there is an epoch mismatch, the entry is empty or in error. + pub(crate) fn get(&self, id: Id) -> T { + let (index, epoch) = id.unzip(); + let (result, storage_epoch) = match self.map.get(index as usize) { + Some(&Element::Occupied(ref v, epoch)) => (v.clone(), epoch), + None | Some(&Element::Vacant) => panic!("{}[{:?}] does not exist", self.kind, id), + }; + assert_eq!( + epoch, storage_epoch, + "{}[{:?}] is no longer alive", + self.kind, id + ); + result + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/common.wgsl b/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/common.wgsl new file mode 100644 index 00000000..cb25220e --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/common.wgsl @@ -0,0 +1,142 @@ +// Common routines for timestamp normalization. +// +// This is split out into its own file so that the tests in `tests` can include +// it without including the normal endpoints and interface definitions. + +/// 64-bit unsigned integer type. +/// +/// We cannot rely on native 64-bit integers, so we define our own 64-bit +/// integer type as two 32-bit integers. +struct Uint64 { + /// Least significant word. + low: u32, + /// Most significant word. + high: u32, +} + +/// 96-bit unsigned integer type. +struct Uint96 { + /// Least significant word. + low: u32, + /// Middle word. + mid: u32, + /// Most significant word. + high: u32, +} + +/// Truncates a 96-bit number to a 64-bit number by discarding the upper 32 bits. +fn truncate_u96_to_u64(x: Uint96) -> Uint64 { + return Uint64( + x.low, + x.mid, + ); +} + +/// Returns the lower 16 bits of a 32-bit integer. +fn low(a: u32) -> u32 { + return a & 0xFFFF; +} + +/// Returns the upper 16 bits of a 32-bit integer. +fn high(a: u32) -> u32 { + return a >> 16; +} + +/// Combines two 16bit words into a single 32bit word. +/// `w1` is the upper 16 bits and `w0` is the lower 16 bits. +/// +/// The high 16 bits of each argument are discarded. +fn u32_from_u16s(w1: u32, w0: u32) -> u32 { + return low(w1) << 16 | low(w0); +} + +// Multiplies a 64-bit number by a 32-bit number and outputs a 96-bit result. +// +// The number of digits (bits) needed to represent the result of a multiplication +// is the sum of the number of input digits (bits). Since we are multiplying a +// 64-bit number by a 32-bit number, we need 96 bits to represent the result. +fn u64_mul_u32(a: Uint64, b: u32) -> Uint96 { + // Does not use any 64-bit operations and we don't have access to `mul(u32, u32) -> u64` + // operations, so we operate entirely on `mul(u16, u16) -> u32`. + + // This implements standard "long multiplication" algorithm using 16-bit words. + // Each element in this diagram is a 16-bit word. + // + // a3 a2 a1 a0 + // * b1 b0 + // ---------------------------- + // i0 = p00 p00 + // i1 = p10 p10 + // i2 = p20 p20 + // i3 = p30 p30 + // i4 = p01 p01 + // i5 = p11 p11 + // i6 = p21 p21 + // i7 = p31 p31 + // ---------------------------- + // r6 r5 r4 r3 r2 r1 r0 + + // Decompose the 64-bit number into four 16-bit words. + let a0 = low(a.low); + let a1 = high(a.low); + let a2 = low(a.high); + let a3 = high(a.high); + + // Decompose the 32-bit number into two 16-bit words. + let b0 = low(b); + let b1 = high(b); + + // Each line represents one row in the diagram above. + let i0 = a0 * b0; + let i1 = a1 * b0; + let i2 = a2 * b0; + let i3 = a3 * b0; + let i4 = a0 * b1; + let i5 = a1 * b1; + let i6 = a2 * b1; + let i7 = a3 * b1; + + // Each line represents one column in the diagram above. + // + // The high 16 bits of each column are the carry to the next column. + let r0 = low(i0); + let r1 = high(i0) + low(i1) + low(i4) + high(r0); + let r2 = high(i1) + low(i2) + high(i4) + low(i5) + high(r1); + let r3 = high(i2) + low(i3) + high(i5) + low(i6) + high(r2); + let r4 = high(i3) + high(i6) + low(i7) + high(r3); + let r5 = high(i7) + high(r4); + // The r5 carry will always be zero. + + let out0 = u32_from_u16s(r1, r0); + let out1 = u32_from_u16s(r3, r2); + let out2 = u32_from_u16s(r5, r4); + + return Uint96(out0, out1, out2); +} + +// Shifts a 96-bit number right by a given number of bits. +// +// The shift is in the range [0, 32]. +fn shift_right_96(x: Uint96, shift: u32) -> Uint96 { + // Shift wraps around at 32, which breaks the algorithm when + // either shift is 32 or inv_shift is 32. + if (shift == 0) { + return x; + } + if (shift == 32) { + return Uint96(x.mid, x.high, 0); + } + + let inv_shift = 32 - shift; + + let carry2 = x.high << inv_shift; + let carry1 = x.mid << inv_shift; + + var out: Uint96; + + out.high = x.high >> shift; + out.mid = (x.mid >> shift) | carry2; + out.low = (x.low >> shift) | carry1; + + return out; +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/mod.rs b/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/mod.rs new file mode 100644 index 00000000..e6ddbad4 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/mod.rs @@ -0,0 +1,427 @@ +//! Utility for normalizing GPU timestamp queries to have a consistent +//! 1GHz period. This uses a compute shader to do the normalization, +//! so the timestamps exist in their correct format on the GPU, as +//! is required by the WebGPU specification. +//! +//! ## Algorithm +//! +//! The fundamental operation is multiplying a u64 timestamp by an f32 +//! value. We have neither f64s nor u64s in shaders, so we need to do +//! something more complicated. +//! +//! We first decompose the f32 into a u32 fraction where the denominator +//! is a power of two. We do the computation with f64 for ease of computation, +//! as those can store u32s losslessly. +//! +//! Because the denominator is a power of two, this means the shader can evaluate +//! this divide by using a shift. Additionally, we always choose the largest denominator +//! we can, so that the fraction is as precise as possible. +//! +//! To evaluate this function, we have two helper operations (both in common.wgsl). +//! +//! 1. `u64_mul_u32` multiplies a u64 by a u32 and returns a u96. +//! 2. `shift_right_u96` shifts a u96 right by a given amount, returning a u96. +//! +//! See their implementations for more details. +//! +//! We then multiply the timestamp by the numerator, and shift it right by the +//! denominator. This gives us the normalized timestamp. + +use core::num::NonZeroU64; + +use alloc::{boxed::Box, string::String, string::ToString, sync::Arc}; + +use hashbrown::HashMap; + +use crate::{ + device::{Device, DeviceError}, + hal_label, + pipeline::{CreateComputePipelineError, CreateShaderModuleError}, + resource::Buffer, + snatch::SnatchGuard, + track::BufferTracker, +}; + +pub const TIMESTAMP_NORMALIZATION_BUFFER_USES: wgt::BufferUses = + wgt::BufferUses::STORAGE_READ_WRITE; + +struct InternalState { + temporary_bind_group_layout: Box, + pipeline_layout: Box, + pipeline: Box, +} + +#[derive(Debug, Clone, thiserror::Error)] +pub enum TimestampNormalizerInitError { + #[error("Failed to initialize bind group layout")] + BindGroupLayout(#[source] DeviceError), + #[cfg(feature = "wgsl")] + #[error("Failed to parse shader")] + ParseWgsl(#[source] naga::error::ShaderError), + #[error("Failed to validate shader module")] + ValidateWgsl(#[source] naga::error::ShaderError>), + #[error("Failed to create shader module")] + CreateShaderModule(#[from] CreateShaderModuleError), + #[error("Failed to create pipeline layout")] + PipelineLayout(#[source] DeviceError), + #[error("Failed to create compute pipeline")] + ComputePipeline(#[from] CreateComputePipelineError), +} + +/// Normalizes GPU timestamps to have a consistent 1GHz period. +/// See module documentation for more information. +pub struct TimestampNormalizer { + state: Option, +} + +impl TimestampNormalizer { + /// Creates a new timestamp normalizer. + /// + /// If the device cannot support automatic timestamp normalization, + /// this will return a normalizer that does nothing. + /// + /// # Errors + /// + /// If any resources are invalid, this will return an error. + pub fn new( + device: &Device, + timestamp_period: f32, + ) -> Result { + unsafe { + if !device + .instance_flags + .contains(wgt::InstanceFlags::AUTOMATIC_TIMESTAMP_NORMALIZATION) + { + return Ok(Self { state: None }); + } + + if !device + .downlevel + .flags + .contains(wgt::DownlevelFlags::COMPUTE_SHADERS) + { + log::error!("Automatic timestamp normalization was requested, but compute shaders are not supported."); + return Ok(Self { state: None }); + } + + if timestamp_period == 1.0 { + // If the period is 1, we don't need to do anything to them. + return Ok(Self { state: None }); + } + + let temporary_bind_group_layout = device + .raw() + .create_bind_group_layout(&hal::BindGroupLayoutDescriptor { + label: hal_label( + Some("(wgpu internal) Timestamp Normalization Bind Group Layout"), + device.instance_flags, + ), + flags: hal::BindGroupLayoutFlags::empty(), + entries: &[wgt::BindGroupLayoutEntry { + binding: 0, + visibility: wgt::ShaderStages::COMPUTE, + ty: wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Storage { read_only: false }, + has_dynamic_offset: false, + min_binding_size: Some(NonZeroU64::new(8).unwrap()), + }, + count: None, + }], + }) + .map_err(|e| { + TimestampNormalizerInitError::BindGroupLayout(device.handle_hal_error(e)) + })?; + + let common_src = include_str!("common.wgsl"); + let src = include_str!("timestamp_normalization.wgsl"); + + let preprocessed_src = alloc::format!("{common_src}\n{src}"); + + #[cfg(feature = "wgsl")] + let module = naga::front::wgsl::parse_str(&preprocessed_src).map_err(|inner| { + TimestampNormalizerInitError::ParseWgsl(naga::error::ShaderError { + source: preprocessed_src.clone(), + label: None, + inner: Box::new(inner), + }) + })?; + #[cfg(not(feature = "wgsl"))] + #[allow(clippy::diverging_sub_expression)] + let module = + panic!("Timestamp normalization requires the wgsl feature flag to be enabled!"); + + let info = crate::device::create_validator( + wgt::Features::IMMEDIATES, + wgt::DownlevelFlags::empty(), + naga::valid::ValidationFlags::all(), + ) + .validate(&module) + .map_err(|inner| { + TimestampNormalizerInitError::ValidateWgsl(naga::error::ShaderError { + source: preprocessed_src.clone(), + label: None, + inner: Box::new(inner), + }) + })?; + let hal_shader = hal::ShaderInput::Naga(hal::NagaShader { + module: alloc::borrow::Cow::Owned(module), + info, + debug_source: None, + }); + let hal_desc = hal::ShaderModuleDescriptor { + label: hal_label( + Some("(wgpu internal) Timestamp normalizer shader module"), + device.instance_flags, + ), + runtime_checks: wgt::ShaderRuntimeChecks::unchecked(), + }; + let module = device + .raw() + .create_shader_module(&hal_desc, hal_shader) + .map_err(|error| match error { + hal::ShaderError::Device(error) => { + CreateShaderModuleError::Device(device.handle_hal_error(error)) + } + hal::ShaderError::Compilation(ref msg) => { + log::error!("Shader error: {msg}"); + CreateShaderModuleError::Generation + } + })?; + + let pipeline_layout = device + .raw() + .create_pipeline_layout(&hal::PipelineLayoutDescriptor { + label: hal_label( + Some("(wgpu internal) Timestamp normalizer pipeline layout"), + device.instance_flags, + ), + bind_group_layouts: &[Some(temporary_bind_group_layout.as_ref())], + immediate_size: 8, + flags: hal::PipelineLayoutFlags::empty(), + }) + .map_err(|e| { + TimestampNormalizerInitError::PipelineLayout(device.handle_hal_error(e)) + })?; + + let (multiplier, shift) = compute_timestamp_period(timestamp_period); + + let mut constants = HashMap::with_capacity(2); + constants.insert(String::from("TIMESTAMP_PERIOD_MULTIPLY"), multiplier as f64); + constants.insert(String::from("TIMESTAMP_PERIOD_SHIFT"), shift as f64); + + let pipeline_desc = hal::ComputePipelineDescriptor { + label: hal_label( + Some("(wgpu internal) Timestamp normalizer pipeline"), + device.instance_flags, + ), + layout: pipeline_layout.as_ref(), + stage: hal::ProgrammableStage { + module: module.as_ref(), + entry_point: "main", + constants: &constants, + zero_initialize_workgroup_memory: false, + }, + cache: None, + }; + let pipeline = device + .raw() + .create_compute_pipeline(&pipeline_desc) + .map_err(|err| match err { + hal::PipelineError::Device(error) => { + CreateComputePipelineError::Device(device.handle_hal_error(error)) + } + hal::PipelineError::Linkage(_stages, msg) => { + CreateComputePipelineError::Internal(msg) + } + hal::PipelineError::EntryPoint(_stage) => CreateComputePipelineError::Internal( + crate::device::ENTRYPOINT_FAILURE_ERROR.to_string(), + ), + hal::PipelineError::PipelineConstants(_, error) => { + CreateComputePipelineError::PipelineConstants(error) + } + })?; + + Ok(Self { + state: Some(InternalState { + temporary_bind_group_layout, + pipeline_layout, + pipeline, + }), + }) + } + } + + /// Create a bind group for normalizing timestamps in `buffer`. + /// + /// This function is unsafe because it does not know that `buffer_size` is + /// the true size of the buffer. + pub unsafe fn create_normalization_bind_group( + &self, + device: &Device, + buffer: &dyn hal::DynBuffer, + buffer_label: Option<&str>, + buffer_size: wgt::BufferSize, + buffer_usages: wgt::BufferUsages, + ) -> Result { + unsafe { + let Some(ref state) = &self.state else { + return Ok(TimestampNormalizationBindGroup { raw: None }); + }; + + if !buffer_usages.contains(wgt::BufferUsages::QUERY_RESOLVE) { + return Ok(TimestampNormalizationBindGroup { raw: None }); + } + + // If this buffer is large enough that we wouldn't be able to bind the entire thing + // at once to normalize the timestamps, we can't use it. We force the buffer to fail + // to allocate. The lowest max binding size is 128MB, and query sets must be small + // (no more than 4096), so this should never be hit in practice by sane programs. + if buffer_size.get() > device.adapter.limits().max_storage_buffer_binding_size { + return Err(DeviceError::OutOfMemory); + } + + let bg_label_alloc; + let label = match buffer_label { + Some(label) => { + bg_label_alloc = alloc::format!("Timestamp normalization bind group ({label})"); + &*bg_label_alloc + } + None => "Timestamp normalization bind group", + }; + + let bg = device + .raw() + .create_bind_group(&hal::BindGroupDescriptor { + label: hal_label(Some(label), device.instance_flags), + layout: &*state.temporary_bind_group_layout, + buffers: &[hal::BufferBinding::new_unchecked(buffer, 0, buffer_size)], + samplers: &[], + textures: &[], + acceleration_structures: &[], + external_textures: &[], + entries: &[hal::BindGroupEntry { + binding: 0, + resource_index: 0, + count: 1, + }], + }) + .map_err(|e| device.handle_hal_error(e))?; + + Ok(TimestampNormalizationBindGroup { raw: Some(bg) }) + } + } + + pub fn normalize( + &self, + snatch_guard: &SnatchGuard<'_>, + encoder: &mut dyn hal::DynCommandEncoder, + tracker: &mut BufferTracker, + bind_group: &TimestampNormalizationBindGroup, + buffer: &Arc, + buffer_offset_bytes: u64, + total_timestamps: u32, + ) { + let Some(ref state) = &self.state else { + return; + }; + + let Some(bind_group) = bind_group.raw.as_deref() else { + return; + }; + + let buffer_offset_timestamps: u32 = (buffer_offset_bytes / 8).try_into().unwrap(); // Unreachable as MAX_QUERIES is way less than u32::MAX + + let pending_barrier = tracker.set_single(buffer, wgt::BufferUses::STORAGE_READ_WRITE); + + let barrier = pending_barrier.map(|pending| pending.into_hal(buffer, snatch_guard)); + + let needed_workgroups = total_timestamps.div_ceil(64); + + unsafe { + encoder.transition_buffers(barrier.as_slice()); + encoder.begin_compute_pass(&hal::ComputePassDescriptor { + label: hal_label( + Some("(wgpu internal) Timestamp normalization pass"), + buffer.device.instance_flags, + ), + timestamp_writes: None, + }); + encoder.set_compute_pipeline(&*state.pipeline); + encoder.set_bind_group(&*state.pipeline_layout, 0, bind_group, &[]); + encoder.set_immediates( + &*state.pipeline_layout, + 0, + &[buffer_offset_timestamps, total_timestamps], + ); + encoder.dispatch([needed_workgroups, 1, 1]); + encoder.end_compute_pass(); + } + } + + pub fn dispose(self, device: &dyn hal::DynDevice) { + unsafe { + let Some(state) = self.state else { + return; + }; + + device.destroy_compute_pipeline(state.pipeline); + device.destroy_pipeline_layout(state.pipeline_layout); + device.destroy_bind_group_layout(state.temporary_bind_group_layout); + } + } + + pub fn enabled(&self) -> bool { + self.state.is_some() + } +} + +#[derive(Debug)] +pub struct TimestampNormalizationBindGroup { + raw: Option>, +} + +impl TimestampNormalizationBindGroup { + pub fn dispose(self, device: &dyn hal::DynDevice) { + unsafe { + if let Some(raw) = self.raw { + device.destroy_bind_group(raw); + } + } + } +} + +fn compute_timestamp_period(input: f32) -> (u32, u32) { + let pow2 = input.log2().ceil() as i32; + let clamped_pow2 = pow2.clamp(-32, 32).unsigned_abs(); + let shift = 32 - clamped_pow2; + + let denominator = (1u64 << shift) as f64; + + // float -> int conversions are defined to saturate. + let multiplier = (input as f64 * denominator).round() as u32; + + (multiplier, shift) +} + +#[cfg(test)] +mod tests { + use core::f64; + + fn assert_timestamp_case(input: f32) { + let (multiplier, shift) = super::compute_timestamp_period(input); + + let output = multiplier as f64 / (1u64 << shift) as f64; + + assert!((input as f64 - output).abs() < 0.0000001); + } + + #[test] + fn compute_timestamp_period() { + assert_timestamp_case(0.01); + assert_timestamp_case(0.5); + assert_timestamp_case(1.0); + assert_timestamp_case(2.0); + assert_timestamp_case(2.7); + assert_timestamp_case(1000.7); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/timestamp_normalization.wgsl b/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/timestamp_normalization.wgsl new file mode 100644 index 00000000..26dfed61 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/timestamp_normalization.wgsl @@ -0,0 +1,41 @@ +// Must have "common.wgsl" preprocessed before this file's contents. +// +// To compile this locally, you can run: +// ``` +// cat common.wgsl timestamp_normalization.wgsl | cargo run -p naga-cli -- --stdin-file-path timestamp_normalization.wgsl +// ``` + +// For an explanation of the timestamp normalization process, see +// the `mod.rs` file in this folder. + +// These is the timestamp period turned into a fraction +// with an integer numerator and denominator. The denominator +// is a power of two, so the division can be done with a shift. +override TIMESTAMP_PERIOD_MULTIPLY: u32 = 1; +override TIMESTAMP_PERIOD_SHIFT: u32 = 0; + +@group(0) @binding(0) +var timestamps: array; + +struct ImmediateData { + timestamp_offset: u32, + timestamp_count: u32, +} + +var im: ImmediateData; + +@compute @workgroup_size(64) +fn main(@builtin(global_invocation_id) id: vec3u) { + if id.x >= im.timestamp_count { + return; + } + + let index = id.x + im.timestamp_offset; + + let input_value = timestamps[index]; + + let tmp1 = u64_mul_u32(input_value, TIMESTAMP_PERIOD_MULTIPLY); + let tmp2 = shift_right_96(tmp1, TIMESTAMP_PERIOD_SHIFT); + + timestamps[index] = truncate_u96_to_u64(tmp2); +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/track/blas.rs b/third_party/wgpu-core-29.0.4-patched/src/track/blas.rs new file mode 100644 index 00000000..dec86aa1 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/track/blas.rs @@ -0,0 +1,59 @@ +use crate::{ + resource::{Blas, Trackable}, + track::metadata::ResourceMetadata, +}; +use alloc::sync::Arc; + +/// A tracker that holds tracks BLASes. +/// +/// This is mostly a safe shell around [`ResourceMetadata`] +#[derive(Debug)] +pub(crate) struct BlasTracker { + metadata: ResourceMetadata>, + size: usize, +} + +impl BlasTracker { + pub fn new() -> Self { + Self { + metadata: ResourceMetadata::new(), + size: 0, + } + } + + /// Inserts a single resource into the resource tracker. + /// + /// Returns a reference to the newly inserted resource. + /// (This allows avoiding a clone/reference count increase in many cases.) + pub fn insert_single(&mut self, resource: Arc) -> &Arc { + let index = resource.tracker_index().as_usize(); + self.allow_index(index); + + unsafe { + // # SAFETY: we just allowed this resource, which makes the metadata object larger if + // it's not in bounds + self.metadata.insert(index, resource) + } + } + + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Texture ID before + /// all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.size = size; + self.metadata.set_size(size) + } + + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.size { + self.set_size(index + 1); + } + } + + /// Returns true if the tracker owns the given texture. + pub fn contains(&self, blas: &Blas) -> bool { + self.metadata.contains(blas.tracker_index().as_usize()) + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/track/buffer.rs b/third_party/wgpu-core-29.0.4-patched/src/track/buffer.rs new file mode 100644 index 00000000..afa389ab --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/track/buffer.rs @@ -0,0 +1,815 @@ +//! Buffer Trackers +//! +//! Buffers are represented by a single state for the whole resource, +//! a 16 bit bitflag of buffer usages. Because there is only ever +//! one subresource, they have no selector. + +use alloc::{ + sync::{Arc, Weak}, + vec::Vec, +}; + +use hal::BufferBarrier; +use wgt::{strict_assert, strict_assert_eq, BufferUses}; + +use super::{PendingTransition, TrackerIndex}; +use crate::{ + resource::{Buffer, Trackable}, + snatch::SnatchGuard, + track::{ + invalid_resource_state, skip_barrier, ResourceMetadata, ResourceMetadataProvider, + ResourceUsageCompatibilityError, ResourceUses, + }, +}; + +impl ResourceUses for BufferUses { + const EXCLUSIVE: Self = Self::EXCLUSIVE; + + type Selector = (); + + fn bits(self) -> u16 { + Self::bits(&self) + } + + fn any_exclusive(self) -> bool { + self.intersects(Self::EXCLUSIVE) + } +} + +/// Stores a bind group's buffers + their usages (within the bind group). +#[derive(Debug)] +pub(crate) struct BufferBindGroupState { + buffers: Vec<(Arc, BufferUses)>, +} +impl BufferBindGroupState { + pub fn new() -> Self { + Self { + buffers: Vec::new(), + } + } + + /// Optimize the buffer bind group state by sorting it by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant ascending order. + pub(crate) fn optimize(&mut self) { + self.buffers + .sort_unstable_by_key(|(b, _)| b.tracker_index()); + } + + /// Returns a list of all buffers tracked. May contain duplicates. + pub fn used_tracker_indices(&self) -> impl Iterator + '_ { + self.buffers + .iter() + .map(|(b, _)| b.tracker_index()) + .collect::>() + .into_iter() + } + + /// Adds the given resource with the given state. + pub fn insert_single(&mut self, buffer: Arc, state: BufferUses) { + self.buffers.push((buffer, state)); + } +} + +/// Stores all buffer state within a single usage scope. +#[derive(Debug)] +pub(crate) struct BufferUsageScope { + state: Vec, + metadata: ResourceMetadata>, + ordered_uses_mask: BufferUses, +} + +impl Default for BufferUsageScope { + fn default() -> Self { + Self { + state: Vec::new(), + metadata: ResourceMetadata::new(), + ordered_uses_mask: BufferUses::empty(), + } + } +} + +impl BufferUsageScope { + fn tracker_assert_in_bounds(&self, index: usize) { + strict_assert!(index < self.state.len()); + self.metadata.tracker_assert_in_bounds(index); + } + pub fn clear(&mut self) { + self.state.clear(); + self.metadata.clear(); + } + + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Buffer ID before + /// all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.state.resize(size, BufferUses::empty()); + self.metadata.set_size(size); + } + + pub fn set_ordered_uses_mask(&mut self, ordered_uses_mask: BufferUses) { + self.ordered_uses_mask = ordered_uses_mask; + } + + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.state.len() { + self.set_size(index + 1); + } + } + + /// Merge the list of buffer states in the given bind group into this usage scope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// Because bind groups do not check if the union of all their states is valid, + /// this method is allowed to return Err on the first bind group bound. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn merge_bind_group( + &mut self, + bind_group: &BufferBindGroupState, + ) -> Result<(), ResourceUsageCompatibilityError> { + for &(ref resource, state) in bind_group.buffers.iter() { + let index = resource.tracker_index().as_usize(); + + unsafe { + self.insert_or_merge( + index as _, + index, + BufferStateProvider::Direct { state }, + ResourceMetadataProvider::Direct { resource }, + )? + }; + } + + Ok(()) + } + + /// Merge the list of buffer states in the given usage scope into this UsageScope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// If the given tracker uses IDs higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn merge_usage_scope( + &mut self, + scope: &Self, + ) -> Result<(), ResourceUsageCompatibilityError> { + let incoming_size = scope.state.len(); + if incoming_size > self.state.len() { + self.set_size(incoming_size); + } + + for index in scope.metadata.owned_indices() { + self.tracker_assert_in_bounds(index); + scope.tracker_assert_in_bounds(index); + + unsafe { + self.insert_or_merge( + index as u32, + index, + BufferStateProvider::Indirect { + state: &scope.state, + }, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + )?; + }; + } + + Ok(()) + } + + /// Merge a single state into the UsageScope. + /// + /// If the resulting state is invalid, returns a usage + /// conflict with the details of the invalid state. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn merge_single( + &mut self, + buffer: &Arc, + new_state: BufferUses, + ) -> Result<(), ResourceUsageCompatibilityError> { + let index = buffer.tracker_index().as_usize(); + + self.allow_index(index); + + self.tracker_assert_in_bounds(index); + + unsafe { + self.insert_or_merge( + index as _, + index, + BufferStateProvider::Direct { state: new_state }, + ResourceMetadataProvider::Direct { resource: buffer }, + )?; + } + + Ok(()) + } + + /// Does an insertion operation if the index isn't tracked + /// in the current metadata, otherwise merges the given state + /// with the current state. If the merging would cause + /// a conflict, returns that usage conflict. + /// + /// # Safety + /// + /// Indexes must be valid indexes into all arrays passed in + /// to this function, either directly or via metadata or provider structs. + #[inline(always)] + unsafe fn insert_or_merge( + &mut self, + index32: u32, + index: usize, + state_provider: BufferStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, Arc>, + ) -> Result<(), ResourceUsageCompatibilityError> { + let currently_owned = unsafe { self.metadata.contains_unchecked(index) }; + + if !currently_owned { + unsafe { + insert( + None, + &mut self.state, + &mut self.metadata, + index, + state_provider, + None, + metadata_provider, + ) + }; + return Ok(()); + } + + unsafe { + merge( + &mut self.state, + index32, + index, + state_provider, + metadata_provider, + ) + } + } + + /// Removes the indicated usage from the scope. + /// + /// Note that multiple uses of the same type get merged. It is only + /// safe to remove a usage if you are certain you aren't going to + /// erase another usage you don't know about. + pub fn remove_usage(&mut self, buffer: &Buffer, usage: BufferUses) { + let index = buffer.tracker_index().as_usize(); + if self.metadata.contains(index) { + // SAFETY: If the buffer is part of this usage scope, then the index + // is in range. + unsafe { + *self.state.get_unchecked_mut(index) &= !usage; + } + } + } +} + +/// Stores all buffer state within a command buffer. +pub(crate) struct BufferTracker { + start: Vec, + end: Vec, + + metadata: ResourceMetadata>, + + temp: Vec>, + + ordered_uses_mask: BufferUses, +} + +impl BufferTracker { + pub fn new(ordered_uses_mask: BufferUses) -> Self { + Self { + start: Vec::new(), + end: Vec::new(), + + metadata: ResourceMetadata::new(), + + temp: Vec::new(), + + ordered_uses_mask, + } + } + + fn tracker_assert_in_bounds(&self, index: usize) { + strict_assert!(index < self.start.len()); + strict_assert!(index < self.end.len()); + self.metadata.tracker_assert_in_bounds(index); + } + + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Buffer ID before + /// all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.start.resize(size, BufferUses::empty()); + self.end.resize(size, BufferUses::empty()); + + self.metadata.set_size(size); + } + + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.start.len() { + self.set_size(index + 1); + } + } + + /// Returns true if the given buffer is tracked. + pub fn contains(&self, buffer: &Buffer) -> bool { + self.metadata.contains(buffer.tracker_index().as_usize()) + } + + /// Returns a list of all buffers tracked. + pub fn used_resources(&self) -> impl Iterator> + '_ { + self.metadata.owned_resources() + } + + /// Drains all currently pending transitions. + pub fn drain_transitions<'a, 'b: 'a>( + &'b mut self, + snatch_guard: &'a SnatchGuard<'a>, + ) -> impl Iterator> { + let buffer_barriers = self.temp.drain(..).map(|pending| { + let buf = unsafe { self.metadata.get_resource_unchecked(pending.id as _) }; + pending.into_hal(buf, snatch_guard) + }); + buffer_barriers + } + + /// Sets the state of a single buffer. + /// + /// If a transition is needed to get the buffer into the given state, that transition + /// is returned. No more than one transition is needed. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_single( + &mut self, + buffer: &Arc, + state: BufferUses, + ) -> Option> { + let index: usize = buffer.tracker_index().as_usize(); + + self.allow_index(index); + + self.tracker_assert_in_bounds(index); + + unsafe { + self.insert_or_barrier_update( + index, + BufferStateProvider::Direct { state }, + None, + ResourceMetadataProvider::Direct { resource: buffer }, + ) + }; + + strict_assert!(self.temp.len() <= 1); + + self.temp.pop() + } + + /// Sets the given state for all buffers in the given tracker. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain_transitions`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_tracker(&mut self, tracker: &Self) { + let incoming_size = tracker.start.len(); + if incoming_size > self.start.len() { + self.set_size(incoming_size); + } + + for index in tracker.metadata.owned_indices() { + self.tracker_assert_in_bounds(index); + tracker.tracker_assert_in_bounds(index); + unsafe { + self.insert_or_barrier_update( + index, + BufferStateProvider::Indirect { + state: &tracker.start, + }, + Some(BufferStateProvider::Indirect { + state: &tracker.end, + }), + ResourceMetadataProvider::Indirect { + metadata: &tracker.metadata, + }, + ) + } + } + } + + /// Sets the given state for all buffers in the given UsageScope. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain_transitions`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_usage_scope(&mut self, scope: &BufferUsageScope) { + let incoming_size = scope.state.len(); + if incoming_size > self.start.len() { + self.set_size(incoming_size); + } + + for index in scope.metadata.owned_indices() { + self.tracker_assert_in_bounds(index); + scope.tracker_assert_in_bounds(index); + unsafe { + self.insert_or_barrier_update( + index, + BufferStateProvider::Indirect { + state: &scope.state, + }, + None, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + ) + } + } + } + + /// Iterates through all buffers in the given bind group and adopts + /// the state given for those buffers in the UsageScope. It also + /// removes all touched buffers from the usage scope. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain_transitions`] is needed to get those transitions. + /// + /// This is a really funky method used by Compute Passes to generate + /// barriers after a call to dispatch without needing to iterate + /// over all elements in the usage scope. We use each the + /// a given iterator of ids as a source of which IDs to look at. + /// All the IDs must have first been added to the usage scope. + /// + /// # Panics + /// + /// If a resource identified by `index_source` is not found in the usage + /// scope. + pub fn set_and_remove_from_usage_scope_sparse( + &mut self, + scope: &mut BufferUsageScope, + index_source: impl IntoIterator, + ) { + let incoming_size = scope.state.len(); + if incoming_size > self.start.len() { + self.set_size(incoming_size); + } + + for index in index_source { + let index = index.as_usize(); + + scope.tracker_assert_in_bounds(index); + + if unsafe { !scope.metadata.contains_unchecked(index) } { + continue; + } + + // SAFETY: we checked that the index is in bounds for the scope, and + // called `set_size` to ensure it is valid for `self`. + unsafe { + self.insert_or_barrier_update( + index, + BufferStateProvider::Indirect { + state: &scope.state, + }, + None, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + ) + }; + + unsafe { scope.metadata.remove(index) }; + } + } + + /// If the resource isn't tracked + /// - Inserts the given resource. + /// - Uses the `start_state_provider` to populate `start_states` + /// - Uses either `end_state_provider` or `start_state_provider` + /// to populate `current_states`. + /// + /// If the resource is tracked + /// - Inserts barriers from the state in `current_states` + /// to the state provided by `start_state_provider`. + /// - Updates the `current_states` with either the state from + /// `end_state_provider` or `start_state_provider`. + /// + /// Any barriers are added to the barrier vector. + /// + /// # Safety + /// + /// Indexes must be valid indexes into all arrays passed in + /// to this function, either directly or via metadata or provider structs. + #[inline(always)] + unsafe fn insert_or_barrier_update( + &mut self, + index: usize, + start_state_provider: BufferStateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, Arc>, + ) { + let currently_owned = unsafe { self.metadata.contains_unchecked(index) }; + + if !currently_owned { + unsafe { + insert( + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index, + start_state_provider, + end_state_provider, + metadata_provider, + ) + }; + return; + } + + let update_state_provider = + end_state_provider.unwrap_or_else(|| start_state_provider.clone()); + unsafe { + barrier( + &mut self.end, + index, + start_state_provider, + &mut self.temp, + self.ordered_uses_mask, + ) + }; + + unsafe { update(&mut self.end, index, update_state_provider) }; + } +} + +/// Stores all buffer state within a device. +pub(crate) struct DeviceBufferTracker { + current_states: Vec, + metadata: ResourceMetadata>, + temp: Vec>, + ordered_uses_mask: BufferUses, +} + +impl DeviceBufferTracker { + pub fn new(ordered_uses_mask: BufferUses) -> Self { + Self { + current_states: Vec::new(), + metadata: ResourceMetadata::new(), + temp: Vec::new(), + ordered_uses_mask, + } + } + + fn tracker_assert_in_bounds(&self, index: usize) { + strict_assert!(index < self.current_states.len()); + self.metadata.tracker_assert_in_bounds(index); + } + + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.current_states.len() { + self.current_states.resize(index + 1, BufferUses::empty()); + self.metadata.set_size(index + 1); + } + } + + /// Returns a list of all buffers tracked. + pub fn used_resources(&self) -> impl Iterator> + '_ { + self.metadata.owned_resources() + } + + /// Inserts a single buffer and its state into the resource tracker. + /// + /// If the resource already exists in the tracker, it will be overwritten. + pub fn insert_single(&mut self, buffer: &Arc, state: BufferUses) { + let index = buffer.tracker_index().as_usize(); + + self.allow_index(index); + + self.tracker_assert_in_bounds(index); + + unsafe { + insert( + None, + &mut self.current_states, + &mut self.metadata, + index, + BufferStateProvider::Direct { state }, + None, + ResourceMetadataProvider::Direct { + resource: &Arc::downgrade(buffer), + }, + ) + } + } + + /// Sets the state of a single buffer. + /// + /// If a transition is needed to get the buffer into the given state, that transition + /// is returned. No more than one transition is needed. + pub fn set_single( + &mut self, + buffer: &Arc, + state: BufferUses, + ) -> Option> { + let index: usize = buffer.tracker_index().as_usize(); + + self.tracker_assert_in_bounds(index); + + let start_state_provider = BufferStateProvider::Direct { state }; + + unsafe { + barrier( + &mut self.current_states, + index, + start_state_provider.clone(), + &mut self.temp, + self.ordered_uses_mask, + ) + }; + unsafe { update(&mut self.current_states, index, start_state_provider) }; + + strict_assert!(self.temp.len() <= 1); + + self.temp.pop() + } + + /// Sets the given state for all buffers in the given tracker. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are returned. + pub fn set_from_tracker_and_drain_transitions<'a, 'b: 'a>( + &'a mut self, + tracker: &'a BufferTracker, + snatch_guard: &'b SnatchGuard<'b>, + ) -> impl Iterator> { + for index in tracker.metadata.owned_indices() { + self.tracker_assert_in_bounds(index); + + let start_state_provider = BufferStateProvider::Indirect { + state: &tracker.start, + }; + let end_state_provider = BufferStateProvider::Indirect { + state: &tracker.end, + }; + unsafe { + barrier( + &mut self.current_states, + index, + start_state_provider, + &mut self.temp, + self.ordered_uses_mask, + ) + }; + unsafe { update(&mut self.current_states, index, end_state_provider) }; + } + + self.temp.drain(..).map(|pending| { + let buf = unsafe { tracker.metadata.get_resource_unchecked(pending.id as _) }; + pending.into_hal(buf, snatch_guard) + }) + } +} + +/// Source of Buffer State. +#[derive(Debug, Clone)] +enum BufferStateProvider<'a> { + /// Get a state that was provided directly. + Direct { state: BufferUses }, + /// Get a state from an an array of states. + Indirect { state: &'a [BufferUses] }, +} +impl BufferStateProvider<'_> { + /// Gets the state from the provider, given a resource ID index. + /// + /// # Safety + /// + /// Index must be in bounds for the indirect source iff this is in the indirect state. + #[inline(always)] + unsafe fn get_state(&self, index: usize) -> BufferUses { + match *self { + BufferStateProvider::Direct { state } => state, + BufferStateProvider::Indirect { state } => { + strict_assert!(index < state.len()); + *unsafe { state.get_unchecked(index) } + } + } + } +} + +#[inline(always)] +unsafe fn insert( + start_states: Option<&mut [BufferUses]>, + current_states: &mut [BufferUses], + resource_metadata: &mut ResourceMetadata, + index: usize, + start_state_provider: BufferStateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, T>, +) { + let new_start_state = unsafe { start_state_provider.get_state(index) }; + let new_end_state = + end_state_provider.map_or(new_start_state, |p| unsafe { p.get_state(index) }); + + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + strict_assert_eq!(invalid_resource_state(new_start_state), false); + strict_assert_eq!(invalid_resource_state(new_end_state), false); + + unsafe { + if let Some(&mut ref mut start_state) = start_states { + *start_state.get_unchecked_mut(index) = new_start_state; + } + *current_states.get_unchecked_mut(index) = new_end_state; + + let resource = metadata_provider.get(index); + resource_metadata.insert(index, resource.clone()); + } +} + +#[inline(always)] +unsafe fn merge( + current_states: &mut [BufferUses], + _index32: u32, + index: usize, + state_provider: BufferStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, Arc>, +) -> Result<(), ResourceUsageCompatibilityError> { + let current_state = unsafe { current_states.get_unchecked_mut(index) }; + let new_state = unsafe { state_provider.get_state(index) }; + + let merged_state = *current_state | new_state; + + if invalid_resource_state(merged_state) { + return Err(ResourceUsageCompatibilityError::from_buffer( + unsafe { metadata_provider.get(index) }, + *current_state, + new_state, + )); + } + + *current_state = merged_state; + + Ok(()) +} + +#[inline(always)] +unsafe fn barrier( + current_states: &mut [BufferUses], + index: usize, + state_provider: BufferStateProvider<'_>, + barriers: &mut Vec>, + ordered_uses_mask: BufferUses, +) { + let current_state = unsafe { *current_states.get_unchecked(index) }; + let new_state = unsafe { state_provider.get_state(index) }; + + if skip_barrier(current_state, ordered_uses_mask, new_state) { + return; + } + + barriers.push(PendingTransition { + id: index as _, + selector: (), + usage: hal::StateTransition { + from: current_state, + to: new_state, + }, + }); +} + +#[inline(always)] +unsafe fn update( + current_states: &mut [BufferUses], + index: usize, + state_provider: BufferStateProvider<'_>, +) { + let current_state = unsafe { current_states.get_unchecked_mut(index) }; + let new_state = unsafe { state_provider.get_state(index) }; + + *current_state = new_state; +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/track/metadata.rs b/third_party/wgpu-core-29.0.4-patched/src/track/metadata.rs new file mode 100644 index 00000000..f946a533 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/track/metadata.rs @@ -0,0 +1,208 @@ +//! The `ResourceMetadata` type. + +use alloc::vec::Vec; + +use bit_vec::BitVec; +use wgt::strict_assert; + +/// A set of resources, holding a `Arc` and epoch for each member. +/// +/// Testing for membership is fast, and iterating over members is +/// reasonably fast in practice. Storage consumption is proportional +/// to the largest id index of any member, not to the number of +/// members, but a bit vector tracks occupancy, so iteration touches +/// only occupied elements. +#[derive(Debug)] +pub(super) struct ResourceMetadata { + /// If the resource with index `i` is a member, `owned[i]` is `true`. + owned: BitVec, + + /// A vector holding clones of members' `T`s. + resources: Vec>, +} + +impl ResourceMetadata { + pub(super) fn new() -> Self { + Self { + owned: BitVec::default(), + resources: Vec::new(), + } + } + + pub(super) fn set_size(&mut self, size: usize) { + self.resources.resize(size, None); + resize_bitvec(&mut self.owned, size); + } + + pub(super) fn clear(&mut self) { + self.resources.clear(); + self.owned.fill(false); + } + + /// Ensures a given index is in bounds for all arrays and does + /// sanity checks of the presence of a refcount. + /// + /// In release mode this function is completely empty and is removed. + pub(super) fn tracker_assert_in_bounds(&self, index: usize) { + strict_assert!(index < self.owned.len()); + strict_assert!(index < self.resources.len()); + strict_assert!(if self.contains(index) { + self.resources[index].is_some() + } else { + true + }); + } + + /// Returns true if the tracker owns no resources. + /// + /// This is a O(n) operation. + pub(super) fn is_empty(&self) -> bool { + !self.owned.any() + } + + /// Returns true if the set contains the resource with the given index. + pub(super) fn contains(&self, index: usize) -> bool { + self.owned.get(index).unwrap_or(false) + } + + /// Returns true if the set contains the resource with the given index. + /// + /// # Safety + /// + /// The given `index` must be in bounds for this `ResourceMetadata`'s + /// existing tables. See `tracker_assert_in_bounds`. + #[inline(always)] + pub(super) unsafe fn contains_unchecked(&self, index: usize) -> bool { + unsafe { self.owned.get(index).unwrap_unchecked() } + } + + /// Insert a resource into the set. + /// + /// Add the resource with the given index, epoch, and reference count to the + /// set. + /// + /// Returns a reference to the newly inserted resource. + /// (This allows avoiding a clone/reference count increase in many cases.) + /// + /// # Safety + /// + /// The given `index` must be in bounds for this `ResourceMetadata`'s + /// existing tables. See `tracker_assert_in_bounds`. + #[inline(always)] + pub(super) unsafe fn insert(&mut self, index: usize, resource: T) -> &T { + self.owned.set(index, true); + let resource_dst = unsafe { self.resources.get_unchecked_mut(index) }; + resource_dst.insert(resource) + } + + /// Get the resource with the given index. + /// + /// # Safety + /// + /// The given `index` must be in bounds for this `ResourceMetadata`'s + /// existing tables. See `tracker_assert_in_bounds`. + #[inline(always)] + pub(super) unsafe fn get_resource_unchecked(&self, index: usize) -> &T { + unsafe { + self.resources + .get_unchecked(index) + .as_ref() + .unwrap_unchecked() + } + } + + /// Returns an iterator over the resources owned by `self`. + pub(super) fn owned_resources(&self) -> impl Iterator + '_ { + if !self.owned.is_empty() { + self.tracker_assert_in_bounds(self.owned.len() - 1) + }; + iterate_bitvec_indices(&self.owned).map(move |index| { + let resource = unsafe { self.resources.get_unchecked(index) }; + resource.as_ref().unwrap() + }) + } + + /// Returns an iterator over the indices of all resources owned by `self`. + pub(super) fn owned_indices(&self) -> impl Iterator + '_ { + if !self.owned.is_empty() { + self.tracker_assert_in_bounds(self.owned.len() - 1) + }; + iterate_bitvec_indices(&self.owned) + } + + /// Remove the resource with the given index from the set. + pub(super) unsafe fn remove(&mut self, index: usize) { + unsafe { + *self.resources.get_unchecked_mut(index) = None; + } + self.owned.set(index, false); + } +} + +/// A source of resource metadata. +/// +/// This is used to abstract over the various places +/// trackers can get new resource metadata from. +pub(super) enum ResourceMetadataProvider<'a, T: Clone> { + /// Comes directly from explicit values. + Direct { resource: &'a T }, + /// Comes from another metadata tracker. + Indirect { metadata: &'a ResourceMetadata }, +} +impl ResourceMetadataProvider<'_, T> { + /// Get a reference to the resource from this. + /// + /// # Safety + /// + /// - The index must be in bounds of the metadata tracker if this uses an indirect source. + #[inline(always)] + pub(super) unsafe fn get(&self, index: usize) -> &T { + match self { + ResourceMetadataProvider::Direct { resource } => resource, + ResourceMetadataProvider::Indirect { metadata } => { + metadata.tracker_assert_in_bounds(index); + { + let resource = unsafe { metadata.resources.get_unchecked(index) }.as_ref(); + unsafe { resource.unwrap_unchecked() } + } + } + } + } +} + +/// Resizes the given bitvec to the given size. I'm not sure why this is hard to do but it is. +fn resize_bitvec(vec: &mut BitVec, size: usize) { + let owned_size_to_grow = size.checked_sub(vec.len()); + if let Some(delta) = owned_size_to_grow { + if delta != 0 { + vec.grow(delta, false); + } + } else { + vec.truncate(size); + } +} + +/// Produces an iterator that yields the indexes of all bits that are set in the bitvec. +/// +/// Will skip entire usize's worth of bits if they are all false. +fn iterate_bitvec_indices(ownership: &BitVec) -> impl Iterator + '_ { + const BITS_PER_BLOCK: usize = usize::BITS as usize; + + let size = ownership.len(); + + ownership + .blocks() + .enumerate() + .filter(|&(_, word)| word != 0) + .flat_map(move |(word_index, mut word)| { + let bit_start = word_index * BITS_PER_BLOCK; + let bit_end = (bit_start + BITS_PER_BLOCK).min(size); + + (bit_start..bit_end).filter(move |_| { + let active = word & 0b1 != 0; + word >>= 1; + + active + }) + }) +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/track/mod.rs b/third_party/wgpu-core-29.0.4-patched/src/track/mod.rs new file mode 100644 index 00000000..7abb8e1a --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/track/mod.rs @@ -0,0 +1,705 @@ +/*! Resource State and Lifetime Trackers + +These structures are responsible for keeping track of resource state, +generating barriers where needednd making sure resources are kept +alive until the trackers die. + +## General Architecture + +Tracking is some of the hottest code in the entire codebase, so the trackers +are designed to be as cache efficient as possible. They store resource state +in flat vectors, storing metadata SOA style, one vector per type of metadata. + +A lot of the tracker code is deeply unsafe, using unchecked accesses all over +to make performance as good as possible. However, for all unsafe accesses, there +is a corresponding debug assert the checks if that access is valid. This helps +get bugs caught fast, while still letting users not need to pay for the bounds +checks. + +In wgpu, each resource ID includes a bitfield holding an index. +Indices are allocated and re-used, so they will always be as low as +reasonably possible. This allows us to use IDs to index into an array +of tracking information. + +## Statefulness + +There are two main types of trackers, stateful and stateless. + +Stateful trackers are for buffers and textures. They both have +resource state attached to them which needs to be used to generate +automatic synchronization. Because of the different requirements of +buffers and textures, they have two separate tracking structures. + +Stateless trackers only store metadata and own the given resource. + +## Use Case + +Within each type of tracker, the trackers are further split into 3 different +use cases, Bind Group, Usage Scopend a full Tracker. + +Bind Group trackers are just a list of different resources, their refcount, +and how they are used. Textures are used via a selector and a usage type. +Buffers by just a usage type. Stateless resources don't have a usage type. + +Usage Scope trackers are only for stateful resources. These trackers represent +a single [`UsageScope`] in the spec. When a use is added to a usage scope, +it is merged with all other uses of that resource in that scope. If there +is a usage conflict, merging will fail and an error will be reported. + +Full trackers represent a before and after state of a resource. These +are used for tracking on the device and on command buffers. The before +state represents the state the resource is first used as in the command buffer, +the after state is the state the command buffer leaves the resource in. +These double ended buffers can then be used to generate the needed transitions +between command buffers. + +## Dense Datastructure with Sparse Data + +This tracking system is based on having completely dense data, but trackers do +not always contain every resource. Some resources (or even most resources) go +unused in any given command buffer. So to help speed up the process of iterating +through possibly thousands of resources, we use a bit vector to represent if +a resource is in the buffer or not. This allows us extremely efficient memory +utilizations well as being able to bail out of whole blocks of 32-64 resources +with a single usize comparison with zero. In practice this means that merging +partially resident buffers is extremely quick. + +The main advantage of this dense datastructure is that we can do merging +of trackers in an extremely efficient fashion that results in us doing linear +scans down a couple of buffers. CPUs and their caches absolutely eat this up. + +## Stateful Resource Operations + +All operations on stateful trackers boil down to one of four operations: +- `insert(tracker, new_state)` adds a resource with a given state to the tracker + for the first time. +- `merge(tracker, new_state)` merges this new state with the previous state, checking + for usage conflicts. +- `barrier(tracker, new_state)` compares the given state to the existing state and + generates the needed barriers. +- `update(tracker, new_state)` takes the given new state and overrides the old state. + +This allows us to compose the operations to form the various kinds of tracker merges +that need to happen in the codebase. For each resource in the given merger, the following +operation applies: + +```text +UsageScope <- Resource = insert(scope, usage) OR merge(scope, usage) +UsageScope <- UsageScope = insert(scope, scope) OR merge(scope, scope) +CommandBuffer <- UsageScope = insert(buffer.start, buffer.end, scope) + OR barrier(buffer.end, scope) + update(buffer.end, scope) +Device <- CommandBuffer = insert(device.start, device.end, buffer.start, buffer.end) + OR barrier(device.end, buffer.start) + update(device.end, buffer.end) +``` + +[`UsageScope`]: https://gpuweb.github.io/gpuweb/#programming-model-synchronization +*/ + +mod blas; +mod buffer; +mod metadata; +mod range; +mod stateless; +mod texture; + +use crate::{ + binding_model, command, + lock::{rank, Mutex}, + pipeline, + resource::{self, Labeled, RawResourceAccess, ResourceErrorIdent}, + snatch::SnatchGuard, + track::blas::BlasTracker, +}; + +use alloc::{sync::Arc, vec::Vec}; +use bitflags::Flags; +use core::{fmt, mem, ops}; + +use thiserror::Error; + +pub(crate) use buffer::{ + BufferBindGroupState, BufferTracker, BufferUsageScope, DeviceBufferTracker, +}; +use metadata::{ResourceMetadata, ResourceMetadataProvider}; +pub(crate) use stateless::StatelessTracker; +pub(crate) use texture::{ + DeviceTextureTracker, TextureTracker, TextureTrackerSetSingle, TextureUsageScope, + TextureViewBindGroupState, +}; +use wgt::{ + error::{ErrorType, WebGpuError}, + strict_assert_ne, +}; + +#[repr(transparent)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) struct TrackerIndex(u32); + +impl TrackerIndex { + pub fn as_usize(self) -> usize { + self.0 as usize + } +} + +/// wgpu-core internally use some array-like storage for tracking resources. +/// To that end, there needs to be a uniquely assigned index for each live resource +/// of a certain type. This index is separate from the resource ID for various reasons: +/// - There can be multiple resource IDs pointing the the same resource. +/// - IDs of dead handles can be recycled while resources are internally held alive (and tracked). +/// - The plan is to remove IDs in the long run +/// ([#5121](https://github.com/gfx-rs/wgpu/issues/5121)). +/// +/// In order to produce these tracker indices, there is a shared TrackerIndexAllocator +/// per resource type. Indices have the same lifetime as the internal resource they +/// are associated to (alloc happens when creating the resource and free is called when +/// the resource is dropped). +struct TrackerIndexAllocator { + unused: Vec, + next_index: TrackerIndex, +} + +impl TrackerIndexAllocator { + pub fn new() -> Self { + TrackerIndexAllocator { + unused: Vec::new(), + next_index: TrackerIndex(0), + } + } + + pub fn alloc(&mut self) -> TrackerIndex { + if let Some(index) = self.unused.pop() { + return index; + } + + let index = self.next_index; + self.next_index.0 += 1; + + index + } + + pub fn free(&mut self, index: TrackerIndex) { + self.unused.push(index); + } + + // This is used to pre-allocate the tracker storage. + pub fn size(&self) -> usize { + self.next_index.0 as usize + } +} + +impl fmt::Debug for TrackerIndexAllocator { + fn fmt(&self, _: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + Ok(()) + } +} + +/// See TrackerIndexAllocator. +#[derive(Debug)] +pub(crate) struct SharedTrackerIndexAllocator { + inner: Mutex, +} + +impl SharedTrackerIndexAllocator { + pub fn new() -> Self { + SharedTrackerIndexAllocator { + inner: Mutex::new( + rank::SHARED_TRACKER_INDEX_ALLOCATOR_INNER, + TrackerIndexAllocator::new(), + ), + } + } + + pub fn alloc(&self) -> TrackerIndex { + self.inner.lock().alloc() + } + + pub fn free(&self, index: TrackerIndex) { + self.inner.lock().free(index); + } + + pub fn size(&self) -> usize { + self.inner.lock().size() + } +} + +pub(crate) struct TrackerIndexAllocators { + pub buffers: Arc, + pub textures: Arc, + pub external_textures: Arc, + pub samplers: Arc, + pub bind_groups: Arc, + pub compute_pipelines: Arc, + pub render_pipelines: Arc, + pub bundles: Arc, + pub query_sets: Arc, + pub blas_s: Arc, + pub tlas_s: Arc, +} + +impl TrackerIndexAllocators { + pub fn new() -> Self { + TrackerIndexAllocators { + buffers: Arc::new(SharedTrackerIndexAllocator::new()), + textures: Arc::new(SharedTrackerIndexAllocator::new()), + external_textures: Arc::new(SharedTrackerIndexAllocator::new()), + samplers: Arc::new(SharedTrackerIndexAllocator::new()), + bind_groups: Arc::new(SharedTrackerIndexAllocator::new()), + compute_pipelines: Arc::new(SharedTrackerIndexAllocator::new()), + render_pipelines: Arc::new(SharedTrackerIndexAllocator::new()), + bundles: Arc::new(SharedTrackerIndexAllocator::new()), + query_sets: Arc::new(SharedTrackerIndexAllocator::new()), + blas_s: Arc::new(SharedTrackerIndexAllocator::new()), + tlas_s: Arc::new(SharedTrackerIndexAllocator::new()), + } + } +} + +/// A structure containing all the information about a particular resource +/// transition. User code should be able to generate a pipeline barrier +/// based on the contents. +#[derive(Debug, PartialEq)] +pub(crate) struct PendingTransition { + pub id: u32, + pub selector: S::Selector, + pub usage: hal::StateTransition, +} + +pub(crate) type PendingTransitionList = Vec>; + +impl PendingTransition { + /// Produce the hal barrier corresponding to the transition. + pub fn into_hal<'a>( + self, + buf: &'a resource::Buffer, + snatch_guard: &'a SnatchGuard<'a>, + ) -> hal::BufferBarrier<'a, dyn hal::DynBuffer> { + let buffer = buf.raw(snatch_guard).expect("Buffer is destroyed"); + hal::BufferBarrier { + buffer, + usage: self.usage, + } + } +} + +impl PendingTransition { + /// Produce the hal barrier corresponding to the transition. + pub fn into_hal( + self, + texture: &dyn hal::DynTexture, + ) -> hal::TextureBarrier<'_, dyn hal::DynTexture> { + // These showing up in a barrier is always a bug + strict_assert_ne!(self.usage.from, wgt::TextureUses::UNKNOWN); + strict_assert_ne!(self.usage.to, wgt::TextureUses::UNKNOWN); + + let mip_count = self.selector.mips.end - self.selector.mips.start; + strict_assert_ne!(mip_count, 0); + let layer_count = self.selector.layers.end - self.selector.layers.start; + strict_assert_ne!(layer_count, 0); + + hal::TextureBarrier { + texture, + range: wgt::ImageSubresourceRange { + aspect: wgt::TextureAspect::All, + base_mip_level: self.selector.mips.start, + mip_level_count: Some(mip_count), + base_array_layer: self.selector.layers.start, + array_layer_count: Some(layer_count), + }, + usage: self.usage, + } + } +} + +/// The uses that a resource or subresource can be in. +pub(crate) trait ResourceUses: + fmt::Debug + ops::BitAnd + ops::BitOr + PartialEq + Sized + Copy +{ + /// All flags that are exclusive. + const EXCLUSIVE: Self; + + /// The selector used by this resource. + type Selector: fmt::Debug; + + /// Turn the resource into a pile of bits. + fn bits(self) -> u16; + /// Returns true if any of the uses are exclusive. + fn any_exclusive(self) -> bool; +} + +/// Returns true if the given states violates the usage scope rule +/// of any(inclusive) XOR one(exclusive) +fn invalid_resource_state(state: T) -> bool { + // Is power of two also means "is one bit set". We check for this as if + // we're in any exclusive state, we must only be in a single state. + state.any_exclusive() && !state.bits().is_power_of_two() +} + +/// Returns true if the transition from one state to another does not require +/// a barrier. +fn skip_barrier(old_state: F, ordered_uses_mask: F, new_state: F) -> bool { + // If the state didn't change and all the usages are ordered, the hardware + // will guarantee the order of accesses, so we do not need to issue a barrier at all + old_state.bits() == new_state.bits() && ordered_uses_mask.contains(old_state) +} + +#[derive(Clone, Debug, Error)] +pub enum ResourceUsageCompatibilityError { + #[error("Attempted to use {res} with {invalid_use}.")] + Buffer { + res: ResourceErrorIdent, + invalid_use: InvalidUse, + }, + #[error( + "Attempted to use {res} (mips {mip_levels:?} layers {array_layers:?}) with {invalid_use}." + )] + Texture { + res: ResourceErrorIdent, + mip_levels: ops::Range, + array_layers: ops::Range, + invalid_use: InvalidUse, + }, +} + +impl WebGpuError for ResourceUsageCompatibilityError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +impl ResourceUsageCompatibilityError { + fn from_buffer( + buffer: &resource::Buffer, + current_state: wgt::BufferUses, + new_state: wgt::BufferUses, + ) -> Self { + Self::Buffer { + res: buffer.error_ident(), + invalid_use: InvalidUse { + current_state, + new_state, + }, + } + } + + fn from_texture( + texture: &resource::Texture, + selector: wgt::TextureSelector, + current_state: wgt::TextureUses, + new_state: wgt::TextureUses, + ) -> Self { + Self::Texture { + res: texture.error_ident(), + mip_levels: selector.mips, + array_layers: selector.layers, + invalid_use: InvalidUse { + current_state, + new_state, + }, + } + } +} + +/// Pretty print helper that shows helpful descriptions of a conflicting usage. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InvalidUse { + current_state: T, + new_state: T, +} + +impl fmt::Display for InvalidUse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let current = self.current_state; + let new = self.new_state; + + let current_exclusive = current & T::EXCLUSIVE; + let new_exclusive = new & T::EXCLUSIVE; + + let exclusive = current_exclusive | new_exclusive; + + // The text starts with "tried to use X resource with {self}" + write!( + f, + "conflicting usages. Current usage {current:?} and new usage {new:?}. \ + {exclusive:?} is an exclusive usage and cannot be used with any other \ + usages within the usage scope (renderpass or compute dispatch)" + ) + } +} + +/// All the usages that a bind group contains. The uses are not deduplicated in any way +/// and may include conflicting uses. This is fully compliant by the WebGPU spec. +/// +/// All bind group states are sorted by their ID so that when adding to a tracker, +/// they are added in the most efficient order possible (ascending order). +#[derive(Debug)] +pub(crate) struct BindGroupStates { + pub buffers: BufferBindGroupState, + pub views: TextureViewBindGroupState, + pub external_textures: StatelessTracker, + pub samplers: StatelessTracker, + pub acceleration_structures: StatelessTracker, +} + +impl BindGroupStates { + pub fn new() -> Self { + Self { + buffers: BufferBindGroupState::new(), + views: TextureViewBindGroupState::new(), + external_textures: StatelessTracker::new(), + samplers: StatelessTracker::new(), + acceleration_structures: StatelessTracker::new(), + } + } + + /// Optimize the bind group states by sorting them by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant ascending order. + pub fn optimize(&mut self) { + self.buffers.optimize(); + // Views are stateless, however, `TextureViewBindGroupState` + // is special as it will be merged with other texture trackers. + self.views.optimize(); + // Samplers and Tlas's are stateless and don't need to be optimized + // since the tracker is never merged with any other tracker. + } +} + +/// This is a render bundle specific usage scope. It includes stateless resources +/// that are not normally included in a usage scope, but are used by render bundles +/// and need to be owned by the render bundles. +#[derive(Debug)] +pub(crate) struct RenderBundleScope { + pub buffers: BufferUsageScope, + pub textures: TextureUsageScope, + // Don't need to track views and samplers, they are never used directly, only by bind groups. + pub bind_groups: StatelessTracker, + pub render_pipelines: StatelessTracker, +} + +impl RenderBundleScope { + /// Create the render bundle scope and pull the maximum IDs from the hubs. + pub fn new() -> Self { + Self { + buffers: BufferUsageScope::default(), + textures: TextureUsageScope::default(), + bind_groups: StatelessTracker::new(), + render_pipelines: StatelessTracker::new(), + } + } + + /// Merge the inner contents of a bind group into the render bundle tracker. + /// + /// Only stateful things are merged in herell other resources are owned + /// indirectly by the bind group. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// length of the storage given at the call to `new`. + pub unsafe fn merge_bind_group( + &mut self, + bind_group: &BindGroupStates, + ) -> Result<(), ResourceUsageCompatibilityError> { + unsafe { self.buffers.merge_bind_group(&bind_group.buffers)? }; + unsafe { self.textures.merge_bind_group(&bind_group.views)? }; + + Ok(()) + } +} + +/// A pool for storing the memory used by [`UsageScope`]s. We take and store this memory when the +/// scope is dropped to avoid reallocating. The memory required only grows and allocation cost is +/// significant when a large number of resources have been used. +pub(crate) type UsageScopePool = Mutex>; + +/// A usage scope tracker. Only needs to store stateful resources as stateless +/// resources cannot possibly have a usage conflict. +#[derive(Debug)] +pub(crate) struct UsageScope<'a> { + pub pool: &'a UsageScopePool, + pub buffers: BufferUsageScope, + pub textures: TextureUsageScope, +} + +impl<'a> Drop for UsageScope<'a> { + fn drop(&mut self) { + // clear vecs and push into pool + self.buffers.clear(); + self.textures.clear(); + self.pool + .lock() + .push((mem::take(&mut self.buffers), mem::take(&mut self.textures))); + } +} + +impl UsageScope<'static> { + pub fn new_pooled<'d>( + pool: &'d UsageScopePool, + tracker_indices: &TrackerIndexAllocators, + ordered_buffer_usages: wgt::BufferUses, + ordered_texture_usages: wgt::TextureUses, + ) -> UsageScope<'d> { + let pooled = pool.lock().pop().unwrap_or_default(); + + let mut scope = UsageScope::<'d> { + pool, + buffers: pooled.0, + textures: pooled.1, + }; + + scope.buffers.set_size(tracker_indices.buffers.size()); + scope.buffers.set_ordered_uses_mask(ordered_buffer_usages); + scope.textures.set_size(tracker_indices.textures.size()); + scope.textures.set_ordered_uses_mask(ordered_texture_usages); + scope + } +} + +impl<'a> UsageScope<'a> { + /// Merge the inner contents of a bind group into the usage scope. + /// + /// Only stateful things are merged in herell other resources are owned + /// indirectly by the bind group. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// length of the storage given at the call to `new`. + pub unsafe fn merge_bind_group( + &mut self, + bind_group: &BindGroupStates, + ) -> Result<(), ResourceUsageCompatibilityError> { + unsafe { + self.buffers.merge_bind_group(&bind_group.buffers)?; + self.textures.merge_bind_group(&bind_group.views)?; + } + + Ok(()) + } + + /// Merge the inner contents of a bind group into the usage scope. + /// + /// Only stateful things are merged in herell other resources are owned + /// indirectly by a bind group or are merged directly into the command buffer tracker. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// length of the storage given at the call to `new`. + pub unsafe fn merge_render_bundle( + &mut self, + render_bundle: &RenderBundleScope, + ) -> Result<(), ResourceUsageCompatibilityError> { + self.buffers.merge_usage_scope(&render_bundle.buffers)?; + self.textures.merge_usage_scope(&render_bundle.textures)?; + + Ok(()) + } +} + +/// A tracker used by Device. +pub(crate) struct DeviceTracker { + pub buffers: DeviceBufferTracker, + pub textures: DeviceTextureTracker, +} + +impl DeviceTracker { + pub fn new( + ordered_buffer_usages: wgt::BufferUses, + ordered_texture_usages: wgt::TextureUses, + ) -> Self { + Self { + buffers: DeviceBufferTracker::new(ordered_buffer_usages), + textures: DeviceTextureTracker::new(ordered_texture_usages), + } + } +} + +/// A full double sided tracker used by CommandBuffers. +pub(crate) struct Tracker { + /// Buffers used within this command buffer. + /// + /// For compute passes, this only includes buffers actually used by the + /// pipeline (contrast with the `bind_groups` member). + pub buffers: BufferTracker, + + /// Textures used within this command buffer. + /// + /// For compute passes, this only includes textures actually used by the + /// pipeline (contrast with the `bind_groups` member). + pub textures: TextureTracker, + + pub blas_s: BlasTracker, + pub tlas_s: StatelessTracker, + pub views: StatelessTracker, + + /// Contains all bind groups that were passed in any call to + /// `set_bind_group` on the encoder. + /// + /// WebGPU requires that submission fails if any resource in any of these + /// bind groups is destroyed, even if the resource is not actually used by + /// the pipeline (e.g. because the pipeline does not use the bound slot, or + /// because the bind group was replaced by a subsequent call to + /// `set_bind_group`). + pub bind_groups: StatelessTracker, + + pub compute_pipelines: StatelessTracker, + pub render_pipelines: StatelessTracker, + pub bundles: StatelessTracker, + pub query_sets: StatelessTracker, +} + +impl Tracker { + pub fn new( + ordered_buffer_usages: wgt::BufferUses, + ordered_texture_usages: wgt::TextureUses, + ) -> Self { + Self { + buffers: BufferTracker::new(ordered_buffer_usages), + textures: TextureTracker::new(ordered_texture_usages), + blas_s: BlasTracker::new(), + tlas_s: StatelessTracker::new(), + views: StatelessTracker::new(), + bind_groups: StatelessTracker::new(), + compute_pipelines: StatelessTracker::new(), + render_pipelines: StatelessTracker::new(), + bundles: StatelessTracker::new(), + query_sets: StatelessTracker::new(), + } + } + + /// Iterates through all resources in the given bind group and adopts + /// the state given for those resources in the UsageScope. It also + /// removes all touched resources from the usage scope. + /// + /// If a transition is needed to get the resources into the needed + /// state, those transitions are stored within the tracker. A + /// subsequent call to [`BufferTracker::drain_transitions`] or + /// [`TextureTracker::drain_transitions`] is needed to get those transitions. + /// + /// This is a really funky method used by Compute Passes to generate + /// barriers after a call to dispatch without needing to iterate + /// over all elements in the usage scope. We use each the + /// bind group as a source of which IDs to look at. The bind groups + /// must have first been added to the usage scope. + /// + /// Only stateful things are merged in here, all other resources are owned + /// indirectly by the bind group. + /// + /// # Panics + /// + /// If a resource in the `bind_group` is not found in the usage scope. + pub fn set_and_remove_from_usage_scope_sparse( + &mut self, + scope: &mut UsageScope, + bind_group: &BindGroupStates, + ) { + self.buffers.set_and_remove_from_usage_scope_sparse( + &mut scope.buffers, + bind_group.buffers.used_tracker_indices(), + ); + self.textures + .set_and_remove_from_usage_scope_sparse(&mut scope.textures, &bind_group.views); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/track/range.rs b/third_party/wgpu-core-29.0.4-patched/src/track/range.rs new file mode 100644 index 00000000..4ee5bcfd --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/track/range.rs @@ -0,0 +1,206 @@ +//Note: this could be the only place where we need `SmallVec`. +//TODO: consider getting rid of it. +use smallvec::SmallVec; + +use core::{fmt::Debug, iter, ops::Range}; + +/// Structure that keeps track of a I -> T mapping, +/// optimized for a case where keys of the same values +/// are often grouped together linearly. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct RangedStates { + /// List of ranges, each associated with a singe value. + /// Ranges of keys have to be non-intersecting and ordered. + ranges: SmallVec<[(Range, T); 1]>, +} + +impl RangedStates { + pub fn from_range(range: Range, value: T) -> Self { + Self { + ranges: iter::once((range, value)).collect(), + } + } + + /// Construct a new instance from a slice of ranges. + #[cfg(test)] + pub fn from_slice(values: &[(Range, T)]) -> Self { + Self { + ranges: values.iter().cloned().collect(), + } + } + + pub fn iter(&self) -> impl Iterator, T)> + Clone { + self.ranges.iter() + } + + pub fn iter_mut(&mut self) -> impl Iterator, T)> { + self.ranges.iter_mut() + } + + /// Check that all the ranges are non-intersecting and ordered. + /// Panics otherwise. + #[cfg(test)] + fn check_sanity(&self) { + for a in self.ranges.iter() { + assert!(a.0.start < a.0.end); + } + for (a, b) in self.ranges.iter().zip(self.ranges[1..].iter()) { + assert!(a.0.end <= b.0.start); + } + } + + /// Merge the neighboring ranges together, where possible. + pub fn coalesce(&mut self) { + let mut num_removed = 0; + let mut iter = self.ranges.iter_mut(); + let mut cur = match iter.next() { + Some(elem) => elem, + None => return, + }; + for next in iter { + if cur.0.end == next.0.start && cur.1 == next.1 { + num_removed += 1; + cur.0.end = next.0.end; + next.0.end = next.0.start; + } else { + cur = next; + } + } + if num_removed != 0 { + self.ranges.retain(|pair| pair.0.start != pair.0.end); + } + } + + pub fn iter_filter<'a>( + &'a self, + range: &'a Range, + ) -> impl Iterator, &'a T)> + 'a { + self.ranges + .iter() + .filter(move |&(inner, ..)| inner.end > range.start && inner.start < range.end) + .map(move |(inner, v)| { + let new_range = inner.start.max(range.start)..inner.end.min(range.end); + + (new_range, v) + }) + } + + /// Split the storage ranges in such a way that there is a linear subset of + /// them occupying exactly `index` range, which is returned mutably. + /// + /// Gaps in the ranges are filled with `default` value. + pub fn isolate(&mut self, index: &Range, default: T) -> &mut [(Range, T)] { + //TODO: implement this in 2 passes: + // 1. scan the ranges to figure out how many extra ones need to be inserted + // 2. go through the ranges by moving them them to the right and inserting the missing ones + + let mut start_pos = match self.ranges.iter().position(|pair| pair.0.end > index.start) { + Some(pos) => pos, + None => { + let pos = self.ranges.len(); + self.ranges.push((index.clone(), default)); + return &mut self.ranges[pos..]; + } + }; + + { + let (range, value) = self.ranges[start_pos].clone(); + if range.start < index.start { + self.ranges[start_pos].0.start = index.start; + self.ranges + .insert(start_pos, (range.start..index.start, value)); + start_pos += 1; + } + } + let mut pos = start_pos; + let mut range_pos = index.start; + loop { + let (range, value) = self.ranges[pos].clone(); + if range.start >= index.end { + self.ranges.insert(pos, (range_pos..index.end, default)); + pos += 1; + break; + } + if range.start > range_pos { + self.ranges.insert(pos, (range_pos..range.start, default)); + pos += 1; + range_pos = range.start; + } + if range.end >= index.end { + if range.end != index.end { + self.ranges[pos].0.start = index.end; + self.ranges.insert(pos, (range_pos..index.end, value)); + } + pos += 1; + break; + } + pos += 1; + range_pos = range.end; + if pos == self.ranges.len() { + self.ranges.push((range_pos..index.end, default)); + pos += 1; + break; + } + } + + &mut self.ranges[start_pos..pos] + } + + /// Helper method for isolation that checks the sanity of the results. + #[cfg(test)] + pub fn sanely_isolated(&self, index: Range, default: T) -> alloc::vec::Vec<(Range, T)> { + let mut clone = self.clone(); + let result = clone.isolate(&index, default).to_vec(); + clone.check_sanity(); + result + } +} + +#[cfg(test)] +mod test { + //TODO: randomized/fuzzy testing + use super::RangedStates; + + #[test] + fn sane_good() { + let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9)]); + rs.check_sanity(); + } + + #[test] + #[should_panic] + fn sane_empty() { + let rs = RangedStates::from_slice(&[(1..4, 9u8), (5..5, 9)]); + rs.check_sanity(); + } + + #[test] + #[should_panic] + fn sane_intersect() { + let rs = RangedStates::from_slice(&[(1..4, 9u8), (3..5, 9)]); + rs.check_sanity(); + } + + #[test] + fn coalesce() { + let mut rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); + rs.coalesce(); + rs.check_sanity(); + assert_eq!(rs.ranges.as_slice(), &[(1..5, 9), (5..7, 1), (8..9, 1),]); + } + + #[test] + fn isolate() { + let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); + assert_eq!(&rs.sanely_isolated(4..5, 0), &[(4..5, 9u8),]); + assert_eq!( + &rs.sanely_isolated(0..6, 0), + &[(0..1, 0), (1..4, 9u8), (4..5, 9), (5..6, 1),] + ); + assert_eq!(&rs.sanely_isolated(8..10, 1), &[(8..9, 1), (9..10, 1),]); + assert_eq!( + &rs.sanely_isolated(6..9, 0), + &[(6..7, 1), (7..8, 0), (8..9, 1),] + ); + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/track/stateless.rs b/third_party/wgpu-core-29.0.4-patched/src/track/stateless.rs new file mode 100644 index 00000000..26aec930 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/track/stateless.rs @@ -0,0 +1,36 @@ +use alloc::{sync::Arc, vec::Vec}; +use core::slice::Iter; + +/// A tracker that holds strong references to resources. +/// +/// This is only used to keep resources alive. +#[derive(Debug)] +pub(crate) struct StatelessTracker { + resources: Vec>, +} + +impl StatelessTracker { + pub fn new() -> Self { + Self { + resources: Vec::new(), + } + } + + /// Inserts a single resource into the resource tracker. + /// + /// Returns a reference to the newly inserted resource. + /// (This allows avoiding a clone/reference count increase in many cases.) + pub fn insert_single(&mut self, resource: Arc) -> &Arc { + self.resources.push(resource); + unsafe { self.resources.last().unwrap_unchecked() } + } +} + +impl<'a, T> IntoIterator for &'a StatelessTracker { + type Item = &'a Arc; + type IntoIter = Iter<'a, Arc>; + + fn into_iter(self) -> Self::IntoIter { + self.resources.as_slice().iter() + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/track/texture.rs b/third_party/wgpu-core-29.0.4-patched/src/track/texture.rs new file mode 100644 index 00000000..aede0428 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/track/texture.rs @@ -0,0 +1,1547 @@ +//! Texture Trackers +//! +//! Texture trackers are significantly more complicated than +//! the buffer trackers because textures can be in a "complex" +//! state where each individual subresource can potentially be +//! in a different state from every other subtresource. These +//! complex states are stored separately from the simple states +//! because they are signifignatly more difficult to track and +//! most resources spend the vast majority of their lives in +//! simple states. +//! +//! There are two special texture usages: `UNKNOWN` and `UNINITIALIZED`. +//! - `UNKNOWN` is only used in complex states and is used to signify +//! that the complex state does not know anything about those subresources. +//! It cannot leak into transitions, it is invalid to transition into UNKNOWN +//! state. +//! - `UNINITIALIZED` is used in both simple and complex states to mean the texture +//! is known to be in some undefined state. Any transition away from UNINITIALIZED +//! will treat the contents as junk. + +use super::{range::RangedStates, PendingTransition, PendingTransitionList}; +use crate::{ + resource::{RawResourceAccess, Texture, TextureInner, TextureView, Trackable}, + snatch::SnatchGuard, + track::{ + invalid_resource_state, skip_barrier, ResourceMetadata, ResourceMetadataProvider, + ResourceUsageCompatibilityError, ResourceUses, + }, +}; +use hal::TextureBarrier; + +use arrayvec::ArrayVec; +use naga::FastHashMap; + +use wgt::{strict_assert, strict_assert_eq, TextureSelector, TextureUses}; + +use alloc::{ + sync::{Arc, Weak}, + vec::{Drain, Vec}, +}; +use core::iter; + +impl ResourceUses for TextureUses { + const EXCLUSIVE: Self = Self::EXCLUSIVE; + + type Selector = TextureSelector; + + fn bits(self) -> u16 { + Self::bits(&self) + } + + fn any_exclusive(self) -> bool { + self.intersects(Self::EXCLUSIVE) + } +} + +/// Represents the complex state of textures where every subresource is potentially +/// in a different state. +#[derive(Clone, Debug, Default, PartialEq)] +struct ComplexTextureState { + mips: ArrayVec, { hal::MAX_MIP_LEVELS as usize }>, +} + +impl ComplexTextureState { + /// Creates complex texture state for the given sizes. + /// + /// This state will be initialized with the UNKNOWN state, a special state + /// which means the trakcer knows nothing about the state. + fn new(mip_level_count: u32, array_layer_count: u32) -> Self { + Self { + mips: iter::repeat_with(|| { + RangedStates::from_range(0..array_layer_count, TextureUses::UNKNOWN) + }) + .take(mip_level_count as usize) + .collect(), + } + } + + /// Initialize a complex state from a selector representing the full size of the texture + /// and an iterator of a selector and a texture use, specifying a usage for a specific + /// set of subresources. + /// + /// [`Self::to_selector_state_iter`] can be used to create such an iterator. + /// + /// # Safety + /// + /// All selectors in the iterator must be inside of the full_range selector. + /// + /// The full range selector must have mips and layers start at 0. + unsafe fn from_selector_state_iter( + full_range: TextureSelector, + state_iter: impl Iterator, + ) -> Self { + strict_assert_eq!(full_range.layers.start, 0); + strict_assert_eq!(full_range.mips.start, 0); + + let mut complex = + ComplexTextureState::new(full_range.mips.len() as u32, full_range.layers.len() as u32); + for (selector, desired_state) in state_iter { + strict_assert!(selector.layers.end <= full_range.layers.end); + strict_assert!(selector.mips.end <= full_range.mips.end); + + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + strict_assert_eq!(invalid_resource_state(desired_state), false); + + let mips = selector.mips.start as usize..selector.mips.end as usize; + for mip in unsafe { complex.mips.get_unchecked_mut(mips) } { + for &mut (_, ref mut state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { + *state = desired_state; + } + } + } + complex + } + + /// Convert a complex state into an iterator over all states stored. + /// + /// [`Self::from_selector_state_iter`] can be used to consume such an iterator. + fn to_selector_state_iter( + &self, + ) -> impl Iterator + Clone + '_ { + self.mips.iter().enumerate().flat_map(|(mip, inner)| { + let mip = mip as u32; + { + inner.iter().map(move |&(ref layers, inner)| { + ( + TextureSelector { + mips: mip..mip + 1, + layers: layers.clone(), + }, + inner, + ) + }) + } + }) + } +} + +/// Stores a bind group's texture views + their usages (within the bind group). +#[derive(Debug)] +pub(crate) struct TextureViewBindGroupState { + views: Vec<(Arc, TextureUses)>, +} +impl TextureViewBindGroupState { + pub fn new() -> Self { + Self { views: Vec::new() } + } + + /// Optimize the texture bind group state by sorting it by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant ascending order. + pub(crate) fn optimize(&mut self) { + self.views + .sort_unstable_by_key(|(view, _)| view.parent.tracker_index()); + } + + /// Adds the given resource with the given state. + pub fn insert_single(&mut self, view: Arc, usage: TextureUses) { + self.views.push((view, usage)); + } +} + +/// Container for corresponding simple and complex texture states. +#[derive(Debug)] +pub(crate) struct TextureStateSet { + simple: Vec, + complex: FastHashMap, +} + +impl TextureStateSet { + fn new() -> Self { + Self { + simple: Vec::new(), + complex: FastHashMap::default(), + } + } + + fn clear(&mut self) { + self.simple.clear(); + self.complex.clear(); + } + + fn set_size(&mut self, size: usize) { + self.simple.resize(size, TextureUses::UNINITIALIZED); + } + + fn size(&self) -> usize { + self.simple.len() + } + + /// SAFETY: `index` must be in bounds. + unsafe fn get_unchecked( + &self, + index: usize, + ) -> SingleOrManyStates { + let simple = unsafe { *self.simple.get_unchecked(index) }; + if simple == TextureUses::COMPLEX { + SingleOrManyStates::Many(unsafe { self.complex.get(&index).unwrap_unchecked() }) + } else { + SingleOrManyStates::Single(simple) + } + } + + /// # Safety + /// + /// The `index` must be in bounds. + unsafe fn get_mut_unchecked( + &mut self, + index: usize, + ) -> SingleOrManyStates<&mut TextureUses, &mut ComplexTextureState> { + let simple = unsafe { self.simple.get_unchecked_mut(index) }; + if *simple == TextureUses::COMPLEX { + SingleOrManyStates::Many(unsafe { self.complex.get_mut(&index).unwrap_unchecked() }) + } else { + SingleOrManyStates::Single(simple) + } + } + + /// # Safety + /// + /// The `index` must be in bounds. + unsafe fn insert_simple_unchecked(&mut self, index: usize, simple: TextureUses) { + unsafe { *self.simple.get_unchecked_mut(index) = simple }; + } + + /// # Safety + /// + /// The `index` must be in bounds. + unsafe fn insert_complex_unchecked(&mut self, index: usize, complex: ComplexTextureState) { + unsafe { *self.simple.get_unchecked_mut(index) = TextureUses::COMPLEX }; + self.complex.insert(index, complex); + } + + /// # Safety + /// + /// The `index` must be in bounds. + unsafe fn make_simple_unchecked(&mut self, index: usize, simple: TextureUses) { + unsafe { *self.simple.get_unchecked_mut(index) = simple }; + unsafe { self.complex.remove(&index).unwrap_unchecked() }; + } + + /// # Safety + /// + /// The `index` must be in bounds. + unsafe fn make_complex_unchecked(&mut self, index: usize, complex: ComplexTextureState) { + unsafe { *self.simple.get_unchecked_mut(index) = TextureUses::COMPLEX }; + self.complex.insert(index, complex); + } + + fn tracker_assert_in_bounds(&self, index: usize) { + strict_assert!(index < self.size()); + } +} + +/// Stores all texture state within a single usage scope. +#[derive(Debug)] +pub(crate) struct TextureUsageScope { + set: TextureStateSet, + metadata: ResourceMetadata>, + ordered_uses_mask: TextureUses, +} + +impl Default for TextureUsageScope { + fn default() -> Self { + Self { + set: TextureStateSet::new(), + metadata: ResourceMetadata::new(), + ordered_uses_mask: TextureUses::empty(), + } + } +} + +impl TextureUsageScope { + fn tracker_assert_in_bounds(&self, index: usize) { + self.metadata.tracker_assert_in_bounds(index); + self.set.tracker_assert_in_bounds(index); + } + + pub fn clear(&mut self) { + self.set.clear(); + self.metadata.clear(); + } + + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Texture ID before + /// all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.set.set_size(size); + self.metadata.set_size(size); + } + + pub fn set_ordered_uses_mask(&mut self, ordered_uses_mask: TextureUses) { + self.ordered_uses_mask = ordered_uses_mask; + } + + /// Returns true if the tracker owns no resources. + /// + /// This is a O(n) operation. + pub(crate) fn is_empty(&self) -> bool { + self.metadata.is_empty() + } + + /// Merge the list of texture states in the given usage scope into this UsageScope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// If the given tracker uses IDs higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn merge_usage_scope( + &mut self, + scope: &Self, + ) -> Result<(), ResourceUsageCompatibilityError> { + let incoming_size = scope.set.size(); + if incoming_size > self.set.size() { + self.set_size(incoming_size); + } + + for index in scope.metadata.owned_indices() { + self.tracker_assert_in_bounds(index); + scope.tracker_assert_in_bounds(index); + + let texture_selector = + unsafe { &scope.metadata.get_resource_unchecked(index).full_range }; + unsafe { + insert_or_merge( + texture_selector, + &mut self.set, + &mut self.metadata, + index, + TextureStateProvider::TextureSet { set: &scope.set }, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + )? + }; + } + + Ok(()) + } + + /// Merge the list of texture states in the given bind group into this usage scope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// Because bind groups do not check if the union of all their states is valid, + /// this method is allowed to return Err on the first bind group bound. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn merge_bind_group( + &mut self, + bind_group: &TextureViewBindGroupState, + ) -> Result<(), ResourceUsageCompatibilityError> { + for (view, usage) in bind_group.views.iter() { + unsafe { self.merge_single(&view.parent, Some(view.selector.clone()), *usage)? }; + } + + Ok(()) + } + + /// Merge a single state into the UsageScope. + /// + /// If the resulting state is invalid, returns a usage + /// conflict with the details of the invalid state. + /// + /// # Safety + /// + /// Unlike other trackers whose merge_single is safe, this method is only + /// called where there is already other unsafe tracking functions active, + /// so we can prove this unsafe "for free". + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn merge_single( + &mut self, + texture: &Arc, + selector: Option, + new_state: TextureUses, + ) -> Result<(), ResourceUsageCompatibilityError> { + let index = texture.tracker_index().as_usize(); + + self.tracker_assert_in_bounds(index); + + let texture_selector = &texture.full_range; + unsafe { + insert_or_merge( + texture_selector, + &mut self.set, + &mut self.metadata, + index, + TextureStateProvider::from_option(selector, new_state), + ResourceMetadataProvider::Direct { resource: texture }, + )? + }; + + Ok(()) + } +} + +pub(crate) trait TextureTrackerSetSingle { + fn set_single( + &mut self, + texture: &Arc, + selector: TextureSelector, + new_state: TextureUses, + ) -> Drain<'_, PendingTransition>; +} + +/// Stores all texture state within a command buffer. +pub(crate) struct TextureTracker { + start_set: TextureStateSet, + end_set: TextureStateSet, + + metadata: ResourceMetadata>, + + temp: Vec>, + + ordered_uses_mask: TextureUses, +} + +impl TextureTracker { + pub fn new(ordered_uses_mask: TextureUses) -> Self { + Self { + start_set: TextureStateSet::new(), + end_set: TextureStateSet::new(), + + metadata: ResourceMetadata::new(), + + temp: Vec::new(), + + ordered_uses_mask, + } + } + + fn tracker_assert_in_bounds(&self, index: usize) { + self.metadata.tracker_assert_in_bounds(index); + self.start_set.tracker_assert_in_bounds(index); + self.end_set.tracker_assert_in_bounds(index); + } + + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Texture ID before + /// all unsafe functions are called. + pub fn set_size(&mut self, size: usize) { + self.start_set.set_size(size); + self.end_set.set_size(size); + + self.metadata.set_size(size); + } + + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.start_set.size() { + self.set_size(index + 1); + } + } + + /// Returns true if the tracker owns the given texture. + pub fn contains(&self, texture: &Texture) -> bool { + self.metadata.contains(texture.tracker_index().as_usize()) + } + + /// Returns a list of all textures tracked. + pub fn used_resources(&self) -> impl Iterator> + '_ { + self.metadata.owned_resources() + } + /// Drain all currently pending transitions. + pub fn drain_transitions<'a>( + &'a mut self, + snatch_guard: &'a SnatchGuard<'a>, + ) -> (PendingTransitionList, Vec>) { + let mut textures = Vec::new(); + let transitions = self + .temp + .drain(..) + .inspect(|pending| { + let tex = unsafe { self.metadata.get_resource_unchecked(pending.id as _) }; + textures.push(tex.inner.get(snatch_guard)); + }) + .collect(); + (transitions, textures) + } + + /// Sets the state of a single texture. + /// + /// If a transition is needed to get the texture into the given state, that transition + /// is returned. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_single( + &mut self, + texture: &Arc, + selector: TextureSelector, + new_state: TextureUses, + ) -> Drain<'_, PendingTransition> { + let index = texture.tracker_index().as_usize(); + + self.allow_index(index); + + self.tracker_assert_in_bounds(index); + + unsafe { + insert_or_barrier_update( + &texture.full_range, + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index, + TextureStateProvider::Selector { + selector, + state: new_state, + }, + None, + ResourceMetadataProvider::Direct { resource: texture }, + &mut self.temp, + self.ordered_uses_mask, + ) + } + + self.temp.drain(..) + } + + /// Sets the given state for all texture in the given tracker. + /// + /// If a transition is needed to get the texture into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain_transitions`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_tracker(&mut self, tracker: &Self) { + let incoming_size = tracker.start_set.size(); + if incoming_size > self.start_set.size() { + self.set_size(incoming_size); + } + + for index in tracker.metadata.owned_indices() { + self.tracker_assert_in_bounds(index); + tracker.tracker_assert_in_bounds(index); + unsafe { + let texture_selector = &tracker.metadata.get_resource_unchecked(index).full_range; + insert_or_barrier_update( + texture_selector, + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index, + TextureStateProvider::TextureSet { + set: &tracker.start_set, + }, + Some(TextureStateProvider::TextureSet { + set: &tracker.end_set, + }), + ResourceMetadataProvider::Indirect { + metadata: &tracker.metadata, + }, + &mut self.temp, + self.ordered_uses_mask, + ); + } + } + } + + /// Sets the given state for all textures in the given UsageScope. + /// + /// If a transition is needed to get the textures into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain_transitions`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_usage_scope(&mut self, scope: &TextureUsageScope) { + let incoming_size = scope.set.size(); + if incoming_size > self.start_set.size() { + self.set_size(incoming_size); + } + + for index in scope.metadata.owned_indices() { + self.tracker_assert_in_bounds(index); + scope.tracker_assert_in_bounds(index); + unsafe { + let texture_selector = &scope.metadata.get_resource_unchecked(index).full_range; + insert_or_barrier_update( + texture_selector, + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index, + TextureStateProvider::TextureSet { set: &scope.set }, + None, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + &mut self.temp, + self.ordered_uses_mask, + ); + } + } + } + + /// Iterates through all textures in the given bind group and adopts + /// the state given for those textures in the UsageScope. It also + /// removes all touched textures from the usage scope. + /// + /// If a transition is needed to get the textures into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain_transitions`] is needed to get those transitions. + /// + /// This is a really funky method used by Compute Passes to generate + /// barriers after a call to dispatch without needing to iterate + /// over all elements in the usage scope. We use each the + /// bind group as a source of which IDs to look at. The bind groups + /// must have first been added to the usage scope. + /// + /// # Panics + /// + /// If a resource in `bind_group_state` is not found in the usage scope. + pub fn set_and_remove_from_usage_scope_sparse( + &mut self, + scope: &mut TextureUsageScope, + bind_group_state: &TextureViewBindGroupState, + ) { + let incoming_size = scope.set.size(); + if incoming_size > self.start_set.size() { + self.set_size(incoming_size); + } + + for (view, _) in bind_group_state.views.iter() { + let index = view.parent.tracker_index().as_usize(); + scope.tracker_assert_in_bounds(index); + + if unsafe { !scope.metadata.contains_unchecked(index) } { + continue; + } + let texture_selector = &view.parent.full_range; + // SAFETY: we checked that the index is in bounds for the scope, and + // called `set_size` to ensure it is valid for `self`. + unsafe { + insert_or_barrier_update( + texture_selector, + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index, + TextureStateProvider::TextureSet { set: &scope.set }, + None, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + &mut self.temp, + self.ordered_uses_mask, + ) + }; + + unsafe { scope.metadata.remove(index) }; + } + } +} + +impl TextureTrackerSetSingle for TextureTracker { + fn set_single( + &mut self, + texture: &Arc, + selector: TextureSelector, + new_state: TextureUses, + ) -> Drain<'_, PendingTransition> { + self.set_single(texture, selector, new_state) + } +} + +/// Stores all texture state within a device. +pub(crate) struct DeviceTextureTracker { + current_state_set: TextureStateSet, + metadata: ResourceMetadata>, + temp: Vec>, + ordered_uses_mask: TextureUses, +} + +impl DeviceTextureTracker { + pub fn new(ordered_uses_mask: TextureUses) -> Self { + Self { + current_state_set: TextureStateSet::new(), + metadata: ResourceMetadata::new(), + temp: Vec::new(), + ordered_uses_mask, + } + } + + fn tracker_assert_in_bounds(&self, index: usize) { + self.metadata.tracker_assert_in_bounds(index); + self.current_state_set.tracker_assert_in_bounds(index); + } + + /// Extend the vectors to let the given index be valid. + fn allow_index(&mut self, index: usize) { + if index >= self.current_state_set.size() { + self.current_state_set.set_size(index + 1); + self.metadata.set_size(index + 1); + } + } + + /// Returns a list of all textures tracked. + pub fn used_resources(&self) -> impl Iterator> + '_ { + self.metadata.owned_resources() + } + + /// Inserts a single texture and a state into the resource tracker. + /// + /// If the resource already exists in the tracker, it will be overwritten. + pub fn insert_single(&mut self, texture: &Arc, state: TextureUses) { + let index = texture.tracker_index().as_usize(); + + self.allow_index(index); + + self.tracker_assert_in_bounds(index); + + unsafe { + insert( + None, + None, + &mut self.current_state_set, + &mut self.metadata, + index, + TextureStateProvider::KnownSingle { state }, + None, + ResourceMetadataProvider::Direct { + resource: &Arc::downgrade(texture), + }, + ) + }; + } + + /// Sets the state of a single texture. + /// + /// If a transition is needed to get the texture into the given state, that transition + /// is returned. + pub fn set_single( + &mut self, + texture: &Arc, + selector: TextureSelector, + new_state: TextureUses, + ) -> Drain<'_, PendingTransition> { + let index = texture.tracker_index().as_usize(); + + self.allow_index(index); + + self.tracker_assert_in_bounds(index); + + let start_state_provider = TextureStateProvider::Selector { + selector, + state: new_state, + }; + unsafe { + barrier( + &texture.full_range, + &self.current_state_set, + index, + start_state_provider.clone(), + &mut self.temp, + self.ordered_uses_mask, + ) + }; + unsafe { + update( + &texture.full_range, + None, + &mut self.current_state_set, + index, + start_state_provider, + ) + }; + + self.temp.drain(..) + } + + /// Sets the given state for all texture in the given tracker. + /// + /// If a transition is needed to get the texture into the needed state, + /// those transitions are returned. + pub fn set_from_tracker_and_drain_transitions<'a, 'b: 'a>( + &'a mut self, + tracker: &'a TextureTracker, + snatch_guard: &'b SnatchGuard<'b>, + ) -> impl Iterator> { + for index in tracker.metadata.owned_indices() { + self.tracker_assert_in_bounds(index); + + let start_state_provider = TextureStateProvider::TextureSet { + set: &tracker.start_set, + }; + let end_state_provider = TextureStateProvider::TextureSet { + set: &tracker.end_set, + }; + unsafe { + let texture_selector = &tracker.metadata.get_resource_unchecked(index).full_range; + barrier( + texture_selector, + &self.current_state_set, + index, + start_state_provider, + &mut self.temp, + self.ordered_uses_mask, + ); + update( + texture_selector, + None, + &mut self.current_state_set, + index, + end_state_provider, + ); + } + } + + self.temp.drain(..).map(|pending| { + let tex = unsafe { tracker.metadata.get_resource_unchecked(pending.id as _) }; + let tex = tex.try_raw(snatch_guard).unwrap(); + pending.into_hal(tex) + }) + } + + /// Sets the given state for all textures in the given UsageScope. + /// + /// If a transition is needed to get the textures into the needed state, + /// those transitions are returned. + pub fn set_from_usage_scope_and_drain_transitions<'a, 'b: 'a>( + &'a mut self, + scope: &'a TextureUsageScope, + snatch_guard: &'b SnatchGuard<'b>, + ) -> impl Iterator> { + for index in scope.metadata.owned_indices() { + self.tracker_assert_in_bounds(index); + + let start_state_provider = TextureStateProvider::TextureSet { set: &scope.set }; + unsafe { + let texture_selector = &scope.metadata.get_resource_unchecked(index).full_range; + barrier( + texture_selector, + &self.current_state_set, + index, + start_state_provider.clone(), + &mut self.temp, + self.ordered_uses_mask, + ); + update( + texture_selector, + None, + &mut self.current_state_set, + index, + start_state_provider, + ); + } + } + + self.temp.drain(..).map(|pending| { + let tex = unsafe { scope.metadata.get_resource_unchecked(pending.id as _) }; + let tex = tex.try_raw(snatch_guard).unwrap(); + pending.into_hal(tex) + }) + } +} + +impl TextureTrackerSetSingle for DeviceTextureTracker { + fn set_single( + &mut self, + texture: &Arc, + selector: TextureSelector, + new_state: TextureUses, + ) -> Drain<'_, PendingTransition> { + self.set_single(texture, selector, new_state) + } +} + +/// An iterator adapter that can store two different iterator types. +#[derive(Clone)] +enum EitherIter { + Left(L), + Right(R), +} + +impl Iterator for EitherIter +where + L: Iterator, + R: Iterator, +{ + type Item = D; + + fn next(&mut self) -> Option { + match *self { + EitherIter::Left(ref mut inner) => inner.next(), + EitherIter::Right(ref mut inner) => inner.next(), + } + } +} + +/// Container that signifies storing both different things +/// if there is a single state or many different states +/// involved in the operation. +#[derive(Debug, Clone)] +enum SingleOrManyStates { + Single(S), + Many(M), +} + +/// A source of texture state. +#[derive(Clone)] +enum TextureStateProvider<'a> { + /// Comes directly from a single state. + KnownSingle { state: TextureUses }, + /// Comes from a selector and a single state. + Selector { + selector: TextureSelector, + state: TextureUses, + }, + /// Comes from another texture set. + TextureSet { set: &'a TextureStateSet }, +} +impl<'a> TextureStateProvider<'a> { + /// Convenience function turning `Option` into this enum. + fn from_option(selector: Option, state: TextureUses) -> Self { + match selector { + Some(selector) => Self::Selector { selector, state }, + None => Self::KnownSingle { state }, + } + } + + /// Get the state provided by this. + /// + /// # Panics + /// + /// Panics if texture_selector is None and this uses a Selector source. + /// + /// # Safety + /// + /// - The index must be in bounds of the state set if this uses an TextureSet source. + #[inline(always)] + unsafe fn get_state( + self, + texture_selector: Option<&TextureSelector>, + index: usize, + ) -> SingleOrManyStates< + TextureUses, + impl Iterator + Clone + 'a, + > { + match self { + TextureStateProvider::KnownSingle { state } => SingleOrManyStates::Single(state), + TextureStateProvider::Selector { selector, state } => { + // We check if the selector given is actually for the full resource, + // and if it is we promote to a simple state. This allows upstream + // code to specify selectors willy nilly, and all that are really + // single states are promoted here. + if *texture_selector.unwrap() == selector { + SingleOrManyStates::Single(state) + } else { + SingleOrManyStates::Many(EitherIter::Left(iter::once((selector, state)))) + } + } + TextureStateProvider::TextureSet { set } => match unsafe { set.get_unchecked(index) } { + SingleOrManyStates::Single(single) => SingleOrManyStates::Single(single), + SingleOrManyStates::Many(complex) => { + SingleOrManyStates::Many(EitherIter::Right(complex.to_selector_state_iter())) + } + }, + } + } +} + +/// Does an insertion operation if the index isn't tracked +/// in the current metadata, otherwise merges the given state +/// with the current state. If the merging would cause +/// a conflict, returns that usage conflict. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. +#[inline(always)] +unsafe fn insert_or_merge( + texture_selector: &TextureSelector, + current_state_set: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata>, + index: usize, + state_provider: TextureStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, Arc>, +) -> Result<(), ResourceUsageCompatibilityError> { + let currently_owned = unsafe { resource_metadata.contains_unchecked(index) }; + + if !currently_owned { + unsafe { + insert( + Some(texture_selector), + None, + current_state_set, + resource_metadata, + index, + state_provider, + None, + metadata_provider, + ) + }; + return Ok(()); + } + + unsafe { + merge( + texture_selector, + current_state_set, + index, + state_provider, + metadata_provider, + ) + } +} + +/// If the resource isn't tracked +/// - Inserts the given resource. +/// - Uses the `start_state_provider` to populate `start_states` +/// - Uses either `end_state_provider` or `start_state_provider` +/// to populate `current_states`. +/// +/// If the resource is tracked +/// - Inserts barriers from the state in `current_states` +/// to the state provided by `start_state_provider`. +/// - Updates the `current_states` with either the state from +/// `end_state_provider` or `start_state_provider`. +/// +/// Any barriers are added to the barrier vector. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. +#[inline(always)] +unsafe fn insert_or_barrier_update( + texture_selector: &TextureSelector, + start_state: Option<&mut TextureStateSet>, + current_state_set: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata>, + index: usize, + start_state_provider: TextureStateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, Arc>, + barriers: &mut Vec>, + ordered_uses_mask: TextureUses, +) { + let currently_owned = unsafe { resource_metadata.contains_unchecked(index) }; + + if !currently_owned { + unsafe { + insert( + Some(texture_selector), + start_state, + current_state_set, + resource_metadata, + index, + start_state_provider, + end_state_provider, + metadata_provider, + ) + }; + return; + } + + let update_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider.clone()); + unsafe { + barrier( + texture_selector, + current_state_set, + index, + start_state_provider, + barriers, + ordered_uses_mask, + ) + }; + unsafe { + update( + texture_selector, + start_state, + current_state_set, + index, + update_state_provider, + ) + }; +} + +#[inline(always)] +unsafe fn insert( + texture_selector: Option<&TextureSelector>, + start_state: Option<&mut TextureStateSet>, + end_state: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata, + index: usize, + start_state_provider: TextureStateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, T>, +) { + let start_layers = unsafe { start_state_provider.get_state(texture_selector, index) }; + match start_layers { + SingleOrManyStates::Single(state) => { + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + strict_assert_eq!(invalid_resource_state(state), false); + + if let Some(start_state) = start_state { + unsafe { start_state.insert_simple_unchecked(index, state) }; + } + + // We only need to insert ourselves the end state if there is no end state provider. + if end_state_provider.is_none() { + unsafe { end_state.insert_simple_unchecked(index, state) }; + } + } + SingleOrManyStates::Many(state_iter) => { + let full_range = texture_selector.unwrap().clone(); + + let complex = + unsafe { ComplexTextureState::from_selector_state_iter(full_range, state_iter) }; + + if let Some(start_state) = start_state { + unsafe { start_state.insert_complex_unchecked(index, complex.clone()) }; + } + + // We only need to insert ourselves the end state if there is no end state provider. + if end_state_provider.is_none() { + unsafe { end_state.insert_complex_unchecked(index, complex) }; + } + } + } + + if let Some(end_state_provider) = end_state_provider { + match unsafe { end_state_provider.get_state(texture_selector, index) } { + SingleOrManyStates::Single(state) => { + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + strict_assert_eq!(invalid_resource_state(state), false); + + // We only need to insert into the end, as there is guaranteed to be + // a start state provider. + unsafe { end_state.insert_simple_unchecked(index, state) }; + } + SingleOrManyStates::Many(state_iter) => { + let full_range = texture_selector.unwrap().clone(); + + let complex = unsafe { + ComplexTextureState::from_selector_state_iter(full_range, state_iter) + }; + + // We only need to insert into the end, as there is guaranteed to be + // a start state provider. + unsafe { end_state.insert_complex_unchecked(index, complex) }; + } + } + } + + unsafe { + let resource = metadata_provider.get(index); + resource_metadata.insert(index, resource.clone()); + } +} + +#[inline(always)] +unsafe fn merge( + texture_selector: &TextureSelector, + current_state_set: &mut TextureStateSet, + index: usize, + state_provider: TextureStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, Arc>, +) -> Result<(), ResourceUsageCompatibilityError> { + let current_state = unsafe { current_state_set.get_mut_unchecked(index) }; + + let new_state = unsafe { state_provider.get_state(Some(texture_selector), index) }; + + match (current_state, new_state) { + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { + let merged_state = *current_simple | new_simple; + + if invalid_resource_state(merged_state) { + return Err(ResourceUsageCompatibilityError::from_texture( + unsafe { metadata_provider.get(index) }, + texture_selector.clone(), + *current_simple, + new_simple, + )); + } + + *current_simple = merged_state; + } + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + // Because we are now demoting this simple state to a complex state, + // we actually need to make a whole new complex state for us to use + // as there wasn't one before. + let mut new_complex = unsafe { + ComplexTextureState::from_selector_state_iter( + texture_selector.clone(), + iter::once((texture_selector.clone(), *current_simple)), + ) + }; + + for (selector, new_state) in new_many { + let merged_state = *current_simple | new_state; + + if invalid_resource_state(merged_state) { + return Err(ResourceUsageCompatibilityError::from_texture( + unsafe { metadata_provider.get(index) }, + selector, + *current_simple, + new_state, + )); + } + + for mip in + &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] + { + for &mut (_, ref mut current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + *current_layer_state = merged_state; + } + + mip.coalesce(); + } + } + + unsafe { current_state_set.make_complex_unchecked(index, new_complex) }; + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_simple)) => { + for (mip_id, mip) in current_complex.mips.iter_mut().enumerate() { + let mip_id = mip_id as u32; + + for &mut (ref layers, ref mut current_layer_state) in mip.iter_mut() { + let merged_state = *current_layer_state | new_simple; + + // Once we remove unknown, this will never be empty, as + // simple states are never unknown. + let merged_state = merged_state - TextureUses::UNKNOWN; + + if invalid_resource_state(merged_state) { + return Err(ResourceUsageCompatibilityError::from_texture( + unsafe { metadata_provider.get(index) }, + TextureSelector { + mips: mip_id..mip_id + 1, + layers: layers.clone(), + }, + *current_layer_state, + new_simple, + )); + } + + *current_layer_state = merged_state; + } + + mip.coalesce(); + } + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + for mip_id in selector.mips { + strict_assert!((mip_id as usize) < current_complex.mips.len()); + + let mip = unsafe { current_complex.mips.get_unchecked_mut(mip_id as usize) }; + + for &mut (ref layers, ref mut current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + let merged_state = *current_layer_state | new_state; + let merged_state = merged_state - TextureUses::UNKNOWN; + + if merged_state.is_empty() { + // We know nothing about this state, lets just move on. + continue; + } + + if invalid_resource_state(merged_state) { + return Err(ResourceUsageCompatibilityError::from_texture( + unsafe { metadata_provider.get(index) }, + TextureSelector { + mips: mip_id..mip_id + 1, + layers: layers.clone(), + }, + *current_layer_state, + new_state, + )); + } + *current_layer_state = merged_state; + } + + mip.coalesce(); + } + } + } + } + Ok(()) +} + +#[inline(always)] +unsafe fn barrier( + texture_selector: &TextureSelector, + current_state_set: &TextureStateSet, + index: usize, + state_provider: TextureStateProvider<'_>, + barriers: &mut Vec>, + ordered_uses_mask: TextureUses, +) { + let current_state = unsafe { current_state_set.get_unchecked(index) }; + + let new_state = unsafe { state_provider.get_state(Some(texture_selector), index) }; + + match (current_state, new_state) { + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { + if skip_barrier(current_simple, ordered_uses_mask, new_simple) { + return; + } + + barriers.push(PendingTransition { + id: index as _, + selector: texture_selector.clone(), + usage: hal::StateTransition { + from: current_simple, + to: new_simple, + }, + }); + } + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + if new_state == TextureUses::UNKNOWN { + continue; + } + + if skip_barrier(current_simple, ordered_uses_mask, new_state) { + continue; + } + + barriers.push(PendingTransition { + id: index as _, + selector, + usage: hal::StateTransition { + from: current_simple, + to: new_state, + }, + }); + } + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_simple)) => { + for (mip_id, mip) in current_complex.mips.iter().enumerate() { + let mip_id = mip_id as u32; + + for &(ref layers, current_layer_state) in mip.iter() { + if current_layer_state == TextureUses::UNKNOWN { + continue; + } + + if skip_barrier(current_layer_state, ordered_uses_mask, new_simple) { + continue; + } + + barriers.push(PendingTransition { + id: index as _, + selector: TextureSelector { + mips: mip_id..mip_id + 1, + layers: layers.clone(), + }, + usage: hal::StateTransition { + from: current_layer_state, + to: new_simple, + }, + }); + } + } + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + for mip_id in selector.mips { + strict_assert!((mip_id as usize) < current_complex.mips.len()); + + let mip = unsafe { current_complex.mips.get_unchecked(mip_id as usize) }; + + for (layers, current_layer_state) in mip.iter_filter(&selector.layers) { + if *current_layer_state == TextureUses::UNKNOWN + || new_state == TextureUses::UNKNOWN + { + continue; + } + + if skip_barrier(*current_layer_state, ordered_uses_mask, new_state) { + continue; + } + + barriers.push(PendingTransition { + id: index as _, + selector: TextureSelector { + mips: mip_id..mip_id + 1, + layers, + }, + usage: hal::StateTransition { + from: *current_layer_state, + to: new_state, + }, + }); + } + } + } + } + } +} + +#[inline(always)] +unsafe fn update( + texture_selector: &TextureSelector, + start_state_set: Option<&mut TextureStateSet>, + current_state_set: &mut TextureStateSet, + index: usize, + state_provider: TextureStateProvider<'_>, +) { + // We only ever need to update the start state here if the state is complex. + // + // If the state is simple, the first insert to the tracker would cover it. + let mut start_complex = start_state_set.and_then(|start_state_set| { + match unsafe { start_state_set.get_mut_unchecked(index) } { + SingleOrManyStates::Single(_) => None, + SingleOrManyStates::Many(complex) => Some(complex), + } + }); + + let current_state = unsafe { current_state_set.get_mut_unchecked(index) }; + + let new_state = unsafe { state_provider.get_state(Some(texture_selector), index) }; + + match (current_state, new_state) { + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { + *current_simple = new_simple; + } + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + // Because we are now demoting this simple state to a complex state, + // we actually need to make a whole new complex state for us to use + // as there wasn't one before. + let mut new_complex = unsafe { + ComplexTextureState::from_selector_state_iter( + texture_selector.clone(), + iter::once((texture_selector.clone(), *current_simple)), + ) + }; + + for (selector, mut new_state) in new_many { + if new_state == TextureUses::UNKNOWN { + new_state = *current_simple; + } + for mip in + &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] + { + for &mut (_, ref mut current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + *current_layer_state = new_state; + } + + mip.coalesce(); + } + } + + unsafe { current_state_set.make_complex_unchecked(index, new_complex) }; + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_single)) => { + for (mip_id, mip) in current_complex.mips.iter().enumerate() { + for &(ref layers, current_layer_state) in mip.iter() { + // If this state is unknown, that means that the start is _also_ unknown. + if current_layer_state == TextureUses::UNKNOWN { + if let Some(&mut ref mut start_complex) = start_complex { + strict_assert!(mip_id < start_complex.mips.len()); + + let start_mip = unsafe { start_complex.mips.get_unchecked_mut(mip_id) }; + + for &mut (_, ref mut current_start_state) in + start_mip.isolate(layers, TextureUses::UNKNOWN) + { + strict_assert_eq!(*current_start_state, TextureUses::UNKNOWN); + *current_start_state = new_single; + } + + start_mip.coalesce(); + } + } + } + } + + unsafe { current_state_set.make_simple_unchecked(index, new_single) }; + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + if new_state == TextureUses::UNKNOWN { + // We know nothing new + continue; + } + + for mip_id in selector.mips { + let mip_id = mip_id as usize; + strict_assert!(mip_id < current_complex.mips.len()); + + let mip = unsafe { current_complex.mips.get_unchecked_mut(mip_id) }; + + for &mut (ref layers, ref mut current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + if *current_layer_state == TextureUses::UNKNOWN + && new_state != TextureUses::UNKNOWN + { + // We now know something about this subresource that + // we didn't before so we should go back and update + // the start state. + // + // We know we must have starter state be complex, + // otherwise we would know about this state. + strict_assert!(start_complex.is_some()); + + let start_complex = + unsafe { start_complex.as_deref_mut().unwrap_unchecked() }; + + strict_assert!(mip_id < start_complex.mips.len()); + + let start_mip = unsafe { start_complex.mips.get_unchecked_mut(mip_id) }; + + for &mut (_, ref mut current_start_state) in + start_mip.isolate(layers, TextureUses::UNKNOWN) + { + strict_assert_eq!(*current_start_state, TextureUses::UNKNOWN); + *current_start_state = new_state; + } + + start_mip.coalesce(); + } + + *current_layer_state = new_state; + } + + mip.coalesce(); + } + } + } + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/validation.rs b/third_party/wgpu-core-29.0.4-patched/src/validation.rs new file mode 100644 index 00000000..ff467be6 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/validation.rs @@ -0,0 +1,1748 @@ +use alloc::{ + boxed::Box, + string::{String, ToString as _}, + sync::Arc, + vec::Vec, +}; +use core::fmt; + +use arrayvec::ArrayVec; +use hashbrown::hash_map::Entry; +use shader_io_deductions::{display_deductions_as_optional_list, MaxVertexShaderOutputDeduction}; +use thiserror::Error; +use wgt::{ + error::{ErrorType, WebGpuError}, + BindGroupLayoutEntry, BindingType, +}; + +use crate::{ + device::bgl, resource::InvalidResourceError, + validation::shader_io_deductions::MaxFragmentShaderInputDeduction, FastHashMap, FastHashSet, +}; + +pub mod shader_io_deductions; + +#[derive(Debug)] +enum ResourceType { + Buffer { + size: wgt::BufferSize, + }, + Texture { + dim: naga::ImageDimension, + arrayed: bool, + class: naga::ImageClass, + }, + Sampler { + comparison: bool, + }, + AccelerationStructure { + vertex_return: bool, + }, +} + +#[derive(Clone, Debug)] +pub enum BindingTypeName { + Buffer, + Texture, + Sampler, + AccelerationStructure, + ExternalTexture, +} + +impl From<&ResourceType> for BindingTypeName { + fn from(ty: &ResourceType) -> BindingTypeName { + match ty { + ResourceType::Buffer { .. } => BindingTypeName::Buffer, + ResourceType::Texture { + class: naga::ImageClass::External, + .. + } => BindingTypeName::ExternalTexture, + ResourceType::Texture { .. } => BindingTypeName::Texture, + ResourceType::Sampler { .. } => BindingTypeName::Sampler, + ResourceType::AccelerationStructure { .. } => BindingTypeName::AccelerationStructure, + } + } +} + +impl From<&BindingType> for BindingTypeName { + fn from(ty: &BindingType) -> BindingTypeName { + match ty { + BindingType::Buffer { .. } => BindingTypeName::Buffer, + BindingType::Texture { .. } => BindingTypeName::Texture, + BindingType::StorageTexture { .. } => BindingTypeName::Texture, + BindingType::Sampler { .. } => BindingTypeName::Sampler, + BindingType::AccelerationStructure { .. } => BindingTypeName::AccelerationStructure, + BindingType::ExternalTexture => BindingTypeName::ExternalTexture, + } + } +} + +#[derive(Debug)] +struct Resource { + #[allow(unused)] + name: Option, + bind: naga::ResourceBinding, + ty: ResourceType, + class: naga::AddressSpace, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum NumericDimension { + Scalar, + Vector(naga::VectorSize), + Matrix(naga::VectorSize, naga::VectorSize), +} + +impl fmt::Display for NumericDimension { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + Self::Scalar => write!(f, ""), + Self::Vector(size) => write!(f, "x{}", size as u8), + Self::Matrix(columns, rows) => write!(f, "x{}{}", columns as u8, rows as u8), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct NumericType { + dim: NumericDimension, + scalar: naga::Scalar, +} + +impl fmt::Display for NumericType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{:?}{}{}", + self.scalar.kind, + self.scalar.width * 8, + self.dim + ) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InterfaceVar { + pub ty: NumericType, + interpolation: Option, + sampling: Option, + per_primitive: bool, +} + +impl InterfaceVar { + pub fn vertex_attribute(format: wgt::VertexFormat) -> Self { + InterfaceVar { + ty: NumericType::from_vertex_format(format), + interpolation: None, + sampling: None, + per_primitive: false, + } + } +} + +impl fmt::Display for InterfaceVar { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{} interpolated as {:?} with sampling {:?}", + self.ty, self.interpolation, self.sampling + ) + } +} + +#[derive(Debug, Eq, PartialEq)] +enum Varying { + Local { location: u32, iv: InterfaceVar }, + BuiltIn(naga::BuiltIn), +} + +#[allow(unused)] +#[derive(Debug)] +struct SpecializationConstant { + id: u32, + ty: NumericType, +} + +#[derive(Debug)] +struct EntryPointMeshInfo { + max_vertices: u32, + max_primitives: u32, +} + +#[derive(Debug, Default)] +struct EntryPoint { + inputs: Vec, + outputs: Vec, + resources: Vec>, + #[allow(unused)] + spec_constants: Vec, + sampling_pairs: FastHashSet<(naga::Handle, naga::Handle)>, + workgroup_size: [u32; 3], + dual_source_blending: bool, + task_payload_size: Option, + mesh_info: Option, +} + +#[derive(Debug)] +pub struct Interface { + limits: wgt::Limits, + resources: naga::Arena, + entry_points: FastHashMap<(naga::ShaderStage, String), EntryPoint>, +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum BindingError { + #[error("Binding is missing from the pipeline layout")] + Missing, + #[error("Visibility flags don't include the shader stage")] + Invisible, + #[error( + "Type on the shader side ({shader:?}) does not match the pipeline binding ({binding:?})" + )] + WrongType { + binding: BindingTypeName, + shader: BindingTypeName, + }, + #[error("Storage class {binding:?} doesn't match the shader {shader:?}")] + WrongAddressSpace { + binding: naga::AddressSpace, + shader: naga::AddressSpace, + }, + #[error("Address space {space:?} is not a valid Buffer address space")] + WrongBufferAddressSpace { space: naga::AddressSpace }, + #[error("Buffer structure size {buffer_size}, added to one element of an unbound array, if it's the last field, ended up greater than the given `min_binding_size`, which is {min_binding_size}")] + WrongBufferSize { + buffer_size: wgt::BufferSize, + min_binding_size: wgt::BufferSize, + }, + #[error("View dimension {dim:?} (is array: {is_array}) doesn't match the binding {binding:?}")] + WrongTextureViewDimension { + dim: naga::ImageDimension, + is_array: bool, + binding: BindingType, + }, + #[error("Texture class {binding:?} doesn't match the shader {shader:?}")] + WrongTextureClass { + binding: naga::ImageClass, + shader: naga::ImageClass, + }, + #[error("Comparison flag doesn't match the shader")] + WrongSamplerComparison, + #[error("Derived bind group layout type is not consistent between stages")] + InconsistentlyDerivedType, + #[error("Texture format {0:?} is not supported for storage use")] + BadStorageFormat(wgt::TextureFormat), +} + +impl WebGpuError for BindingError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum FilteringError { + #[error("Integer textures can't be sampled with a filtering sampler")] + Integer, + #[error("Non-filterable float textures can't be sampled with a filtering sampler")] + Float, +} + +impl WebGpuError for FilteringError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum InputError { + #[error("Input is not provided by the earlier stage in the pipeline")] + Missing, + #[error("Input type is not compatible with the provided {0}")] + WrongType(NumericType), + #[error("Input interpolation doesn't match provided {0:?}")] + InterpolationMismatch(Option), + #[error("Input sampling doesn't match provided {0:?}")] + SamplingMismatch(Option), + #[error("Pipeline input has per_primitive={pipeline_input}, but shader expects per_primitive={shader}")] + WrongPerPrimitive { pipeline_input: bool, shader: bool }, +} + +impl WebGpuError for InputError { + fn webgpu_error_type(&self) -> ErrorType { + ErrorType::Validation + } +} + +/// Errors produced when validating a programmable stage of a pipeline. +#[derive(Clone, Debug, Error)] +#[non_exhaustive] +pub enum StageError { + #[error( + "Shader entry point's workgroup size {dimensions:?} ({total} total invocations) must be \ + less or equal to the per-dimension limit `Limits::{per_dimension_limits_desc}` of \ + {per_dimension_limits:?} and the total invocation limit `Limits::{total_limit_desc}` of \ + {total_limit}" + )] + InvalidWorkgroupSize { + dimensions: [u32; 3], + per_dimension_limits: [u32; 3], + per_dimension_limits_desc: &'static str, + total: u32, + total_limit: u32, + total_limit_desc: &'static str, + }, + #[error("Unable to find entry point '{0}'")] + MissingEntryPoint(String), + #[error("Shader global {0:?} is not available in the pipeline layout")] + Binding(naga::ResourceBinding, #[source] BindingError), + #[error("Unable to filter the texture ({texture:?}) by the sampler ({sampler:?})")] + Filtering { + texture: naga::ResourceBinding, + sampler: naga::ResourceBinding, + #[source] + error: FilteringError, + }, + #[error("Location[{location}] {var} is not provided by the previous stage outputs")] + Input { + location: wgt::ShaderLocation, + var: InterfaceVar, + #[source] + error: InputError, + }, + #[error( + "Unable to select an entry point: no entry point was found in the provided shader module" + )] + NoEntryPointFound, + #[error( + "Unable to select an entry point: \ + multiple entry points were found in the provided shader module, \ + but no entry point was specified" + )] + MultipleEntryPointsFound, + #[error(transparent)] + InvalidResource(#[from] InvalidResourceError), + #[error( + "vertex shader output location Location[{location}] ({var}) exceeds the \ + `max_inter_stage_shader_variables` limit ({}, 0-based){}", + // NOTE: Remember: the limit is 0-based for indices. + limit - 1, + display_deductions_as_optional_list(deductions, |d| d.for_location()) + )] + VertexOutputLocationTooLarge { + location: u32, + var: InterfaceVar, + limit: u32, + deductions: Vec, + }, + #[error( + "found {num_found} user-defined vertex shader output variables, which exceeds the \ + `max_inter_stage_shader_variables` limit ({limit}){}", + display_deductions_as_optional_list(deductions, |d| d.for_variables()) + )] + TooManyUserDefinedVertexOutputs { + num_found: u32, + limit: u32, + deductions: Vec, + }, + #[error( + "fragment shader input location Location[{location}] ({var}) exceeds the \ + `max_inter_stage_shader_variables` limit ({}, 0-based){}", + // NOTE: Remember: the limit is 0-based for indices. + limit - 1, + // NOTE: WebGPU spec. validation for fragment inputs is expressed in terms of variables + // (unlike vertex outputs), so we use `MaxFragmentShaderInputDeduction::for_variables` here + // (and not a non-existent `for_locations`). + display_deductions_as_optional_list(deductions, |d| d.for_variables()) + )] + FragmentInputLocationTooLarge { + location: u32, + var: InterfaceVar, + limit: u32, + deductions: Vec, + }, + #[error( + "found {num_found} user-defined fragment shader input variables, which exceeds the \ + `max_inter_stage_shader_variables` limit ({limit}){}", + display_deductions_as_optional_list(deductions, |d| d.for_variables()) + )] + TooManyUserDefinedFragmentInputs { + num_found: u32, + limit: u32, + deductions: Vec, + }, + #[error( + "Location[{location}] {var}'s index exceeds the `max_color_attachments` limit ({limit})" + )] + ColorAttachmentLocationTooLarge { + location: u32, + var: InterfaceVar, + limit: u32, + }, + #[error("Mesh shaders are limited to {limit} output vertices by `Limits::max_mesh_output_vertices`, but the shader has a maximum number of {value}")] + TooManyMeshVertices { limit: u32, value: u32 }, + #[error("Mesh shaders are limited to {limit} output primitives by `Limits::max_mesh_output_primitives`, but the shader has a maximum number of {value}")] + TooManyMeshPrimitives { limit: u32, value: u32 }, + #[error("Mesh or task shaders are limited to {limit} bytes of task payload by `Limits::max_task_payload_size`, but the shader has a task payload of size {value}")] + TaskPayloadTooLarge { limit: u32, value: u32 }, + #[error("Mesh shader's task payload has size ({shader:?}), which doesn't match the payload declared in the task stage ({input:?})")] + TaskPayloadMustMatch { + input: Option, + shader: Option, + }, + #[error("Primitive index can only be used in a fragment shader if the preceding shader was a vertex shader or a mesh shader that writes to primitive index.")] + InvalidPrimitiveIndex, + #[error("If a mesh shader writes to primitive index, it must be read by the fragment shader.")] + MissingPrimitiveIndex, + #[error("DrawId cannot be used in the same pipeline as a task shader")] + DrawIdError, + #[error("Pipeline uses dual-source blending, but the shader does not support it")] + InvalidDualSourceBlending, + #[error("Fragment shader writes depth, but pipeline does not have a depth attachment")] + MissingFragDepthAttachment, +} + +impl WebGpuError for StageError { + fn webgpu_error_type(&self) -> ErrorType { + match self { + Self::Binding(_, e) => e.webgpu_error_type(), + Self::InvalidResource(e) => e.webgpu_error_type(), + Self::Filtering { + texture: _, + sampler: _, + error, + } => error.webgpu_error_type(), + Self::Input { + location: _, + var: _, + error, + } => error.webgpu_error_type(), + Self::InvalidWorkgroupSize { .. } + | Self::MissingEntryPoint(..) + | Self::NoEntryPointFound + | Self::MultipleEntryPointsFound + | Self::VertexOutputLocationTooLarge { .. } + | Self::TooManyUserDefinedVertexOutputs { .. } + | Self::FragmentInputLocationTooLarge { .. } + | Self::TooManyUserDefinedFragmentInputs { .. } + | Self::ColorAttachmentLocationTooLarge { .. } + | Self::TooManyMeshVertices { .. } + | Self::TooManyMeshPrimitives { .. } + | Self::TaskPayloadTooLarge { .. } + | Self::TaskPayloadMustMatch { .. } + | Self::InvalidPrimitiveIndex + | Self::MissingPrimitiveIndex + | Self::DrawIdError + | Self::InvalidDualSourceBlending + | Self::MissingFragDepthAttachment => ErrorType::Validation, + } + } +} + +pub use wgpu_naga_bridge::map_storage_format_from_naga; +pub use wgpu_naga_bridge::map_storage_format_to_naga; + +impl Resource { + fn check_binding_use(&self, entry: &BindGroupLayoutEntry) -> Result<(), BindingError> { + match self.ty { + ResourceType::Buffer { size } => { + let min_size = match entry.ty { + BindingType::Buffer { + ty, + has_dynamic_offset: _, + min_binding_size, + } => { + let class = match ty { + wgt::BufferBindingType::Uniform => naga::AddressSpace::Uniform, + wgt::BufferBindingType::Storage { read_only } => { + let mut naga_access = naga::StorageAccess::LOAD; + naga_access.set(naga::StorageAccess::STORE, !read_only); + naga::AddressSpace::Storage { + access: naga_access, + } + } + }; + if self.class != class { + return Err(BindingError::WrongAddressSpace { + binding: class, + shader: self.class, + }); + } + min_binding_size + } + _ => { + return Err(BindingError::WrongType { + binding: (&entry.ty).into(), + shader: (&self.ty).into(), + }) + } + }; + match min_size { + Some(non_zero) if non_zero < size => { + return Err(BindingError::WrongBufferSize { + buffer_size: size, + min_binding_size: non_zero, + }) + } + _ => (), + } + } + ResourceType::Sampler { comparison } => match entry.ty { + BindingType::Sampler(ty) => { + if (ty == wgt::SamplerBindingType::Comparison) != comparison { + return Err(BindingError::WrongSamplerComparison); + } + } + _ => { + return Err(BindingError::WrongType { + binding: (&entry.ty).into(), + shader: (&self.ty).into(), + }) + } + }, + ResourceType::Texture { + dim, + arrayed, + class: shader_class, + } => { + let view_dimension = match entry.ty { + BindingType::Texture { view_dimension, .. } + | BindingType::StorageTexture { view_dimension, .. } => view_dimension, + BindingType::ExternalTexture => wgt::TextureViewDimension::D2, + _ => { + return Err(BindingError::WrongTextureViewDimension { + dim, + is_array: false, + binding: entry.ty, + }) + } + }; + if arrayed { + match (dim, view_dimension) { + (naga::ImageDimension::D2, wgt::TextureViewDimension::D2Array) => (), + (naga::ImageDimension::Cube, wgt::TextureViewDimension::CubeArray) => (), + _ => { + return Err(BindingError::WrongTextureViewDimension { + dim, + is_array: true, + binding: entry.ty, + }) + } + } + } else { + match (dim, view_dimension) { + (naga::ImageDimension::D1, wgt::TextureViewDimension::D1) => (), + (naga::ImageDimension::D2, wgt::TextureViewDimension::D2) => (), + (naga::ImageDimension::D3, wgt::TextureViewDimension::D3) => (), + (naga::ImageDimension::Cube, wgt::TextureViewDimension::Cube) => (), + _ => { + return Err(BindingError::WrongTextureViewDimension { + dim, + is_array: false, + binding: entry.ty, + }) + } + } + } + match entry.ty { + BindingType::Texture { + sample_type, + view_dimension: _, + multisampled: multi, + } => { + let binding_class = match sample_type { + wgt::TextureSampleType::Float { .. } => naga::ImageClass::Sampled { + kind: naga::ScalarKind::Float, + multi, + }, + wgt::TextureSampleType::Sint => naga::ImageClass::Sampled { + kind: naga::ScalarKind::Sint, + multi, + }, + wgt::TextureSampleType::Uint => naga::ImageClass::Sampled { + kind: naga::ScalarKind::Uint, + multi, + }, + wgt::TextureSampleType::Depth => naga::ImageClass::Depth { multi }, + }; + if shader_class == binding_class { + Ok(()) + } else { + Err(binding_class) + } + } + BindingType::StorageTexture { + access: wgt_binding_access, + format: wgt_binding_format, + view_dimension: _, + } => { + const LOAD_STORE: naga::StorageAccess = + naga::StorageAccess::LOAD.union(naga::StorageAccess::STORE); + let binding_format = map_storage_format_to_naga(wgt_binding_format) + .ok_or(BindingError::BadStorageFormat(wgt_binding_format))?; + let binding_access = match wgt_binding_access { + wgt::StorageTextureAccess::ReadOnly => naga::StorageAccess::LOAD, + wgt::StorageTextureAccess::WriteOnly => naga::StorageAccess::STORE, + wgt::StorageTextureAccess::ReadWrite => LOAD_STORE, + wgt::StorageTextureAccess::Atomic => { + naga::StorageAccess::ATOMIC | LOAD_STORE + } + }; + match shader_class { + // Formats must match exactly. A write-only shader (but not a + // read-only shader) is compatible with a read-write binding. + naga::ImageClass::Storage { + format: shader_format, + access: shader_access, + } if shader_format == binding_format + && (shader_access == binding_access + || shader_access == naga::StorageAccess::STORE + && binding_access == LOAD_STORE) => + { + Ok(()) + } + _ => Err(naga::ImageClass::Storage { + format: binding_format, + access: binding_access, + }), + } + } + BindingType::ExternalTexture => { + let binding_class = naga::ImageClass::External; + if shader_class == binding_class { + Ok(()) + } else { + Err(binding_class) + } + } + _ => { + return Err(BindingError::WrongType { + binding: (&entry.ty).into(), + shader: (&self.ty).into(), + }) + } + } + .map_err(|binding_class| BindingError::WrongTextureClass { + binding: binding_class, + shader: shader_class, + })?; + } + ResourceType::AccelerationStructure { vertex_return } => match entry.ty { + BindingType::AccelerationStructure { + vertex_return: entry_vertex_return, + } if vertex_return == entry_vertex_return => (), + _ => { + return Err(BindingError::WrongType { + binding: (&entry.ty).into(), + shader: (&self.ty).into(), + }) + } + }, + }; + + Ok(()) + } + + fn derive_binding_type( + &self, + is_reffed_by_sampler_in_entrypoint: bool, + ) -> Result { + Ok(match self.ty { + ResourceType::Buffer { size } => BindingType::Buffer { + ty: match self.class { + naga::AddressSpace::Uniform => wgt::BufferBindingType::Uniform, + naga::AddressSpace::Storage { access } => wgt::BufferBindingType::Storage { + read_only: access == naga::StorageAccess::LOAD, + }, + _ => return Err(BindingError::WrongBufferAddressSpace { space: self.class }), + }, + has_dynamic_offset: false, + min_binding_size: Some(size), + }, + ResourceType::Sampler { comparison } => BindingType::Sampler(if comparison { + wgt::SamplerBindingType::Comparison + } else { + wgt::SamplerBindingType::Filtering + }), + ResourceType::Texture { + dim, + arrayed, + class, + } => { + let view_dimension = match dim { + naga::ImageDimension::D1 => wgt::TextureViewDimension::D1, + naga::ImageDimension::D2 if arrayed => wgt::TextureViewDimension::D2Array, + naga::ImageDimension::D2 => wgt::TextureViewDimension::D2, + naga::ImageDimension::D3 => wgt::TextureViewDimension::D3, + naga::ImageDimension::Cube if arrayed => wgt::TextureViewDimension::CubeArray, + naga::ImageDimension::Cube => wgt::TextureViewDimension::Cube, + }; + match class { + naga::ImageClass::Sampled { multi, kind } => BindingType::Texture { + sample_type: match kind { + naga::ScalarKind::Float => wgt::TextureSampleType::Float { + filterable: is_reffed_by_sampler_in_entrypoint, + }, + naga::ScalarKind::Sint => wgt::TextureSampleType::Sint, + naga::ScalarKind::Uint => wgt::TextureSampleType::Uint, + naga::ScalarKind::AbstractInt + | naga::ScalarKind::AbstractFloat + | naga::ScalarKind::Bool => unreachable!(), + }, + view_dimension, + multisampled: multi, + }, + naga::ImageClass::Depth { multi } => BindingType::Texture { + sample_type: wgt::TextureSampleType::Depth, + view_dimension, + multisampled: multi, + }, + naga::ImageClass::Storage { format, access } => BindingType::StorageTexture { + access: { + const LOAD_STORE: naga::StorageAccess = + naga::StorageAccess::LOAD.union(naga::StorageAccess::STORE); + match access { + naga::StorageAccess::LOAD => wgt::StorageTextureAccess::ReadOnly, + naga::StorageAccess::STORE => wgt::StorageTextureAccess::WriteOnly, + LOAD_STORE => wgt::StorageTextureAccess::ReadWrite, + _ if access.contains(naga::StorageAccess::ATOMIC) => { + wgt::StorageTextureAccess::Atomic + } + _ => unreachable!(), + } + }, + view_dimension, + format: { + let f = map_storage_format_from_naga(format); + let original = map_storage_format_to_naga(f) + .ok_or(BindingError::BadStorageFormat(f))?; + debug_assert_eq!(format, original); + f + }, + }, + naga::ImageClass::External => BindingType::ExternalTexture, + } + } + ResourceType::AccelerationStructure { vertex_return } => { + BindingType::AccelerationStructure { vertex_return } + } + }) + } +} + +impl NumericType { + fn from_vertex_format(format: wgt::VertexFormat) -> Self { + use naga::{Scalar, VectorSize as Vs}; + use wgt::VertexFormat as Vf; + + let (dim, scalar) = match format { + Vf::Uint8 | Vf::Uint16 | Vf::Uint32 => (NumericDimension::Scalar, Scalar::U32), + Vf::Uint8x2 | Vf::Uint16x2 | Vf::Uint32x2 => { + (NumericDimension::Vector(Vs::Bi), Scalar::U32) + } + Vf::Uint32x3 => (NumericDimension::Vector(Vs::Tri), Scalar::U32), + Vf::Uint8x4 | Vf::Uint16x4 | Vf::Uint32x4 => { + (NumericDimension::Vector(Vs::Quad), Scalar::U32) + } + Vf::Sint8 | Vf::Sint16 | Vf::Sint32 => (NumericDimension::Scalar, Scalar::I32), + Vf::Sint8x2 | Vf::Sint16x2 | Vf::Sint32x2 => { + (NumericDimension::Vector(Vs::Bi), Scalar::I32) + } + Vf::Sint32x3 => (NumericDimension::Vector(Vs::Tri), Scalar::I32), + Vf::Sint8x4 | Vf::Sint16x4 | Vf::Sint32x4 => { + (NumericDimension::Vector(Vs::Quad), Scalar::I32) + } + Vf::Unorm8 | Vf::Unorm16 | Vf::Snorm8 | Vf::Snorm16 | Vf::Float16 | Vf::Float32 => { + (NumericDimension::Scalar, Scalar::F32) + } + Vf::Unorm8x2 + | Vf::Snorm8x2 + | Vf::Unorm16x2 + | Vf::Snorm16x2 + | Vf::Float16x2 + | Vf::Float32x2 => (NumericDimension::Vector(Vs::Bi), Scalar::F32), + Vf::Float32x3 => (NumericDimension::Vector(Vs::Tri), Scalar::F32), + Vf::Unorm8x4 + | Vf::Snorm8x4 + | Vf::Unorm16x4 + | Vf::Snorm16x4 + | Vf::Float16x4 + | Vf::Float32x4 + | Vf::Unorm10_10_10_2 + | Vf::Unorm8x4Bgra => (NumericDimension::Vector(Vs::Quad), Scalar::F32), + Vf::Float64 => (NumericDimension::Scalar, Scalar::F64), + Vf::Float64x2 => (NumericDimension::Vector(Vs::Bi), Scalar::F64), + Vf::Float64x3 => (NumericDimension::Vector(Vs::Tri), Scalar::F64), + Vf::Float64x4 => (NumericDimension::Vector(Vs::Quad), Scalar::F64), + }; + + NumericType { + dim, + //Note: Shader always sees data as int, uint, or float. + // It doesn't know if the original is normalized in a tighter form. + scalar, + } + } + + fn from_texture_format(format: wgt::TextureFormat) -> Self { + use naga::{Scalar, VectorSize as Vs}; + use wgt::TextureFormat as Tf; + + let (dim, scalar) = match format { + Tf::R8Unorm | Tf::R8Snorm | Tf::R16Float | Tf::R32Float => { + (NumericDimension::Scalar, Scalar::F32) + } + Tf::R8Uint | Tf::R16Uint | Tf::R32Uint => (NumericDimension::Scalar, Scalar::U32), + Tf::R8Sint | Tf::R16Sint | Tf::R32Sint => (NumericDimension::Scalar, Scalar::I32), + Tf::Rg8Unorm | Tf::Rg8Snorm | Tf::Rg16Float | Tf::Rg32Float => { + (NumericDimension::Vector(Vs::Bi), Scalar::F32) + } + Tf::R64Uint => (NumericDimension::Scalar, Scalar::U64), + Tf::Rg8Uint | Tf::Rg16Uint | Tf::Rg32Uint => { + (NumericDimension::Vector(Vs::Bi), Scalar::U32) + } + Tf::Rg8Sint | Tf::Rg16Sint | Tf::Rg32Sint => { + (NumericDimension::Vector(Vs::Bi), Scalar::I32) + } + Tf::R16Snorm | Tf::R16Unorm => (NumericDimension::Scalar, Scalar::F32), + Tf::Rg16Snorm | Tf::Rg16Unorm => (NumericDimension::Vector(Vs::Bi), Scalar::F32), + Tf::Rgba16Snorm | Tf::Rgba16Unorm => (NumericDimension::Vector(Vs::Quad), Scalar::F32), + Tf::Rgba8Unorm + | Tf::Rgba8UnormSrgb + | Tf::Rgba8Snorm + | Tf::Bgra8Unorm + | Tf::Bgra8UnormSrgb + | Tf::Rgb10a2Unorm + | Tf::Rgba16Float + | Tf::Rgba32Float => (NumericDimension::Vector(Vs::Quad), Scalar::F32), + Tf::Rgba8Uint | Tf::Rgba16Uint | Tf::Rgba32Uint | Tf::Rgb10a2Uint => { + (NumericDimension::Vector(Vs::Quad), Scalar::U32) + } + Tf::Rgba8Sint | Tf::Rgba16Sint | Tf::Rgba32Sint => { + (NumericDimension::Vector(Vs::Quad), Scalar::I32) + } + Tf::Rg11b10Ufloat => (NumericDimension::Vector(Vs::Tri), Scalar::F32), + Tf::Stencil8 + | Tf::Depth16Unorm + | Tf::Depth32Float + | Tf::Depth32FloatStencil8 + | Tf::Depth24Plus + | Tf::Depth24PlusStencil8 => { + panic!("Unexpected depth format") + } + Tf::NV12 => panic!("Unexpected nv12 format"), + Tf::P010 => panic!("Unexpected p010 format"), + Tf::Rgb9e5Ufloat => (NumericDimension::Vector(Vs::Tri), Scalar::F32), + Tf::Bc1RgbaUnorm + | Tf::Bc1RgbaUnormSrgb + | Tf::Bc2RgbaUnorm + | Tf::Bc2RgbaUnormSrgb + | Tf::Bc3RgbaUnorm + | Tf::Bc3RgbaUnormSrgb + | Tf::Bc7RgbaUnorm + | Tf::Bc7RgbaUnormSrgb + | Tf::Etc2Rgb8A1Unorm + | Tf::Etc2Rgb8A1UnormSrgb + | Tf::Etc2Rgba8Unorm + | Tf::Etc2Rgba8UnormSrgb => (NumericDimension::Vector(Vs::Quad), Scalar::F32), + Tf::Bc4RUnorm | Tf::Bc4RSnorm | Tf::EacR11Unorm | Tf::EacR11Snorm => { + (NumericDimension::Scalar, Scalar::F32) + } + Tf::Bc5RgUnorm | Tf::Bc5RgSnorm | Tf::EacRg11Unorm | Tf::EacRg11Snorm => { + (NumericDimension::Vector(Vs::Bi), Scalar::F32) + } + Tf::Bc6hRgbUfloat | Tf::Bc6hRgbFloat | Tf::Etc2Rgb8Unorm | Tf::Etc2Rgb8UnormSrgb => { + (NumericDimension::Vector(Vs::Tri), Scalar::F32) + } + Tf::Astc { + block: _, + channel: _, + } => (NumericDimension::Vector(Vs::Quad), Scalar::F32), + }; + + NumericType { + dim, + //Note: Shader always sees data as int, uint, or float. + // It doesn't know if the original is normalized in a tighter form. + scalar, + } + } + + fn is_subtype_of(&self, other: &NumericType) -> bool { + if self.scalar.width > other.scalar.width { + return false; + } + if self.scalar.kind != other.scalar.kind { + return false; + } + match (self.dim, other.dim) { + (NumericDimension::Scalar, NumericDimension::Scalar) => true, + (NumericDimension::Scalar, NumericDimension::Vector(_)) => true, + (NumericDimension::Vector(s0), NumericDimension::Vector(s1)) => s0 <= s1, + (NumericDimension::Matrix(c0, r0), NumericDimension::Matrix(c1, r1)) => { + c0 == c1 && r0 == r1 + } + _ => false, + } + } +} + +/// Return true if the fragment `format` is covered by the provided `output`. +pub fn check_texture_format( + format: wgt::TextureFormat, + output: &NumericType, +) -> Result<(), NumericType> { + let nt = NumericType::from_texture_format(format); + if nt.is_subtype_of(output) { + Ok(()) + } else { + Err(nt) + } +} + +pub enum BindingLayoutSource { + /// The binding layout is derived from the pipeline layout. + /// + /// This will be filled in by the shader binding validation, as it iterates the shader's interfaces. + Derived(Box>), + /// The binding layout is provided by the user in BGLs. + /// + /// This will be validated against the shader's interfaces. + Provided(Arc), +} + +impl BindingLayoutSource { + pub fn new_derived(limits: &wgt::Limits) -> Self { + let mut array = ArrayVec::new(); + for _ in 0..limits.max_bind_groups { + array.push(Default::default()); + } + BindingLayoutSource::Derived(Box::new(array)) + } +} + +#[derive(Debug, Clone, Default)] +pub struct StageIo { + pub varyings: FastHashMap, + /// This must match between mesh & task shaders + pub task_payload_size: Option, + /// Fragment shaders cannot input primitive index on mesh shaders that don't output it on DX12. + /// Therefore, we track between shader stages if primitive index is written (or if vertex shader + /// is used). + /// + /// This is Some if it was a mesh shader. + pub primitive_index: Option, +} + +impl Interface { + fn populate( + list: &mut Vec, + binding: Option<&naga::Binding>, + ty: naga::Handle, + arena: &naga::UniqueArena, + ) { + let numeric_ty = match arena[ty].inner { + naga::TypeInner::Scalar(scalar) => NumericType { + dim: NumericDimension::Scalar, + scalar, + }, + naga::TypeInner::Vector { size, scalar } => NumericType { + dim: NumericDimension::Vector(size), + scalar, + }, + naga::TypeInner::Matrix { + columns, + rows, + scalar, + } => NumericType { + dim: NumericDimension::Matrix(columns, rows), + scalar, + }, + naga::TypeInner::Struct { ref members, .. } => { + for member in members { + Self::populate(list, member.binding.as_ref(), member.ty, arena); + } + return; + } + ref other => { + //Note: technically this should be at least `log::error`, but + // the reality is - every shader coming from `glslc` outputs an array + // of clip distances and hits this path :( + // So we lower it to `log::debug` to be less annoying as + // there's nothing the user can do about it. + log::debug!("Unexpected varying type: {other:?}"); + return; + } + }; + + let varying = match binding { + Some(&naga::Binding::Location { + location, + interpolation, + sampling, + per_primitive, + blend_src: _, + }) => Varying::Local { + location, + iv: InterfaceVar { + ty: numeric_ty, + interpolation, + sampling, + per_primitive, + }, + }, + Some(&naga::Binding::BuiltIn(built_in)) => Varying::BuiltIn(built_in), + None => { + log::error!("Missing binding for a varying"); + return; + } + }; + list.push(varying); + } + + pub fn new(module: &naga::Module, info: &naga::valid::ModuleInfo, limits: wgt::Limits) -> Self { + let mut resources = naga::Arena::new(); + let mut resource_mapping = FastHashMap::default(); + for (var_handle, var) in module.global_variables.iter() { + let bind = match var.binding { + Some(br) => br, + _ => continue, + }; + let naga_ty = &module.types[var.ty].inner; + + let inner_ty = match *naga_ty { + naga::TypeInner::BindingArray { base, .. } => &module.types[base].inner, + ref ty => ty, + }; + + let ty = match *inner_ty { + naga::TypeInner::Image { + dim, + arrayed, + class, + } => ResourceType::Texture { + dim, + arrayed, + class, + }, + naga::TypeInner::Sampler { comparison } => ResourceType::Sampler { comparison }, + naga::TypeInner::AccelerationStructure { vertex_return } => { + ResourceType::AccelerationStructure { vertex_return } + } + ref other => ResourceType::Buffer { + size: wgt::BufferSize::new(other.size(module.to_ctx()) as u64).unwrap(), + }, + }; + let handle = resources.append( + Resource { + name: var.name.clone(), + bind, + ty, + class: var.space, + }, + Default::default(), + ); + resource_mapping.insert(var_handle, handle); + } + + let mut entry_points = FastHashMap::default(); + entry_points.reserve(module.entry_points.len()); + for (index, entry_point) in module.entry_points.iter().enumerate() { + let info = info.get_entry_point(index); + let mut ep = EntryPoint::default(); + for arg in entry_point.function.arguments.iter() { + Self::populate(&mut ep.inputs, arg.binding.as_ref(), arg.ty, &module.types); + } + if let Some(ref result) = entry_point.function.result { + Self::populate( + &mut ep.outputs, + result.binding.as_ref(), + result.ty, + &module.types, + ); + } + + for (var_handle, var) in module.global_variables.iter() { + let usage = info[var_handle]; + if !usage.is_empty() && var.binding.is_some() { + ep.resources.push(resource_mapping[&var_handle]); + } + } + + for key in info.sampling_set.iter() { + ep.sampling_pairs + .insert((resource_mapping[&key.image], resource_mapping[&key.sampler])); + } + ep.dual_source_blending = info.dual_source_blending; + ep.workgroup_size = entry_point.workgroup_size; + + if let Some(task_payload) = entry_point.task_payload { + ep.task_payload_size = Some( + module.types[module.global_variables[task_payload].ty] + .inner + .size(module.to_ctx()), + ); + } + if let Some(ref mesh_info) = entry_point.mesh_info { + ep.mesh_info = Some(EntryPointMeshInfo { + max_vertices: mesh_info.max_vertices, + max_primitives: mesh_info.max_primitives, + }); + Self::populate( + &mut ep.outputs, + None, + mesh_info.vertex_output_type, + &module.types, + ); + Self::populate( + &mut ep.outputs, + None, + mesh_info.primitive_output_type, + &module.types, + ); + } + + entry_points.insert((entry_point.stage, entry_point.name.clone()), ep); + } + + Self { + limits, + resources, + entry_points, + } + } + + pub fn finalize_entry_point_name( + &self, + stage: naga::ShaderStage, + entry_point_name: Option<&str>, + ) -> Result { + entry_point_name + .map(|ep| ep.to_string()) + .map(Ok) + .unwrap_or_else(|| { + let mut entry_points = self + .entry_points + .keys() + .filter_map(|(ep_stage, name)| (ep_stage == &stage).then_some(name)); + let first = entry_points.next().ok_or(StageError::NoEntryPointFound)?; + if entry_points.next().is_some() { + return Err(StageError::MultipleEntryPointsFound); + } + Ok(first.clone()) + }) + } + + /// Among other things, this implements some validation logic defined by the WebGPU spec. at + /// . + pub fn check_stage( + &self, + layouts: &mut BindingLayoutSource, + shader_binding_sizes: &mut FastHashMap, + entry_point_name: &str, + shader_stage: ShaderStageForValidation, + inputs: StageIo, + ) -> Result { + // Since a shader module can have multiple entry points with the same name, + // we need to look for one with the right execution model. + let pair = (shader_stage.to_naga(), entry_point_name.to_string()); + let entry_point = match self.entry_points.get(&pair) { + Some(some) => some, + None => return Err(StageError::MissingEntryPoint(pair.1)), + }; + let (_, entry_point_name) = pair; + + let stage_bit = shader_stage.to_wgt_bit(); + + // check resources visibility + for &handle in entry_point.resources.iter() { + let res = &self.resources[handle]; + let result = 'err: { + match layouts { + BindingLayoutSource::Provided(pipeline_layout) => { + // update the required binding size for this buffer + if let ResourceType::Buffer { size } = res.ty { + match shader_binding_sizes.entry(res.bind) { + Entry::Occupied(e) => { + *e.into_mut() = size.max(*e.get()); + } + Entry::Vacant(e) => { + e.insert(size); + } + } + } + + let Some(entry) = + pipeline_layout.get_bgl_entry(res.bind.group, res.bind.binding) + else { + break 'err Err(BindingError::Missing); + }; + + if !entry.visibility.contains(stage_bit) { + break 'err Err(BindingError::Invisible); + } + + res.check_binding_use(entry) + } + BindingLayoutSource::Derived(layouts) => { + let Some(map) = layouts.get_mut(res.bind.group as usize) else { + break 'err Err(BindingError::Missing); + }; + + let ty = match res.derive_binding_type( + entry_point + .sampling_pairs + .iter() + .any(|&(im, _samp)| im == handle), + ) { + Ok(ty) => ty, + Err(error) => break 'err Err(error), + }; + + match map.entry(res.bind.binding) { + indexmap::map::Entry::Occupied(e) if e.get().ty != ty => { + break 'err Err(BindingError::InconsistentlyDerivedType) + } + indexmap::map::Entry::Occupied(e) => { + e.into_mut().visibility |= stage_bit; + } + indexmap::map::Entry::Vacant(e) => { + e.insert(BindGroupLayoutEntry { + binding: res.bind.binding, + ty, + visibility: stage_bit, + count: None, + }); + } + } + Ok(()) + } + } + }; + if let Err(error) = result { + return Err(StageError::Binding(res.bind, error)); + } + } + + // Check the compatibility between textures and samplers + // + // We only need to do this if the binding layout is provided by the user, as derived + // layouts will inherently be correctly tagged. + if let BindingLayoutSource::Provided(pipeline_layout) = layouts { + for &(texture_handle, sampler_handle) in entry_point.sampling_pairs.iter() { + let texture_bind = &self.resources[texture_handle].bind; + let sampler_bind = &self.resources[sampler_handle].bind; + let texture_layout = pipeline_layout + .get_bgl_entry(texture_bind.group, texture_bind.binding) + .unwrap(); + let sampler_layout = pipeline_layout + .get_bgl_entry(sampler_bind.group, sampler_bind.binding) + .unwrap(); + assert!(texture_layout.visibility.contains(stage_bit)); + assert!(sampler_layout.visibility.contains(stage_bit)); + + let sampler_filtering = matches!( + sampler_layout.ty, + BindingType::Sampler(wgt::SamplerBindingType::Filtering) + ); + let texture_sample_type = match texture_layout.ty { + BindingType::Texture { sample_type, .. } => sample_type, + BindingType::ExternalTexture => { + wgt::TextureSampleType::Float { filterable: true } + } + _ => unreachable!(), + }; + + let error = match (sampler_filtering, texture_sample_type) { + (true, wgt::TextureSampleType::Float { filterable: false }) => { + Some(FilteringError::Float) + } + (true, wgt::TextureSampleType::Sint) => Some(FilteringError::Integer), + (true, wgt::TextureSampleType::Uint) => Some(FilteringError::Integer), + _ => None, + }; + + if let Some(error) = error { + return Err(StageError::Filtering { + texture: *texture_bind, + sampler: *sampler_bind, + error, + }); + } + } + } + + // check workgroup size limits + if shader_stage.to_naga().compute_like() { + let ( + max_workgroup_size_limits, + max_workgroup_size_total, + per_dimension_limit, + total_limit, + ) = match shader_stage.to_naga() { + naga::ShaderStage::Compute => ( + [ + self.limits.max_compute_workgroup_size_x, + self.limits.max_compute_workgroup_size_y, + self.limits.max_compute_workgroup_size_z, + ], + self.limits.max_compute_invocations_per_workgroup, + "max_compute_workgroup_size_*", + "max_compute_invocations_per_workgroup", + ), + naga::ShaderStage::Task => ( + [ + self.limits.max_task_invocations_per_dimension, + self.limits.max_task_invocations_per_dimension, + self.limits.max_task_invocations_per_dimension, + ], + self.limits.max_task_invocations_per_workgroup, + "max_task_invocations_per_dimension", + "max_task_invocations_per_workgroup", + ), + naga::ShaderStage::Mesh => ( + [ + self.limits.max_mesh_invocations_per_dimension, + self.limits.max_mesh_invocations_per_dimension, + self.limits.max_mesh_invocations_per_dimension, + ], + self.limits.max_mesh_invocations_per_workgroup, + "max_mesh_invocations_per_dimension", + "max_mesh_invocations_per_workgroup", + ), + _ => unreachable!(), + }; + + let total_invocations = entry_point + .workgroup_size + .iter() + .fold(1u32, |total, &dim| total.saturating_mul(dim)); + let invalid_total_invocations = + total_invocations > max_workgroup_size_total || total_invocations == 0; + + assert_eq!( + entry_point.workgroup_size.len(), + max_workgroup_size_limits.len() + ); + let dimension_too_large = entry_point + .workgroup_size + .iter() + .zip(max_workgroup_size_limits.iter()) + .any(|(dim, limit)| dim > limit); + + if invalid_total_invocations || dimension_too_large { + return Err(StageError::InvalidWorkgroupSize { + dimensions: entry_point.workgroup_size, + total: total_invocations, + per_dimension_limits: max_workgroup_size_limits, + total_limit: max_workgroup_size_total, + per_dimension_limits_desc: per_dimension_limit, + total_limit_desc: total_limit, + }); + } + } + + let mut this_stage_primitive_index = false; + let mut has_draw_id = false; + + // check inputs compatibility + for input in entry_point.inputs.iter() { + match *input { + Varying::Local { location, ref iv } => { + let result = inputs + .varyings + .get(&location) + .ok_or(InputError::Missing) + .and_then(|provided| { + let (compatible, per_primitive_correct) = match shader_stage.to_naga() { + // For vertex attributes, there are defaults filled out + // by the driver if data is not provided. + naga::ShaderStage::Vertex => { + let is_compatible = + iv.ty.scalar.kind == provided.ty.scalar.kind; + // vertex inputs don't count towards inter-stage + (is_compatible, !iv.per_primitive) + } + naga::ShaderStage::Fragment => { + if iv.interpolation != provided.interpolation { + return Err(InputError::InterpolationMismatch( + provided.interpolation, + )); + } + if iv.sampling != provided.sampling { + return Err(InputError::SamplingMismatch( + provided.sampling, + )); + } + ( + iv.ty.is_subtype_of(&provided.ty), + iv.per_primitive == provided.per_primitive, + ) + } + // These can't have varying inputs + naga::ShaderStage::Compute + | naga::ShaderStage::Task + | naga::ShaderStage::Mesh => (false, false), + naga::ShaderStage::RayGeneration + | naga::ShaderStage::AnyHit + | naga::ShaderStage::ClosestHit + | naga::ShaderStage::Miss => { + unreachable!() + } + }; + if !compatible { + return Err(InputError::WrongType(provided.ty)); + } else if !per_primitive_correct { + return Err(InputError::WrongPerPrimitive { + pipeline_input: provided.per_primitive, + shader: iv.per_primitive, + }); + } + Ok(()) + }); + + if let Err(error) = result { + return Err(StageError::Input { + location, + var: iv.clone(), + error, + }); + } + } + Varying::BuiltIn(naga::BuiltIn::PrimitiveIndex) => { + this_stage_primitive_index = true; + } + Varying::BuiltIn(naga::BuiltIn::DrawIndex) => { + has_draw_id = true; + } + Varying::BuiltIn(_) => {} + } + } + + match shader_stage { + ShaderStageForValidation::Vertex { + topology, + compare_function, + } => { + let mut max_vertex_shader_output_variables = + self.limits.max_inter_stage_shader_variables; + let mut max_vertex_shader_output_location = max_vertex_shader_output_variables - 1; + + let point_list_deduction = if topology == wgt::PrimitiveTopology::PointList { + Some(MaxVertexShaderOutputDeduction::PointListPrimitiveTopology) + } else { + None + }; + + let deductions = point_list_deduction.into_iter(); + + for deduction in deductions.clone() { + // NOTE: Deductions, in the current version of the spec. we implement, do not + // ever exceed the minimum variables available. + max_vertex_shader_output_variables = max_vertex_shader_output_variables + .checked_sub(deduction.for_variables()) + .unwrap(); + max_vertex_shader_output_location = max_vertex_shader_output_location + .checked_sub(deduction.for_location()) + .unwrap(); + } + + let mut num_user_defined_outputs = 0; + + for output in entry_point.outputs.iter() { + match *output { + Varying::Local { ref iv, location } => { + if location > max_vertex_shader_output_location { + return Err(StageError::VertexOutputLocationTooLarge { + location, + var: iv.clone(), + limit: self.limits.max_inter_stage_shader_variables, + deductions: deductions.collect(), + }); + } + num_user_defined_outputs += 1; + } + Varying::BuiltIn(_) => {} + }; + + if let Some( + cmp @ wgt::CompareFunction::Equal | cmp @ wgt::CompareFunction::NotEqual, + ) = compare_function + { + if let Varying::BuiltIn(naga::BuiltIn::Position { invariant: false }) = + *output + { + log::warn!( + concat!( + "Vertex shader with entry point {} outputs a ", + "@builtin(position) without the @invariant attribute and ", + "is used in a pipeline with {cmp:?}. On some machines, ", + "this can cause bad artifacting as {cmp:?} assumes the ", + "values output from the vertex shader exactly match the ", + "value in the depth buffer. The @invariant attribute on the ", + "@builtin(position) vertex output ensures that the exact ", + "same pixel depths are used every render." + ), + entry_point_name, + cmp = cmp + ); + } + } + } + + if num_user_defined_outputs > max_vertex_shader_output_variables { + return Err(StageError::TooManyUserDefinedVertexOutputs { + num_found: num_user_defined_outputs, + limit: self.limits.max_inter_stage_shader_variables, + deductions: deductions.collect(), + }); + } + } + ShaderStageForValidation::Fragment { + dual_source_blending, + has_depth_attachment, + } => { + let mut max_fragment_shader_input_variables = + self.limits.max_inter_stage_shader_variables; + + let deductions = entry_point.inputs.iter().filter_map(|output| match output { + Varying::Local { .. } => None, + Varying::BuiltIn(builtin) => { + MaxFragmentShaderInputDeduction::from_inter_stage_builtin(*builtin).or_else( + || { + unreachable!( + concat!( + "unexpected built-in provided; ", + "{:?} is not used for fragment stage input", + ), + builtin + ) + }, + ) + } + }); + + for deduction in deductions.clone() { + // NOTE: Deductions, in the current version of the spec. we implement, do not + // ever exceed the minimum variables available. + max_fragment_shader_input_variables = max_fragment_shader_input_variables + .checked_sub(deduction.for_variables()) + .unwrap(); + } + + let mut num_user_defined_inputs = 0; + + for output in entry_point.inputs.iter() { + match *output { + Varying::Local { ref iv, location } => { + if location >= self.limits.max_inter_stage_shader_variables { + return Err(StageError::FragmentInputLocationTooLarge { + location, + var: iv.clone(), + limit: self.limits.max_inter_stage_shader_variables, + deductions: deductions.collect(), + }); + } + num_user_defined_inputs += 1; + } + Varying::BuiltIn(_) => {} + }; + } + + if num_user_defined_inputs > max_fragment_shader_input_variables { + return Err(StageError::TooManyUserDefinedFragmentInputs { + num_found: num_user_defined_inputs, + limit: self.limits.max_inter_stage_shader_variables, + deductions: deductions.collect(), + }); + } + + for output in &entry_point.outputs { + let &Varying::Local { location, ref iv } = output else { + continue; + }; + if location >= self.limits.max_color_attachments { + return Err(StageError::ColorAttachmentLocationTooLarge { + location, + var: iv.clone(), + limit: self.limits.max_color_attachments, + }); + } + } + + // If the pipeline uses dual-source blending, then the shader + // must configure appropriate I/O, but it is not an error to + // use a shader that defines the I/O in a pipeline that only + // uses one blend source. + if dual_source_blending && !entry_point.dual_source_blending { + return Err(StageError::InvalidDualSourceBlending); + } + + if entry_point + .outputs + .contains(&Varying::BuiltIn(naga::BuiltIn::FragDepth)) + && !has_depth_attachment + { + return Err(StageError::MissingFragDepthAttachment); + } + } + ShaderStageForValidation::Mesh => { + for output in &entry_point.outputs { + if matches!(output, Varying::BuiltIn(naga::BuiltIn::PrimitiveIndex)) { + this_stage_primitive_index = true; + } + } + } + _ => (), + } + + if let Some(ref mesh_info) = entry_point.mesh_info { + if mesh_info.max_vertices > self.limits.max_mesh_output_vertices { + return Err(StageError::TooManyMeshVertices { + limit: self.limits.max_mesh_output_vertices, + value: mesh_info.max_vertices, + }); + } + if mesh_info.max_primitives > self.limits.max_mesh_output_primitives { + return Err(StageError::TooManyMeshPrimitives { + limit: self.limits.max_mesh_output_primitives, + value: mesh_info.max_primitives, + }); + } + } + if let Some(task_payload_size) = entry_point.task_payload_size { + if task_payload_size > self.limits.max_task_payload_size { + return Err(StageError::TaskPayloadTooLarge { + limit: self.limits.max_task_payload_size, + value: task_payload_size, + }); + } + } + if shader_stage.to_naga() == naga::ShaderStage::Mesh + && entry_point.task_payload_size != inputs.task_payload_size + { + return Err(StageError::TaskPayloadMustMatch { + input: inputs.task_payload_size, + shader: entry_point.task_payload_size, + }); + } + + // Fragment shader primitive index is treated like a varying + if shader_stage.to_naga() == naga::ShaderStage::Fragment + && this_stage_primitive_index + && inputs.primitive_index == Some(false) + { + return Err(StageError::InvalidPrimitiveIndex); + } else if shader_stage.to_naga() == naga::ShaderStage::Fragment + && !this_stage_primitive_index + && inputs.primitive_index == Some(true) + { + return Err(StageError::MissingPrimitiveIndex); + } + if shader_stage.to_naga() == naga::ShaderStage::Mesh + && inputs.task_payload_size.is_some() + && has_draw_id + { + return Err(StageError::DrawIdError); + } + + let outputs = entry_point + .outputs + .iter() + .filter_map(|output| match *output { + Varying::Local { location, ref iv } => Some((location, iv.clone())), + Varying::BuiltIn(_) => None, + }) + .collect(); + + Ok(StageIo { + task_payload_size: entry_point.task_payload_size, + varyings: outputs, + primitive_index: if shader_stage.to_naga() == naga::ShaderStage::Mesh { + Some(this_stage_primitive_index) + } else { + None + }, + }) + } + + pub fn fragment_uses_dual_source_blending( + &self, + entry_point_name: &str, + ) -> Result { + let pair = (naga::ShaderStage::Fragment, entry_point_name.to_string()); + self.entry_points + .get(&pair) + .ok_or(StageError::MissingEntryPoint(pair.1)) + .map(|ep| ep.dual_source_blending) + } +} + +/// Validate a list of color attachment formats against `maxColorAttachmentBytesPerSample`. +/// +/// The color attachments can be from a render pass descriptor or a pipeline descriptor. +/// +/// Implements . +pub fn validate_color_attachment_bytes_per_sample( + attachment_formats: impl IntoIterator, + limit: u32, +) -> Result<(), crate::command::ColorAttachmentError> { + let mut total_bytes_per_sample: u32 = 0; + for format in attachment_formats { + let byte_cost = format.target_pixel_byte_cost().unwrap(); + let alignment = format.target_component_alignment().unwrap(); + + total_bytes_per_sample = total_bytes_per_sample.next_multiple_of(alignment); + total_bytes_per_sample += byte_cost; + } + + if total_bytes_per_sample > limit { + return Err( + crate::command::ColorAttachmentError::TooManyBytesPerSample { + total: total_bytes_per_sample, + limit, + }, + ); + } + + Ok(()) +} + +pub enum ShaderStageForValidation { + Vertex { + topology: wgt::PrimitiveTopology, + compare_function: Option, + }, + Mesh, + Fragment { + dual_source_blending: bool, + has_depth_attachment: bool, + }, + Compute, + Task, +} + +impl ShaderStageForValidation { + pub fn to_naga(&self) -> naga::ShaderStage { + match self { + Self::Vertex { .. } => naga::ShaderStage::Vertex, + Self::Mesh => naga::ShaderStage::Mesh, + Self::Fragment { .. } => naga::ShaderStage::Fragment, + Self::Compute => naga::ShaderStage::Compute, + Self::Task => naga::ShaderStage::Task, + } + } + + pub fn to_wgt_bit(&self) -> wgt::ShaderStages { + match self { + Self::Vertex { .. } => wgt::ShaderStages::VERTEX, + Self::Mesh => wgt::ShaderStages::MESH, + Self::Fragment { .. } => wgt::ShaderStages::FRAGMENT, + Self::Compute => wgt::ShaderStages::COMPUTE, + Self::Task => wgt::ShaderStages::TASK, + } + } +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/validation/shader_io_deductions.rs b/third_party/wgpu-core-29.0.4-patched/src/validation/shader_io_deductions.rs new file mode 100644 index 00000000..678fff5d --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/validation/shader_io_deductions.rs @@ -0,0 +1,176 @@ +use core::fmt::{self, Debug, Display, Formatter}; + +#[cfg(doc)] +#[expect(unused_imports)] +use crate::validation::StageError; + +/// Max shader I/O variable deductions for vertex shader output. Used by +/// [`StageError::TooManyUserDefinedVertexOutputs`] and +/// [`StageError::VertexOutputLocationTooLarge`]. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MaxVertexShaderOutputDeduction { + /// When a pipeline's [`crate::pipeline::RenderPipelineDescriptor::primitive`] is set to + /// [`wgt::PrimitiveTopology::PointList`]. + PointListPrimitiveTopology, +} + +impl MaxVertexShaderOutputDeduction { + pub fn for_variables(self) -> u32 { + match self { + Self::PointListPrimitiveTopology => 1, + } + } + + pub fn for_location(self) -> u32 { + match self { + Self::PointListPrimitiveTopology => 0, + } + } +} + +/// Max shader I/O variable deductions for vertex shader output. Used by +/// [`StageError::TooManyUserDefinedFragmentInputs`] and +/// [`StageError::FragmentInputLocationTooLarge`]. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MaxFragmentShaderInputDeduction { + InterStageBuiltIn(InterStageBuiltIn), +} + +impl MaxFragmentShaderInputDeduction { + pub fn for_variables(self) -> u32 { + match self { + Self::InterStageBuiltIn(builtin) => match builtin { + InterStageBuiltIn::FrontFacing + | InterStageBuiltIn::SampleIndex + | InterStageBuiltIn::SampleMask + | InterStageBuiltIn::PrimitiveIndex + | InterStageBuiltIn::SubgroupInvocationId + | InterStageBuiltIn::SubgroupSize + | InterStageBuiltIn::ViewIndex + | InterStageBuiltIn::PointCoord => 1, + InterStageBuiltIn::Barycentric => 3, + InterStageBuiltIn::Position => 0, + }, + } + } + + pub fn from_inter_stage_builtin(builtin: naga::BuiltIn) -> Option { + use naga::BuiltIn; + + Some(Self::InterStageBuiltIn(match builtin { + BuiltIn::Position { .. } => InterStageBuiltIn::Position, + BuiltIn::FrontFacing => InterStageBuiltIn::FrontFacing, + BuiltIn::SampleIndex => InterStageBuiltIn::SampleIndex, + BuiltIn::SampleMask => InterStageBuiltIn::SampleMask, + BuiltIn::PrimitiveIndex => InterStageBuiltIn::PrimitiveIndex, + BuiltIn::SubgroupSize => InterStageBuiltIn::SubgroupSize, + BuiltIn::SubgroupInvocationId => InterStageBuiltIn::SubgroupInvocationId, + BuiltIn::PointCoord => InterStageBuiltIn::PointCoord, + BuiltIn::Barycentric { .. } => InterStageBuiltIn::Barycentric, + BuiltIn::ViewIndex => InterStageBuiltIn::ViewIndex, + BuiltIn::BaseInstance + | BuiltIn::BaseVertex + | BuiltIn::ClipDistance + | BuiltIn::CullDistance + | BuiltIn::InstanceIndex + | BuiltIn::PointSize + | BuiltIn::VertexIndex + | BuiltIn::DrawIndex + | BuiltIn::FragDepth + | BuiltIn::GlobalInvocationId + | BuiltIn::LocalInvocationId + | BuiltIn::LocalInvocationIndex + | BuiltIn::WorkGroupId + | BuiltIn::WorkGroupSize + | BuiltIn::NumWorkGroups + | BuiltIn::NumSubgroups + | BuiltIn::SubgroupId + | BuiltIn::MeshTaskSize + | BuiltIn::CullPrimitive + | BuiltIn::PointIndex + | BuiltIn::LineIndices + | BuiltIn::TriangleIndices + | BuiltIn::VertexCount + | BuiltIn::Vertices + | BuiltIn::PrimitiveCount + | BuiltIn::Primitives + | BuiltIn::RayInvocationId + | BuiltIn::NumRayInvocations + | BuiltIn::InstanceCustomData + | BuiltIn::GeometryIndex + | BuiltIn::WorldRayOrigin + | BuiltIn::WorldRayDirection + | BuiltIn::ObjectRayOrigin + | BuiltIn::ObjectRayDirection + | BuiltIn::RayTmin + | BuiltIn::RayTCurrentMax + | BuiltIn::ObjectToWorld + | BuiltIn::WorldToObject + | BuiltIn::HitKind => return None, + })) + } +} + +/// A [`naga::BuiltIn`] that counts towards +/// a [`MaxFragmentShaderInputDeduction::InterStageBuiltIn`]. +/// +/// See also . +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum InterStageBuiltIn { + // Standard for WebGPU + Position, + FrontFacing, + SampleIndex, + SampleMask, + PrimitiveIndex, + SubgroupInvocationId, + SubgroupSize, + + // Non-standard + PointCoord, + Barycentric, + ViewIndex, +} + +pub(in crate::validation) fn display_deductions_as_optional_list( + deductions: &[T], + accessor: fn(&T) -> u32, +) -> impl Display + '_ +where + T: Debug, +{ + struct DisplayFromFn(F); + + impl Display for DisplayFromFn + where + F: Fn(&mut Formatter<'_>) -> fmt::Result, + { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let Self(inner) = self; + inner(f) + } + } + + DisplayFromFn(move |f: &mut Formatter<'_>| { + let relevant_deductions = deductions + .iter() + .map(|deduction| (deduction, accessor(deduction))) + .filter(|(_, effective_deduction)| *effective_deduction > 0); + if relevant_deductions.clone().next().is_some() { + writeln!(f, "; note that some deductions apply during validation:")?; + let mut wrote_something = false; + for deduction in deductions { + let deducted_amount = accessor(deduction); + if deducted_amount > 0 { + writeln!(f, "\n- {deduction:?}: {}", accessor(deduction))?; + wrote_something = true; + } + } + debug_assert!( + wrote_something, + "no substantial deductions found in error display" + ); + } + Ok(()) + }) +} diff --git a/third_party/wgpu-core-29.0.4-patched/src/weak_vec.rs b/third_party/wgpu-core-29.0.4-patched/src/weak_vec.rs new file mode 100644 index 00000000..af69a481 --- /dev/null +++ b/third_party/wgpu-core-29.0.4-patched/src/weak_vec.rs @@ -0,0 +1,66 @@ +//! Module containing the [`WeakVec`] API. + +use alloc::{sync::Weak, vec::Vec}; + +/// An optimized container for `Weak` references of `T` that minimizes reallocations by +/// dropping older elements that no longer have strong references to them. +#[derive(Debug)] +pub(crate) struct WeakVec { + inner: Vec>, +} + +impl Default for WeakVec { + fn default() -> Self { + Self { + inner: Default::default(), + } + } +} + +impl WeakVec { + pub(crate) fn new() -> Self { + Self { inner: Vec::new() } + } + + /// Pushes a new element to this collection. + /// + /// If the inner Vec needs to be reallocated, we will first drop older elements that + /// no longer have strong references to them. + pub(crate) fn push(&mut self, value: Weak) { + if self.inner.len() == self.inner.capacity() { + // Iterating backwards has the advantage that we don't do more work than we have to. + for i in (0..self.inner.len()).rev() { + if self.inner[i].strong_count() == 0 { + self.inner.swap_remove(i); + } + } + + // Make sure our capacity is twice the number of live elements. + // Leaving some spare capacity ensures that we won't re-scan immediately. + self.inner.reserve_exact(self.inner.len()); + } + + self.inner.push(value); + } +} + +pub(crate) struct WeakVecIter { + inner: alloc::vec::IntoIter>, +} + +impl Iterator for WeakVecIter { + type Item = Weak; + fn next(&mut self) -> Option { + self.inner.next() + } +} + +impl IntoIterator for WeakVec { + type Item = Weak; + type IntoIter = WeakVecIter; + fn into_iter(self) -> Self::IntoIter { + WeakVecIter { + inner: self.inner.into_iter(), + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/.cargo-ok b/third_party/wgpu-hal-29.0.4-patched/.cargo-ok new file mode 100644 index 00000000..5f8b7958 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/third_party/wgpu-hal-29.0.4-patched/.cargo_vcs_info.json b/third_party/wgpu-hal-29.0.4-patched/.cargo_vcs_info.json new file mode 100644 index 00000000..e2415048 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "e99f5305ded96ff7006f0714d043a7f735bd45c2" + }, + "path_in_vcs": "wgpu-hal" +} \ No newline at end of file diff --git a/third_party/wgpu-hal-29.0.4-patched/Cargo.lock b/third_party/wgpu-hal-29.0.4-patched/Cargo.lock new file mode 100644 index 00000000..d4a584ea --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/Cargo.lock @@ -0,0 +1,2512 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ab_glyph" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c0457472c38ea5bd1c3b5ada5e368271cb550be7a4ca4a0b4634e9913f6cc2" +dependencies = [ + "ab_glyph_rasterizer", + "owned_ttf_parser", +] + +[[package]] +name = "ab_glyph_rasterizer" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android-activity" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef6978589202a00cd7e118380c448a08b6ed394c3a8df3a430d0898e3a42d046" +dependencies = [ + "android-properties", + "bitflags 2.11.0", + "cc", + "cesu8", + "jni", + "jni-sys", + "libc", + "log", + "ndk", + "ndk-context", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "android-properties" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7eb209b1518d6bb87b283c20095f5228ecda460da70b44f0802523dea6da04" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "as-raw-xcb-connection" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175571dd1d178ced59193a6fc02dde1b972eb0bc56c892cde9beeceac5bf0f6b" + +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bit-set" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ddef2995421ab6a5c779542c81ee77c115206f4ad9d5a8e05f4ff49716a3dd" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +dependencies = [ + "objc2 0.5.2", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2 0.6.4", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "calloop" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b99da2f8558ca23c71f4fd15dc57c906239752dd27ff3c00a1d56b685b7cbfec" +dependencies = [ + "bitflags 2.11.0", + "log", + "polling", + "rustix 0.38.44", + "slab", + "thiserror 1.0.69", +] + +[[package]] +name = "calloop-wayland-source" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a66a987056935f7efce4ab5668920b5d0dac4a7c99991a67395f13702ddd20" +dependencies = [ + "calloop", + "rustix 0.38.44", + "wayland-backend", + "wayland-client", +] + +[[package]] +name = "cc" +version = "1.2.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "cgl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" +dependencies = [ + "libc", +] + +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core-graphics" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "libc", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "cursor-icon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" + +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.11.0", + "objc2 0.6.4", +] + +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dpi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "env_filter", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix 1.1.4", + "windows-link", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + +[[package]] +name = "glam" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f70749695b063ecbf6b62949ccccde2e733ec3ecbbd71d467dca4e5c6c97cca0" + +[[package]] +name = "glow" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29038e1c483364cc6bb3cf78feee1816002e127c331a1eec55a4d202b9e1adb5" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "glutin" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12124de845cacfebedff80e877bb37b5b75c34c5a4c89e47e1cdd67fb6041325" +dependencies = [ + "bitflags 2.11.0", + "cfg_aliases", + "cgl", + "dispatch2", + "glutin_egl_sys", + "glutin_wgl_sys", + "libloading", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "once_cell", + "raw-window-handle", + "wayland-sys", + "windows-sys 0.52.0", + "x11-dl", +] + +[[package]] +name = "glutin-winit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85edca7075f8fc728f28cb8fbb111a96c3b89e930574369e3e9c27eb75d3788f" +dependencies = [ + "cfg_aliases", + "glutin", + "raw-window-handle", + "winit", +] + +[[package]] +name = "glutin_egl_sys" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4680ba6195f424febdc3ba46e7a42a0e58743f2edb115297b86d7f8ecc02d2" +dependencies = [ + "gl_generator", + "windows-sys 0.52.0", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "gpu-allocator" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795" +dependencies = [ + "ash", + "hashbrown 0.16.1", + "log", + "presser", + "thiserror 2.0.18", + "windows", +] + +[[package]] +name = "gpu-descriptor" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" +dependencies = [ + "bitflags 2.11.0", + "gpu-descriptor-types", + "hashbrown 0.15.5", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +dependencies = [ + "bitflags 2.11.0", + "libc", + "plain", + "redox_syscall 0.7.3", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "mach-dxcompiler-rs" +version = "0.1.4+2024.11.22-df583a3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e3cd67e8ea2ba061339150970542cf1c60ba44c6d17e31279cbc133a4b018f8" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "naga" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2bf919621e7975acb27d881bae2fb993e0d45c8e0446e85e6272971e00dc8df" +dependencies = [ + "arrayvec", + "bit-set", + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases", + "codespan-reporting", + "half", + "hashbrown 0.16.1", + "hexf-parse", + "indexmap", + "libm", + "log", + "num-traits", + "once_cell", + "rustc-hash", + "spirv", + "thiserror 2.0.18", + "unicode-ident", +] + +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags 2.11.0", + "jni-sys", + "log", + "ndk-sys", + "num_enum", + "raw-window-handle", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" +dependencies = [ + "objc-sys", + "objc2-encode", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "libc", + "objc2 0.5.2", + "objc2-core-data", + "objc2-core-image", + "objc2-foundation 0.2.2", + "objc2-quartz-core 0.2.2", +] + +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.11.0", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74dd3b56391c7a0596a295029734d3c1c5e7e510a4cb30245f8221ccea96b009" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-core-location", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-contacts" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ff520e9c33812fd374d8deecef01d4a840e7b41862d849513de77e44aa4889" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-core-data" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.11.0", + "dispatch2", + "objc2 0.6.4", +] + +[[package]] +name = "objc2-core-image" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", + "objc2-metal 0.2.2", +] + +[[package]] +name = "objc2-core-location" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "000cfee34e683244f284252ee206a27953279d370e309649dc3ee317b37e5781" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.2", + "objc2-contacts", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "dispatch", + "libc", + "objc2 0.5.2", +] + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.11.0", + "objc2 0.6.4", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-link-presentation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a1ae721c5e35be65f01a03b6d2ac13a54cb4fa70d8a5da293d7b0020261398" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.2", + "objc2-app-kit 0.2.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-metal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags 2.11.0", + "block2 0.6.2", + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", + "objc2-metal 0.2.2", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" +dependencies = [ + "bitflags 2.11.0", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "objc2-metal 0.3.2", +] + +[[package]] +name = "objc2-symbols" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a684efe3dec1b305badae1a28f6555f6ddd3bb2c2267896782858d5a78404dc" +dependencies = [ + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-ui-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8bb46798b20cd6b91cbd113524c490f1686f4c4e8f49502431415f3512e2b6f" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-cloud-kit", + "objc2-core-data", + "objc2-core-image", + "objc2-core-location", + "objc2-foundation 0.2.2", + "objc2-link-presentation", + "objc2-quartz-core 0.2.2", + "objc2-symbols", + "objc2-uniform-type-identifiers", + "objc2-user-notifications", +] + +[[package]] +name = "objc2-uniform-type-identifiers" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fa5f9748dbfe1ca6c0b79ad20725a11eca7c2218bceb4b005cb1be26273bfe" +dependencies = [ + "block2 0.5.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "objc2-user-notifications" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76cfcbf642358e8689af64cee815d139339f3ed8ad05103ed5eaf73db8d84cb3" +dependencies = [ + "bitflags 2.11.0", + "block2 0.5.1", + "objc2 0.5.2", + "objc2-core-location", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "orbclient" +version = "0.3.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59aed3b33578edcfa1bc96a321d590d31832b6ad55a26f0313362ce687e9abd6" +dependencies = [ + "libc", + "libredox", +] + +[[package]] +name = "ordered-float" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +dependencies = [ + "num-traits", +] + +[[package]] +name = "owned_ttf_parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36820e9051aca1014ddc75770aab4d68bc1e9e632f0f5627c4086bc216fb583b" +dependencies = [ + "ttf-parser", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "profiling" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" + +[[package]] +name = "quick-xml" +version = "0.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "range-alloc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" + +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + +[[package]] +name = "raw-window-metal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d213455a5f1dc59214213c7330e074ddf8114c9a42411eb890c767357ce135" +dependencies = [ + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "objc2-quartz-core 0.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "redox_syscall" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sctk-adwaita" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6277f0217056f77f1d8f49f2950ac6c278c0d607c45f5ee99328d792ede24ec" +dependencies = [ + "ab_glyph", + "log", + "memmap2", + "smithay-client-toolkit", + "tiny-skia", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smithay-client-toolkit" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3457dea1f0eb631b4034d61d4d8c32074caa6cd1ab2d59f2327bd8461e2c0016" +dependencies = [ + "bitflags 2.11.0", + "calloop", + "calloop-wayland-source", + "cursor-icon", + "libc", + "log", + "memmap2", + "rustix 0.38.44", + "thiserror 1.0.69", + "wayland-backend", + "wayland-client", + "wayland-csd-frame", + "wayland-cursor", + "wayland-protocols", + "wayland-protocols-wlr", + "wayland-scanner", + "xkeysym", +] + +[[package]] +name = "smol_str" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +dependencies = [ + "serde", +] + +[[package]] +name = "spirv" +version = "0.4.0+sdk-1.4.341.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9571ea910ebd84c86af4b3ed27f9dbdc6ad06f17c5f96146b2b671e2976744f" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "strict-num" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiny-skia" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d13394d44dae3207b52a326c0c85a8bf87f1541f23b0d143811088497b09ab" +dependencies = [ + "arrayref", + "arrayvec", + "bytemuck", + "cfg-if", + "log", + "tiny-skia-path", +] + +[[package]] +name = "tiny-skia-path" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e7fc0c2e86a30b117d0462aa261b72b7a99b7ebd7deb3a14ceda95c5bdc93" +dependencies = [ + "arrayref", + "bytemuck", + "strict-num", +] + +[[package]] +name = "toml_datetime" +version = "1.0.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.4+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7193cbd0ce53dc966037f54351dbbcf0d5a642c7f0038c382ef9e677ce8c13f2" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.0.9+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +dependencies = [ + "winnow", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wayland-backend" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa75f400b7f719bcd68b3f47cd939ba654cedeef690f486db71331eec4c6a406" +dependencies = [ + "cc", + "downcast-rs", + "rustix 1.1.4", + "scoped-tls", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab51d9f7c071abeee76007e2b742499e535148035bb835f97aaed1338cf516c3" +dependencies = [ + "bitflags 2.11.0", + "rustix 1.1.4", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-csd-frame" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625c5029dbd43d25e6aa9615e88b829a5cad13b2819c4ae129fdbb7c31ab4c7e" +dependencies = [ + "bitflags 2.11.0", + "cursor-icon", + "wayland-backend", +] + +[[package]] +name = "wayland-cursor" +version = "0.31.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3298683470fbdc6ca40151dfc48c8f2fd4c41a26e13042f801f85002384091" +dependencies = [ + "rustix 1.1.4", + "wayland-client", + "xcursor", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b23b5df31ceff1328f06ac607591d5ba360cf58f90c8fad4ac8d3a55a3c4aec7" +dependencies = [ + "bitflags 2.11.0", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-plasma" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d392fc283a87774afc9beefcd6f931582bb97fe0e6ced0b306a62cb1d026527c" +dependencies = [ + "bitflags 2.11.0", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78248e4cc0eff8163370ba5c158630dcae1f3497a586b826eca2ef5f348d6235" +dependencies = [ + "bitflags 2.11.0", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86287151a309799b821ca709b7345a048a2956af05957c89cb824ab919fa4e3" +dependencies = [ + "proc-macro2", + "quick-xml", + "quote", +] + +[[package]] +name = "wayland-sys" +version = "0.31.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374f6b70e8e0d6bf9461a32988fd553b59ff630964924dad6e4a4eb6bd538d17" +dependencies = [ + "dlib", + "log", + "once_cell", + "pkg-config", +] + +[[package]] +name = "web-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wgpu-hal" +version = "29.0.4" +dependencies = [ + "android_system_properties", + "arrayvec", + "ash", + "bit-set", + "bitflags 2.11.0", + "block2 0.6.2", + "bytemuck", + "cfg-if", + "cfg_aliases", + "env_logger", + "glam", + "glow", + "glutin", + "glutin-winit", + "glutin_wgl_sys", + "gpu-allocator", + "gpu-descriptor", + "hashbrown 0.16.1", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "mach-dxcompiler-rs", + "naga", + "ndk-sys", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "objc2-metal 0.3.2", + "objc2-quartz-core 0.3.2", + "once_cell", + "ordered-float", + "parking_lot", + "portable-atomic", + "portable-atomic-util", + "profiling", + "range-alloc", + "raw-window-handle", + "raw-window-metal", + "renderdoc-sys", + "smallvec", + "thiserror 2.0.18", + "wasm-bindgen", + "wayland-sys", + "web-sys", + "wgpu-naga-bridge", + "wgpu-types", + "windows", + "windows-core", + "windows-result", + "winit", +] + +[[package]] +name = "wgpu-naga-bridge" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95226013f547544b223281cd16a4fb549aa9dcb562adbda0faae4c73ffbbc161" +dependencies = [ + "naga", + "wgpu-types", +] + +[[package]] +name = "wgpu-types" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bf84cd9ca8ca45e2b223a3868f1adf9bfc0c66aeac212e76ee7e40fdadf8f5" +dependencies = [ + "bitflags 2.11.0", + "bytemuck", + "js-sys", + "log", + "raw-window-handle", + "web-sys", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winit" +version = "0.30.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6755fa58a9f8350bd1e472d4c3fcc25f824ec358933bba33306d0b63df5978d" +dependencies = [ + "ahash", + "android-activity", + "atomic-waker", + "bitflags 2.11.0", + "block2 0.5.1", + "bytemuck", + "calloop", + "cfg_aliases", + "concurrent-queue", + "core-foundation", + "core-graphics", + "cursor-icon", + "dpi", + "js-sys", + "libc", + "memmap2", + "ndk", + "objc2 0.5.2", + "objc2-app-kit 0.2.2", + "objc2-foundation 0.2.2", + "objc2-ui-kit", + "orbclient", + "percent-encoding", + "pin-project", + "raw-window-handle", + "redox_syscall 0.4.1", + "rustix 0.38.44", + "sctk-adwaita", + "smithay-client-toolkit", + "smol_str", + "tracing", + "unicode-segmentation", + "wasm-bindgen", + "wasm-bindgen-futures", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-protocols-plasma", + "web-sys", + "web-time", + "windows-sys 0.52.0", + "x11-dl", + "x11rb", + "xkbcommon-dl", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "x11-dl" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38735924fedd5314a6e548792904ed8c6de6636285cb9fec04d5b1db85c1516f" +dependencies = [ + "libc", + "once_cell", + "pkg-config", +] + +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "as-raw-xcb-connection", + "gethostname", + "libc", + "libloading", + "once_cell", + "rustix 1.1.4", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + +[[package]] +name = "xcursor" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec9e4a500ca8864c5b47b8b482a73d62e4237670e5b5f1d6b9e3cae50f28f2b" + +[[package]] +name = "xkbcommon-dl" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039de8032a9a8856a6be89cea3e5d12fdd82306ab7c94d74e6deab2460651c5" +dependencies = [ + "bitflags 2.11.0", + "dlib", + "log", + "once_cell", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "zerocopy" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/third_party/wgpu-hal-29.0.4-patched/Cargo.toml b/third_party/wgpu-hal-29.0.4-patched/Cargo.toml new file mode 100644 index 00000000..f2546186 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/Cargo.toml @@ -0,0 +1,520 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.87" +name = "wgpu-hal" +version = "29.0.4" +authors = ["gfx-rs developers"] +build = "build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Hardware abstraction layer for wgpu, the cross-platform, safe, pure-rust graphics API" +homepage = "https://wgpu.rs/" +readme = "README.md" +keywords = ["graphics"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/gfx-rs/wgpu" + +[package.metadata.docs.rs] +features = [ + "metal", + "vulkan", + "gles", + "renderdoc", +] +rustdoc-args = [ + "--cfg", + "docsrs", +] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "wasm32-unknown-unknown", +] + +[package.metadata.cargo-machete] +ignored = ["cfg_aliases"] + +[features] +device_lost_panic = [] +dx12 = [ + "dep:arrayvec", + "dep:bit-set", + "dep:bytemuck", + "dep:gpu-allocator", + "dep:hashbrown", + "dep:libloading", + "dep:once_cell", + "dep:ordered-float", + "dep:parking_lot", + "dep:profiling", + "dep:range-alloc", + "dep:windows-core", + "gpu-allocator/d3d12", + "naga/hlsl-out", + "once_cell/std", + "windows/Win32_Devices_DeviceAndDriverInstallation", + "windows/Win32_Graphics_Direct3D_Dxc", + "windows/Win32_Graphics_Direct3D_Fxc", + "windows/Win32_Graphics_Direct3D", + "windows/Win32_Graphics_Direct3D12", + "windows/Win32_Graphics_DirectComposition", + "windows/Win32_Graphics_Dxgi_Common", + "windows/Win32_Security", + "windows/Win32_System_Diagnostics_Debug", + "windows/Win32_System_Kernel", + "windows/Win32_System_Performance", + "windows/Win32_System_Threading", + "windows/Win32_UI_WindowsAndMessaging", +] +fragile-send-sync-non-atomic-wasm = ["wgpu-types/fragile-send-sync-non-atomic-wasm"] +gles = [ + "dep:arrayvec", + "dep:bytemuck", + "dep:glow", + "dep:glutin_wgl_sys", + "dep:hashbrown", + "dep:js-sys", + "dep:khronos-egl", + "dep:libloading", + "dep:ndk-sys", + "dep:objc2", + "dep:parking_lot", + "dep:profiling", + "dep:wasm-bindgen", + "dep:wayland-sys", + "dep:web-sys", + "dep:windows-result", + "naga/glsl-out", + "wgpu-types/web", + "windows-result/std", + "windows/Win32_Graphics_Gdi", + "windows/Win32_Graphics_OpenGL", + "windows/Win32_System_LibraryLoader", + "windows/Win32_UI_WindowsAndMessaging", +] +internal_error_panic = [] +metal = [ + "naga/msl-out", + "dep:arrayvec", + "dep:block2", + "dep:bytemuck", + "dep:hashbrown", + "dep:libc", + "dep:objc2", + "dep:objc2-core-foundation", + "dep:objc2-foundation", + "dep:objc2-metal", + "dep:objc2-quartz-core", + "dep:parking_lot", + "dep:profiling", + "dep:smallvec", + "dep:raw-window-metal", +] +portable-atomic = [ + "dep:portable-atomic", + "dep:portable-atomic-util", +] +renderdoc = [ + "dep:libloading", + "dep:renderdoc-sys", +] +static-dxc = ["dep:mach-dxcompiler-rs"] +validation_canary = ["dep:parking_lot"] +vulkan = [ + "naga/spv-out", + "dep:android_system_properties", + "dep:arrayvec", + "dep:ash", + "dep:bytemuck", + "dep:gpu-descriptor", + "dep:hashbrown", + "dep:libc", + "dep:libloading", + "dep:ordered-float", + "dep:parking_lot", + "dep:profiling", + "dep:raw-window-metal", + "dep:smallvec", + "dep:windows", + "gpu-allocator/vulkan", + "windows/Win32", +] + +[lib] +name = "wgpu_hal" +path = "src/lib.rs" + +[[example]] +name = "halmark" +path = "examples/halmark/main.rs" + +[[example]] +name = "raw-gles" +path = "examples/raw-gles.rs" +required-features = ["gles"] + +[[example]] +name = "ray-traced-triangle" +path = "examples/ray-traced-triangle/main.rs" + +[dependencies.arrayvec] +version = "0.7.1" +optional = true +default-features = false + +[dependencies.bitflags] +version = "2.9" + +[dependencies.bytemuck] +version = "1.22" +features = [ + "extern_crate_alloc", + "min_const_generics", + "derive", +] +optional = true + +[dependencies.cfg-if] +version = "1" + +[dependencies.glow] +version = "0.17" +optional = true + +[dependencies.hashbrown] +version = "0.16" +features = [ + "default-hasher", + "inline-more", +] +optional = true +default-features = false + +[dependencies.log] +version = "0.4.29" + +[dependencies.naga] +version = "29.0.1" + +[dependencies.ordered-float] +version = ">=3, <6.0" +optional = true +default-features = false + +[dependencies.parking_lot] +version = "0.12.3" +optional = true + +[dependencies.profiling] +version = "1.0.1" +optional = true +default-features = false + +[dependencies.raw-window-handle] +version = "0.6.2" +default-features = false + +[dependencies.thiserror] +version = "2.0.12" +default-features = false + +[dependencies.wgpu-naga-bridge] +version = "29.0.1" + +[dependencies.wgpu-types] +version = "29.0.1" +default-features = false + +[dev-dependencies.env_logger] +version = "0.11" +default-features = false + +[dev-dependencies.glam] +version = "0.32.0" + +[dev-dependencies.naga] +version = "29.0.1" +features = [ + "wgsl-in", + "termcolor", +] + +[dev-dependencies.winit] +version = "0.30.8" +features = ["android-native-activity"] + +[build-dependencies.cfg_aliases] +version = "0.2.1" + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.js-sys] +version = "0.3.85" +optional = true +default-features = true + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.wasm-bindgen] +version = "0.2.108" +optional = true +default-features = false + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies.web-sys] +version = "0.3.85" +features = [ + "default", + "Window", + "HtmlCanvasElement", + "WebGl2RenderingContext", + "OffscreenCanvas", +] +optional = true +default-features = false + +[target.'cfg(all(windows, not(target_arch = "aarch64"), target_env = "msvc"))'.dependencies.mach-dxcompiler-rs] +version = "0.1.4" +optional = true +default-features = false + +[target.'cfg(any(not(target_has_atomic = "64"), not(target_has_atomic = "ptr")))'.dependencies.portable-atomic] +version = "1.10" +optional = true + +[target.'cfg(not(any(target_arch = "wasm32", target_os = "ios", target_os = "visionos", target_env = "ohos")))'.dev-dependencies.glutin] +version = "0.32" +features = [ + "egl", + "wgl", + "wayland", + "x11", +] +default-features = false + +[target.'cfg(not(any(target_arch = "wasm32", target_os = "ios", target_os = "visionos", target_env = "ohos")))'.dev-dependencies.glutin-winit] +version = "0.5" +features = [ + "egl", + "wgl", + "wayland", + "x11", +] +default-features = false + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.ash] +version = "0.38" +optional = true + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.gpu-allocator] +version = "0.28" +features = ["hashbrown"] +optional = true +default-features = false + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.gpu-descriptor] +version = "0.3.2" +optional = true + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.khronos-egl] +version = "6" +features = ["dynamic"] +optional = true + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.libloading] +version = "0.8" +optional = true + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.renderdoc-sys] +version = "1" +optional = true + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.smallvec] +version = "1.14" +features = ["union"] +optional = true + +[target.'cfg(not(target_has_atomic = "ptr"))'.dependencies.portable-atomic-util] +version = "0.2.4" +features = ["alloc"] +optional = true + +[target.'cfg(target_os = "android")'.dependencies.android_system_properties] +version = "0.1.1" +optional = true + +[target.'cfg(target_os = "android")'.dependencies.ndk-sys] +version = "0.6" +optional = true + +[target.'cfg(target_os = "emscripten")'.dependencies.khronos-egl] +version = "6" +features = [ + "static", + "no-pkg-config", +] +optional = true + +[target.'cfg(target_os = "emscripten")'.dependencies.libloading] +version = "0.8" +optional = true + +[target.'cfg(target_vendor = "apple")'.dependencies.block2] +version = "0.6.2" +optional = true + +[target.'cfg(target_vendor = "apple")'.dependencies.bytemuck] +version = "1.22" +features = [ + "extern_crate_alloc", + "min_const_generics", +] +optional = true + +[target.'cfg(target_vendor = "apple")'.dependencies.objc2] +version = "0.6.3" +optional = true + +[target.'cfg(target_vendor = "apple")'.dependencies.objc2-core-foundation] +version = "0.3.2" +features = [ + "std", + "CFCGTypes", +] +optional = true +default-features = false + +[target.'cfg(target_vendor = "apple")'.dependencies.objc2-foundation] +version = "0.3.2" +features = [ + "std", + "NSError", + "NSProcessInfo", + "NSRange", + "NSString", +] +optional = true +default-features = false + +[target.'cfg(target_vendor = "apple")'.dependencies.objc2-metal] +version = "0.3.2" +features = [ + "std", + "block2", + "MTLAllocation", + "MTLBlitCommandEncoder", + "MTLBlitPass", + "MTLBuffer", + "MTLCaptureManager", + "MTLCaptureScope", + "MTLCommandBuffer", + "MTLCommandEncoder", + "MTLCommandQueue", + "MTLComputeCommandEncoder", + "MTLComputePass", + "MTLComputePipeline", + "MTLCounters", + "MTLDepthStencil", + "MTLDevice", + "MTLDrawable", + "MTLEvent", + "MTLLibrary", + "MTLPipeline", + "MTLPixelFormat", + "MTLRenderCommandEncoder", + "MTLRenderPass", + "MTLRenderPipeline", + "MTLResource", + "MTLSampler", + "MTLStageInputOutputDescriptor", + "MTLTexture", + "MTLTypes", + "MTLVertexDescriptor", + "MTLAccelerationStructure", + "MTLAccelerationStructureTypes", + "MTLAccelerationStructureCommandEncoder", + "MTLResidencySet", +] +optional = true +default-features = false + +[target.'cfg(target_vendor = "apple")'.dependencies.objc2-quartz-core] +version = "0.3.2" +features = [ + "std", + "objc2-core-foundation", + "CALayer", + "CAMetalLayer", + "objc2-metal", +] +optional = true +default-features = false + +[target.'cfg(target_vendor = "apple")'.dependencies.raw-window-metal] +version = "1.0" +optional = true + +[target."cfg(unix)".dependencies.libc] +version = "0.2.172" +optional = true +default-features = false + +[target."cfg(unix)".dependencies.wayland-sys] +version = "0.31.3" +features = [ + "client", + "dlopen", + "egl", +] +optional = true + +[target."cfg(windows)".dependencies.bit-set] +version = "0.9" +optional = true +default-features = false + +[target."cfg(windows)".dependencies.glutin_wgl_sys] +version = "0.6" +optional = true + +[target."cfg(windows)".dependencies.once_cell] +version = "1.21" +optional = true +default-features = false + +[target."cfg(windows)".dependencies.range-alloc] +version = "0.1" +optional = true + +[target."cfg(windows)".dependencies.windows] +version = "0.62" +optional = true +default-features = false + +[target."cfg(windows)".dependencies.windows-core] +version = "0.62" +optional = true +default-features = false + +[target."cfg(windows)".dependencies.windows-result] +version = "0.4" +optional = true +default-features = false + +[lints.rust.unexpected_cfgs] +level = "warn" +priority = 0 +check-cfg = [ + 'cfg(feature, values("cargo-clippy"))', + "cfg(web_sys_unstable_apis)", +] diff --git a/third_party/wgpu-hal-29.0.4-patched/Cargo.toml.orig b/third_party/wgpu-hal-29.0.4-patched/Cargo.toml.orig new file mode 100644 index 00000000..30265268 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/Cargo.toml.orig @@ -0,0 +1,355 @@ +[package] +name = "wgpu-hal" +version.workspace = true +authors.workspace = true +edition.workspace = true +description = "Hardware abstraction layer for wgpu, the cross-platform, safe, pure-rust graphics API" +homepage.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true + +# Override the workspace's `rust-version` key. `wgpu-core` and its dependencies +# have a less strict MSRV, to allow firefox more leeway in updating their Rust toolchain. +# +# See the repo README for more information on MSRV policy. +rust-version = "1.87" + +[package.metadata.docs.rs] +# Ideally we would enable all the features. +# +# However, the dx12 features fail to be documented because the docs.rs runner cross-compiling under +# x86_64-unknown-linux-gnu cannot compile in that environment at the moment. +features = ["metal", "vulkan", "gles", "renderdoc"] +rustdoc-args = ["--cfg", "docsrs"] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "wasm32-unknown-unknown", +] + +[package.metadata.cargo-machete] +# Cargo machete can't check build.rs dependencies. See https://github.com/bnjbvr/cargo-machete/issues/100 +ignored = ["cfg_aliases"] + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = [ + 'cfg(feature, values("cargo-clippy"))', # objc's `msg_send` macro injects this in our code https://github.com/SSheldon/rust-objc/issues/125 + 'cfg(web_sys_unstable_apis)', # web-sys uses this +] } + +[lib] + +[features] + +######################## +### Backend Features ### +######################## + +# The interaction of features between wgpu-core and wgpu-hal is a bit nuanced to get +# the desired behavior on all platforms. +# +# At the wgpu-hal level the features are defined to enable the backends on all platforms +# that can compile the backend. Vulkan for example will have an effect on Windows, Mac, Linux, and Android. +# This is done with target conditional dependencies in wgpu-hal. This allows `--all-features` +# to compile on all platforms. +# +# wgpu-core's features are defined to enable the backends on their "default" platforms. For example we +# exclude the Vulkan backend on MacOS unless a separate feature `vulkan-portability` is enabled. In response +# to these features, it enables features of platform specific crates. For example, the `vulkan` feature in wgpu-core +# enables the `vulkan` feature in `wgpu-core-deps-windows-linux-android` which in turn enables the +# `vulkan` feature in `wgpu-hal` _only_ on those platforms. If you enable the `vulkan-portability` feature, it +# will enable the `vulkan` feature in `wgpu-core-deps-apple`. The only way to do this is unfortunately to have +# a separate crate for each platform category that participates in the feature unification. +# +# This trick doesn't work at the `wgpu` level, because the `wgpu` -> `wgpu-core` dependency is conditional, +# making the Cargo.toml signifigantly more complicated in all areas. +# +# See https://github.com/gfx-rs/wgpu/issues/3514, https://github.com/gfx-rs/wgpu/pull/7076, +# and https://github.com/rust-lang/cargo/issues/1197 for more information. + +## Enables the Metal backend when targeting Apple platforms. +metal = [ + # Metal is only available on Apple platforms, therefore request MSL output also only if we target an Apple platform. + "naga/msl-out", + "dep:arrayvec", + "dep:block2", + "dep:bytemuck", + "dep:hashbrown", + "dep:libc", + "dep:objc2", + "dep:objc2-core-foundation", + "dep:objc2-foundation", + "dep:objc2-metal", + "dep:objc2-quartz-core", + "dep:parking_lot", + "dep:profiling", + "dep:smallvec", + "dep:raw-window-metal", +] +vulkan = [ + "naga/spv-out", + "dep:android_system_properties", + "dep:arrayvec", + "dep:ash", + "dep:bytemuck", + "dep:gpu-descriptor", + "dep:hashbrown", + "dep:libc", + "dep:libloading", + "dep:ordered-float", + "dep:parking_lot", + "dep:profiling", + "dep:raw-window-metal", + "dep:smallvec", + "dep:windows", + "gpu-allocator/vulkan", + "windows/Win32", +] +gles = [ + "dep:arrayvec", + "dep:bytemuck", + "dep:glow", + "dep:glutin_wgl_sys", + "dep:hashbrown", + "dep:js-sys", + "dep:khronos-egl", + "dep:libloading", + "dep:ndk-sys", + "dep:objc2", + "dep:parking_lot", + "dep:profiling", + "dep:wasm-bindgen", + "dep:wayland-sys", + "dep:web-sys", + "dep:windows-result", + "naga/glsl-out", + "wgpu-types/web", + "windows-result/std", # Need to enable this for the `Error` impl on `windows_result::Error` to work. + "windows/Win32_Graphics_Gdi", + "windows/Win32_Graphics_OpenGL", + "windows/Win32_System_LibraryLoader", + "windows/Win32_UI_WindowsAndMessaging", +] +## Enables the DX12 backend when targeting Windows. +dx12 = [ + "dep:arrayvec", + "dep:bit-set", + "dep:bytemuck", + "dep:gpu-allocator", + "dep:hashbrown", + "dep:libloading", + "dep:once_cell", + "dep:ordered-float", + "dep:parking_lot", + "dep:profiling", + "dep:range-alloc", + "dep:windows-core", + "gpu-allocator/d3d12", + "naga/hlsl-out", + "once_cell/std", + "windows/Win32_Devices_DeviceAndDriverInstallation", + "windows/Win32_Graphics_Direct3D_Dxc", + "windows/Win32_Graphics_Direct3D_Fxc", + "windows/Win32_Graphics_Direct3D", + "windows/Win32_Graphics_Direct3D12", + "windows/Win32_Graphics_DirectComposition", + "windows/Win32_Graphics_Dxgi_Common", + "windows/Win32_Security", + "windows/Win32_System_Diagnostics_Debug", + "windows/Win32_System_Kernel", + "windows/Win32_System_Performance", + "windows/Win32_System_Threading", + "windows/Win32_UI_WindowsAndMessaging", +] + +########################### +### Misc Other Features ### +########################### + +static-dxc = ["dep:mach-dxcompiler-rs"] +renderdoc = ["dep:libloading", "dep:renderdoc-sys"] +fragile-send-sync-non-atomic-wasm = [ + "wgpu-types/fragile-send-sync-non-atomic-wasm", +] +portable-atomic = ["dep:portable-atomic", "dep:portable-atomic-util"] + +################################### +### Internal Debugging Features ### +################################### + +# Panic when running into a device lost error (for debugging purposes). +# Only affects the d3d12 and vulkan backends. +device_lost_panic = [] +# Panic when running into an internal error other than out-of-memory and device lost +# (for debugging purposes). +# +# Only affects the d3d12 and vulkan backends. +internal_error_panic = [] +# Tracks validation errors in a `VALIDATION_CANARY` static. +validation_canary = ["dep:parking_lot"] + +[[example]] +name = "halmark" + +[[example]] +name = "raw-gles" +required-features = ["gles"] + +##################### +### Platform: All ### +##################### + +[dependencies] +naga.workspace = true +wgpu-naga-bridge.workspace = true +wgpu-types = { workspace = true, default-features = false } + +# Dependencies in the lib and empty backend +bitflags.workspace = true +cfg-if.workspace = true +log = { workspace = true } +parking_lot = { workspace = true, optional = true } +raw-window-handle.workspace = true +thiserror.workspace = true + +# Target agnostic dependencies used only in backends. +arrayvec = { workspace = true, optional = true } +bytemuck = { workspace = true, optional = true, features = ["derive"] } +hashbrown = { workspace = true, optional = true } +ordered-float = { workspace = true, optional = true } +profiling = { workspace = true, optional = true, default-features = false } + +# Backend: GLES +glow = { workspace = true, optional = true } + +######################## +### Platform: Native ### +######################## + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# Backend: Vulkan and Dx12 +gpu-allocator = { workspace = true, optional = true } +# Backend: Vulkan +ash = { workspace = true, optional = true } +gpu-descriptor = { workspace = true, optional = true } +smallvec = { workspace = true, optional = true, features = ["union"] } +# Backend: GLES +khronos-egl = { workspace = true, features = ["dynamic"], optional = true } +libloading = { workspace = true, optional = true } +renderdoc-sys = { workspace = true, optional = true } + +########################## +### Platform: All Unix ### +########################## + +[target.'cfg(unix)'.dependencies] +# Backend: Vulkan +libc = { workspace = true, optional = true } +# backend: GLES +wayland-sys = { version = "0.31.3", features = [ + "client", + "dlopen", + "egl", +], optional = true } + +######################### +### Platform: Windows ### +######################### + +[target.'cfg(windows)'.dependencies] +# Backend: Dx12 and GLES +windows = { workspace = true, optional = true } +windows-core = { workspace = true, optional = true } +windows-result = { workspace = true, optional = true } +# Backend: Dx12 +bit-set = { workspace = true, optional = true } +range-alloc = { workspace = true, optional = true } +once_cell = { workspace = true, optional = true } +# backend: GLES +glutin_wgl_sys = { workspace = true, optional = true } + +### Platform: x86/x86_64 Windows ### +# This doesn't support aarch64. See https://github.com/gfx-rs/wgpu/issues/6860. +# This doesn't support x86_64-pc-windows-gnu. See https://github.com/gfx-rs/wgpu/issues/8303 +# +# ⚠️ Keep in sync with static_dxc cfg in build.rs and cfg_alias in `wgpu` crate ⚠️ +[target.'cfg(all(windows, not(target_arch = "aarch64"), target_env = "msvc"))'.dependencies] +mach-dxcompiler-rs = { workspace = true, optional = true } + +####################### +### Platform: Apple ### +####################### + +[target.'cfg(target_vendor = "apple")'.dependencies] +# Backend: Metal +block2 = { workspace = true, optional = true } +bytemuck = { workspace = true, optional = true } +objc2 = { workspace = true, optional = true } +objc2-core-foundation = { workspace = true, optional = true } +objc2-foundation = { workspace = true, optional = true } +objc2-metal = { workspace = true, optional = true } +objc2-quartz-core = { workspace = true, optional = true } + +# backend: Metal + Vulkan +raw-window-metal = { workspace = true, optional = true } + +######################### +### Platform: Android ### +######################### + +[target.'cfg(target_os = "android")'.dependencies] +android_system_properties = { workspace = true, optional = true } +ndk-sys = { workspace = true, optional = true } + +############################# +### Platform: Webassembly ### +############################# + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies] +# Backend: GLES +wasm-bindgen = { workspace = true, optional = true } +web-sys = { workspace = true, optional = true, features = [ + "default", + "Window", + "HtmlCanvasElement", + "WebGl2RenderingContext", + "OffscreenCanvas", +] } +js-sys = { workspace = true, optional = true, default-features = true } + +############################ +### Platform: Emscripten ### +############################ + +[target.'cfg(target_os = "emscripten")'.dependencies] +# Backend: GLES +khronos-egl = { workspace = true, optional = true, features = [ + "static", + "no-pkg-config", +] } +# Note: it's unused by emscripten, but we keep it to have single code base in egl.rs +libloading = { workspace = true, optional = true } + +[target.'cfg(any(not(target_has_atomic = "64"), not(target_has_atomic = "ptr")))'.dependencies] +portable-atomic = { workspace = true, optional = true } + +[target.'cfg(not(target_has_atomic = "ptr"))'.dependencies] +portable-atomic-util = { workspace = true, features = [ + "alloc", +], optional = true } + +[build-dependencies] +cfg_aliases.workspace = true + +[dev-dependencies] +env_logger.workspace = true +glam.workspace = true # for ray-traced-triangle example +naga = { workspace = true, features = ["wgsl-in", "termcolor"] } +winit.workspace = true # for "halmark" example + +### Platform: Windows + MacOS + Linux for "raw-gles" example ### +[target.'cfg(not(any(target_arch = "wasm32", target_os = "ios", target_os = "visionos", target_env = "ohos")))'.dev-dependencies] +glutin-winit = { workspace = true, features = ["egl", "wgl", "wayland", "x11"] } +glutin = { workspace = true, features = ["egl", "wgl", "wayland", "x11"] } diff --git a/third_party/wgpu-hal-29.0.4-patched/LICENSE.APACHE b/third_party/wgpu-hal-29.0.4-patched/LICENSE.APACHE new file mode 100644 index 00000000..d9a10c0d --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/LICENSE.APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/third_party/wgpu-hal-29.0.4-patched/LICENSE.MIT b/third_party/wgpu-hal-29.0.4-patched/LICENSE.MIT new file mode 100644 index 00000000..8d02e4db --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/LICENSE.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 The gfx-rs developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md b/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md new file mode 100644 index 00000000..01157fb8 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md @@ -0,0 +1,20 @@ +# Patch provenance + +Source: `wgpu-hal` 29.0.4 from crates.io, checksum +`97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d`. + +Local change: add three Metal-only retained raw-handle accessors, for the +selected `MTLDevice`, its `MTLCommandQueue`, and an `MTLBuffer`. Each accessor +transfers a single Objective-C +1 retain as an opaque pointer. `j2k-ml` +immediately adopts that retain into the existing `metal` crate owner while the +corresponding wgpu resource guard is live; raw handles do not enter its public +API. The queue accessor lets `j2k-ml` pass Burn's exact native queue to the +codec before producer commit. Exact-queue submissions rely on queue order and +allocate no event bridge. A same-device, different-queue codec caller uses a +session-owned `MTLEvent` timeline; the legacy API that chooses a consumer queue +after submission retains its compatibility `MTLSharedEvent` bridge. The Burn +adapter uses the exact-queue route. + +This patch exists only for Burn/wgpu-to-J2K external destination interop. It +must be removed when the targeted wgpu release exposes an equivalent audited +Metal buffer/device ownership bridge. diff --git a/third_party/wgpu-hal-29.0.4-patched/README.md b/third_party/wgpu-hal-29.0.4-patched/README.md new file mode 100644 index 00000000..c1abf60a --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/README.md @@ -0,0 +1,120 @@ +# `wgpu_hal`: a cross-platform unsafe graphics abstraction + +This crate defines a set of traits abstracting over modern graphics APIs, +with implementations ("backends") for Vulkan, Metal, Direct3D, and GL. + +`wgpu_hal` is a spiritual successor to +[gfx-hal](https://github.com/gfx-rs/gfx), but with reduced scope, and +oriented towards WebGPU implementation goals. It has no overhead for +validation or tracking, and the API translation overhead is kept to the bare +minimum by the design of WebGPU. This API can be used for resource-demanding +applications and engines. + +The `wgpu_hal` crate's main design choices: + +- Our traits are meant to be *portable*: proper use + should get equivalent results regardless of the backend. + +- Our traits' contracts are *unsafe*: implementations perform minimal + validation, if any, and incorrect use will often cause undefined behavior. + This allows us to minimize the overhead we impose over the underlying + graphics system. If you need safety, the [`wgpu-core`] crate provides a + safe API for driving `wgpu_hal`, implementing all necessary validation, + resource state tracking, and so on. (Note that `wgpu-core` is designed for + use via FFI; the [`wgpu`] crate provides more idiomatic Rust bindings for + `wgpu-core`.) Or, you can do your own validation. + +- In the same vein, returned errors *only cover cases the user can't + anticipate*, like running out of memory or losing the device. Any errors + that the user could reasonably anticipate are their responsibility to + avoid. For example, `wgpu_hal` returns no error for mapping a buffer that's + not mappable: as the buffer creator, the user should already know if they + can map it. + +- We use *static dispatch*. The traits are not + generally object-safe. You must select a specific backend type + like [`vulkan::Api`] or [`metal::Api`], and then use that + according to the main traits, or call backend-specific methods. + +- We use *idiomatic Rust parameter passing*, + taking objects by reference, returning them by value, and so on, + unlike `wgpu-core`, which refers to objects by ID. + +- We map buffer contents *persistently*. This means that the buffer + can remain mapped on the CPU while the GPU reads or writes to it. + You must explicitly indicate when data might need to be + transferred between CPU and GPU, if `wgpu_hal` indicates that the + mapping is not coherent (that is, automatically synchronized + between the two devices). + +- You must record *explicit barriers* between different usages of a + resource. For example, if a buffer is written to by a compute + shader, and then used as and index buffer to a draw call, you + must use [`CommandEncoder::transition_buffers`] between those two + operations. + +- Pipeline layouts are *explicitly specified* when setting bind + group. Incompatible layouts disturb groups bound at higher indices. + +- The API *accepts collections as iterators*, to avoid forcing the user to + store data in particular containers. The implementation doesn't guarantee + that any of the iterators are drained, unless stated otherwise by the + function documentation. For this reason, we recommend that iterators don't + do any mutating work. + +Unfortunately, `wgpu_hal`'s safety requirements are not fully documented. +Ideally, all trait methods would have doc comments setting out the +requirements users must meet to ensure correct and portable behavior. If you +are aware of a specific requirement that a backend imposes that is not +ensured by the traits' documented rules, please file an issue. Or, if you are +a capable technical writer, please file a pull request! + +[`wgpu-core`]: https://crates.io/crates/wgpu-core +[`wgpu`]: https://crates.io/crates/wgpu +[`vulkan::Api`]: vulkan/struct.Api.html +[`metal::Api`]: metal/struct.Api.html + +## Primary backends + +The `wgpu_hal` crate has full-featured backends implemented on the following +platform graphics APIs: + +- Vulkan, available on Linux, Android, and Windows, using the [`ash`] crate's + Vulkan bindings. It's also available on macOS, if you install [MoltenVK]. + +- Metal on macOS, using the [`metal`] crate's bindings. + +- Direct3D 12 on Windows, using the [`windows`] crate's bindings. + +[`ash`]: https://crates.io/crates/ash +[MoltenVK]: https://github.com/KhronosGroup/MoltenVK +[`metal`]: https://crates.io/crates/metal +[`windows`]: https://crates.io/crates/windows + +## Secondary backends + +The `wgpu_hal` crate has a partial implementation based on the following +platform graphics API: + +- The GL backend is available anywhere OpenGL, OpenGL ES, or WebGL are + available. See the [`gles`] module documentation for details. + +[`gles`]: gles/index.html + +You can see what capabilities an adapter is missing by checking the +[`DownlevelCapabilities`][tdc] in [`ExposedAdapter::capabilities`], available +from [`Instance::enumerate_adapters`]. + +The API is generally designed to fit the primary backends better than the +secondary backends, so the latter may impose more overhead. + +[tdc]: wgt::DownlevelCapabilities + +## Debugging + +Most of the information on the wiki [Debugging wgpu Applications][wiki-debug] +page still applies to this API, with the exception of API tracing/replay +functionality, which is only available in `wgpu-core`. + +[wiki-debug]: https://github.com/gfx-rs/wgpu/wiki/Debugging-wgpu-Applications + diff --git a/third_party/wgpu-hal-29.0.4-patched/build.rs b/third_party/wgpu-hal-29.0.4-patched/build.rs new file mode 100644 index 00000000..7bb56e99 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/build.rs @@ -0,0 +1,32 @@ +fn main() { + cfg_aliases::cfg_aliases! { + native: { not(target_arch = "wasm32") }, + send_sync: { any( + not(target_arch = "wasm32"), + all(feature = "fragile-send-sync-non-atomic-wasm", not(target_feature = "atomics")) + ) }, + webgl: { all(target_arch = "wasm32", not(target_os = "emscripten"), gles) }, + Emscripten: { all(target_os = "emscripten", gles) }, + dx12: { all(target_os = "windows", feature = "dx12") }, + gles: { all(feature = "gles") }, + // Within the GL ES backend, use `std` and be Send + Sync only if we are using a target + // that, among the ones where the GL ES backend is supported, has `std`. + gles_with_std: { all( + feature = "gles", + any( + not(target_arch = "wasm32"), + // Accept wasm32-unknown-unknown, which uniquely has a stub `std` + all(target_vendor = "unknown", target_os = "unknown"), + // Accept wasm32-unknown-emscripten and similar, which has a real `std` + target_os = "emscripten" + ) + ) }, + metal: { all(target_vendor = "apple", feature = "metal") }, + vulkan: { all(not(target_arch = "wasm32"), feature = "vulkan") }, + any_backend: { any(dx12, metal, vulkan, gles) }, + // ⚠️ Keep in sync with target.cfg() definition in Cargo.toml and cfg_alias in `wgpu` crate ⚠️ + static_dxc: { all(target_os = "windows", feature = "static-dxc", not(target_arch = "aarch64"), target_env = "msvc") }, + supports_64bit_atomics: { target_has_atomic = "64" }, + supports_ptr_atomics: { target_has_atomic = "ptr" } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/examples/halmark/main.rs b/third_party/wgpu-hal-29.0.4-patched/examples/halmark/main.rs new file mode 100644 index 00000000..9983de95 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/examples/halmark/main.rs @@ -0,0 +1,918 @@ +//! This example shows basic usage of wgpu-hal by rendering +//! a ton of moving sprites, each with a separate texture and draw call. +extern crate wgpu_hal as hal; + +use hal::{ + Adapter as _, CommandEncoder as _, Device as _, Instance as _, Queue as _, Surface as _, +}; +use raw_window_handle::{HasDisplayHandle, HasWindowHandle}; +use winit::{ + application::ApplicationHandler, + event::{ElementState, KeyEvent, WindowEvent}, + event_loop::{ActiveEventLoop, ControlFlow}, + keyboard::{Key, NamedKey}, + window::Window, +}; + +use std::{ + borrow::{Borrow, Cow}, + iter, + num::NonZeroU64, + ptr, + time::Instant, +}; + +const MAX_BUNNIES: usize = 1 << 20; +const BUNNY_SIZE: f32 = 0.15 * 256.0; +const GRAVITY: f32 = -9.8 * 100.0; +const MAX_VELOCITY: f32 = 750.0; +const DESIRED_MAX_LATENCY: u32 = 2; + +#[repr(C)] +#[derive(Clone, Copy)] +struct Globals { + mvp: [[f32; 4]; 4], + size: [f32; 2], + pad: [f32; 2], +} + +#[repr(C, align(256))] +#[derive(Clone, Copy)] +struct Locals { + position: [f32; 2], + velocity: [f32; 2], + color: u32, + _pad: u32, +} + +struct ExecutionContext { + encoder: A::CommandEncoder, + fence: A::Fence, + fence_value: hal::FenceValue, + used_views: Vec, + used_cmd_bufs: Vec, + frames_recorded: usize, +} + +impl ExecutionContext { + unsafe fn wait_and_clear(&mut self, device: &A::Device) { + device.wait(&self.fence, self.fence_value, None).unwrap(); + self.encoder.reset_all(self.used_cmd_bufs.drain(..)); + for view in self.used_views.drain(..) { + device.destroy_texture_view(view); + } + self.frames_recorded = 0; + } +} + +#[allow(dead_code)] +struct Example { + instance: A::Instance, + adapter: A::Adapter, + surface: A::Surface, + surface_format: wgpu_types::TextureFormat, + device: A::Device, + queue: A::Queue, + global_group: A::BindGroup, + local_group: A::BindGroup, + global_group_layout: A::BindGroupLayout, + local_group_layout: A::BindGroupLayout, + pipeline_layout: A::PipelineLayout, + shader: A::ShaderModule, + pipeline: A::RenderPipeline, + bunnies: Vec, + local_buffer: A::Buffer, + local_alignment: u32, + global_buffer: A::Buffer, + sampler: A::Sampler, + texture: A::Texture, + texture_view: A::TextureView, + contexts: Vec>, + context_index: usize, + extent: [u32; 2], + start: Instant, +} + +impl Example { + fn init(window: &winit::window::Window) -> Result> { + // The Instance can be initialized with the DisplayHandle from the EventLoop as well + let raw_display_handle = window.display_handle()?; + + let instance_desc = hal::InstanceDescriptor { + name: "example", + flags: wgpu_types::InstanceFlags::from_build_config().with_env(), + memory_budget_thresholds: wgpu_types::MemoryBudgetThresholds::default(), + // Can't rely on having DXC available, so use FXC instead + backend_options: wgpu_types::BackendOptions::default(), + telemetry: None, + display: Some(raw_display_handle), + }; + let instance = unsafe { A::Instance::init(&instance_desc)? }; + let surface = { + let raw_window_handle = window.window_handle()?.as_raw(); + + unsafe { + instance + .create_surface(raw_display_handle.as_raw(), raw_window_handle) + .unwrap() + } + }; + + let (adapter, capabilities) = unsafe { + let mut adapters = instance.enumerate_adapters(Some(&surface)); + if adapters.is_empty() { + return Err("no adapters found".into()); + } + let exposed = adapters.swap_remove(0); + (exposed.adapter, exposed.capabilities) + }; + + let surface_caps = unsafe { adapter.surface_capabilities(&surface) } + .ok_or("failed to get surface capabilities")?; + log::info!("Surface caps: {surface_caps:#?}"); + + let hal::OpenDevice { device, queue } = unsafe { + adapter + .open( + wgpu_types::Features::empty(), + &wgpu_types::Limits::default(), + &wgpu_types::MemoryHints::default(), + ) + .unwrap() + }; + + let window_size: (u32, u32) = window.inner_size().into(); + let surface_config = hal::SurfaceConfiguration { + maximum_frame_latency: DESIRED_MAX_LATENCY.clamp( + *surface_caps.maximum_frame_latency.start(), + *surface_caps.maximum_frame_latency.end(), + ), + present_mode: wgpu_types::PresentMode::Fifo, + composite_alpha_mode: wgpu_types::CompositeAlphaMode::Opaque, + format: wgpu_types::TextureFormat::Bgra8UnormSrgb, + extent: wgpu_types::Extent3d { + width: window_size.0, + height: window_size.1, + depth_or_array_layers: 1, + }, + usage: wgpu_types::TextureUses::COLOR_TARGET, + view_formats: vec![], + }; + unsafe { + surface.configure(&device, &surface_config).unwrap(); + }; + + let naga_shader = { + let shader_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("examples") + .join("halmark") + .join("shader.wgsl"); + let source = std::fs::read_to_string(shader_file).unwrap(); + let module = naga::front::wgsl::Frontend::new().parse(&source).unwrap(); + let info = naga::valid::Validator::new( + naga::valid::ValidationFlags::all(), + naga::valid::Capabilities::empty(), + ) + .validate(&module) + .unwrap(); + hal::NagaShader { + module: Cow::Owned(module), + info, + debug_source: None, + } + }; + let shader_desc = hal::ShaderModuleDescriptor { + label: None, + runtime_checks: wgpu_types::ShaderRuntimeChecks::checked(), + }; + let shader = unsafe { + device + .create_shader_module(&shader_desc, hal::ShaderInput::Naga(naga_shader)) + .unwrap() + }; + + let global_bgl_desc = hal::BindGroupLayoutDescriptor { + label: None, + flags: hal::BindGroupLayoutFlags::empty(), + entries: &[ + wgpu_types::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu_types::ShaderStages::VERTEX, + ty: wgpu_types::BindingType::Buffer { + ty: wgpu_types::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: wgpu_types::BufferSize::new(size_of::() as _), + }, + count: None, + }, + wgpu_types::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu_types::ShaderStages::FRAGMENT, + ty: wgpu_types::BindingType::Texture { + sample_type: wgpu_types::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu_types::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu_types::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu_types::ShaderStages::FRAGMENT, + ty: wgpu_types::BindingType::Sampler(wgpu_types::SamplerBindingType::Filtering), + count: None, + }, + ], + }; + + let global_group_layout = + unsafe { device.create_bind_group_layout(&global_bgl_desc).unwrap() }; + + let local_bgl_desc = hal::BindGroupLayoutDescriptor { + label: None, + flags: hal::BindGroupLayoutFlags::empty(), + entries: &[wgpu_types::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu_types::ShaderStages::VERTEX, + ty: wgpu_types::BindingType::Buffer { + ty: wgpu_types::BufferBindingType::Uniform, + has_dynamic_offset: true, + min_binding_size: wgpu_types::BufferSize::new(size_of::() as _), + }, + count: None, + }], + }; + let local_group_layout = + unsafe { device.create_bind_group_layout(&local_bgl_desc).unwrap() }; + + let pipeline_layout_desc = hal::PipelineLayoutDescriptor { + label: None, + flags: hal::PipelineLayoutFlags::empty(), + bind_group_layouts: &[Some(&global_group_layout), Some(&local_group_layout)], + immediate_size: 0, + }; + let pipeline_layout = unsafe { + device + .create_pipeline_layout(&pipeline_layout_desc) + .unwrap() + }; + + let constants = naga::back::PipelineConstants::default(); + let pipeline_desc = hal::RenderPipelineDescriptor { + label: None, + layout: &pipeline_layout, + vertex_processor: hal::VertexProcessor::Standard { + vertex_stage: hal::ProgrammableStage { + module: &shader, + entry_point: "vs_main", + constants: &constants, + zero_initialize_workgroup_memory: true, + }, + vertex_buffers: &[], + }, + fragment_stage: Some(hal::ProgrammableStage { + module: &shader, + entry_point: "fs_main", + constants: &constants, + zero_initialize_workgroup_memory: true, + }), + primitive: wgpu_types::PrimitiveState { + topology: wgpu_types::PrimitiveTopology::TriangleStrip, + ..wgpu_types::PrimitiveState::default() + }, + depth_stencil: None, + multisample: wgpu_types::MultisampleState::default(), + color_targets: &[Some(wgpu_types::ColorTargetState { + format: surface_config.format, + blend: Some(wgpu_types::BlendState::ALPHA_BLENDING), + write_mask: wgpu_types::ColorWrites::default(), + })], + multiview_mask: None, + cache: None, + }; + let pipeline = unsafe { device.create_render_pipeline(&pipeline_desc).unwrap() }; + + let texture_data = [0xFFu8; 4]; + + let staging_buffer_desc = hal::BufferDescriptor { + label: Some("stage"), + size: texture_data.len() as wgpu_types::BufferAddress, + usage: wgpu_types::BufferUses::MAP_WRITE | wgpu_types::BufferUses::COPY_SRC, + memory_flags: hal::MemoryFlags::TRANSIENT | hal::MemoryFlags::PREFER_COHERENT, + }; + let staging_buffer = unsafe { device.create_buffer(&staging_buffer_desc).unwrap() }; + unsafe { + let mapping = device + .map_buffer(&staging_buffer, 0..staging_buffer_desc.size) + .unwrap(); + ptr::copy_nonoverlapping( + texture_data.as_ptr(), + mapping.ptr.as_ptr(), + texture_data.len(), + ); + device.unmap_buffer(&staging_buffer); + assert!(mapping.is_coherent); + } + + let texture_desc = hal::TextureDescriptor { + label: None, + size: wgpu_types::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu_types::TextureDimension::D2, + format: wgpu_types::TextureFormat::Rgba8UnormSrgb, + usage: wgpu_types::TextureUses::COPY_DST | wgpu_types::TextureUses::RESOURCE, + memory_flags: hal::MemoryFlags::empty(), + view_formats: vec![], + }; + let texture = unsafe { device.create_texture(&texture_desc).unwrap() }; + + let cmd_encoder_desc = hal::CommandEncoderDescriptor { + label: None, + queue: &queue, + }; + let mut cmd_encoder = unsafe { device.create_command_encoder(&cmd_encoder_desc).unwrap() }; + unsafe { cmd_encoder.begin_encoding(Some("init")).unwrap() }; + { + let buffer_barrier = hal::BufferBarrier { + buffer: &staging_buffer, + usage: hal::StateTransition { + from: wgpu_types::BufferUses::empty(), + to: wgpu_types::BufferUses::COPY_SRC, + }, + }; + let texture_barrier1 = hal::TextureBarrier { + texture: &texture, + range: wgpu_types::ImageSubresourceRange::default(), + usage: hal::StateTransition { + from: wgpu_types::TextureUses::UNINITIALIZED, + to: wgpu_types::TextureUses::COPY_DST, + }, + }; + let texture_barrier2 = hal::TextureBarrier { + texture: &texture, + range: wgpu_types::ImageSubresourceRange::default(), + usage: hal::StateTransition { + from: wgpu_types::TextureUses::COPY_DST, + to: wgpu_types::TextureUses::RESOURCE, + }, + }; + let copy = hal::BufferTextureCopy { + buffer_layout: wgpu_types::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(4), + rows_per_image: None, + }, + texture_base: hal::TextureCopyBase { + origin: wgpu_types::Origin3d::ZERO, + mip_level: 0, + array_layer: 0, + aspect: hal::FormatAspects::COLOR, + }, + size: hal::CopyExtent { + width: 1, + height: 1, + depth: 1, + }, + }; + unsafe { + cmd_encoder.transition_buffers(iter::once(buffer_barrier)); + cmd_encoder.transition_textures(iter::once(texture_barrier1)); + cmd_encoder.copy_buffer_to_texture(&staging_buffer, &texture, iter::once(copy)); + cmd_encoder.transition_textures(iter::once(texture_barrier2)); + } + } + + let sampler_desc = hal::SamplerDescriptor { + label: None, + address_modes: [wgpu_types::AddressMode::ClampToEdge; 3], + mag_filter: wgpu_types::FilterMode::Linear, + min_filter: wgpu_types::FilterMode::Nearest, + mipmap_filter: wgpu_types::MipmapFilterMode::Nearest, + lod_clamp: 0.0..32.0, + compare: None, + anisotropy_clamp: 1, + border_color: None, + }; + let sampler = unsafe { device.create_sampler(&sampler_desc).unwrap() }; + + let globals = Globals { + // cgmath::ortho() projection + mvp: [ + [2.0 / window_size.0 as f32, 0.0, 0.0, 0.0], + [0.0, 2.0 / window_size.1 as f32, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [-1.0, -1.0, 0.0, 1.0], + ], + size: [BUNNY_SIZE; 2], + pad: [0.0; 2], + }; + + let global_buffer_desc = hal::BufferDescriptor { + label: Some("global"), + size: size_of::() as wgpu_types::BufferAddress, + usage: wgpu_types::BufferUses::MAP_WRITE | wgpu_types::BufferUses::UNIFORM, + memory_flags: hal::MemoryFlags::PREFER_COHERENT, + }; + let global_buffer = unsafe { + let buffer = device.create_buffer(&global_buffer_desc).unwrap(); + let mapping = device + .map_buffer(&buffer, 0..global_buffer_desc.size) + .unwrap(); + ptr::copy_nonoverlapping( + &globals as *const Globals as *const u8, + mapping.ptr.as_ptr(), + size_of::(), + ); + device.unmap_buffer(&buffer); + assert!(mapping.is_coherent); + buffer + }; + + let local_alignment = wgpu_types::math::align_to( + size_of::() as u32, + capabilities.limits.min_uniform_buffer_offset_alignment, + ); + let local_buffer_desc = hal::BufferDescriptor { + label: Some("local"), + size: (MAX_BUNNIES as wgpu_types::BufferAddress) + * (local_alignment as wgpu_types::BufferAddress), + usage: wgpu_types::BufferUses::MAP_WRITE | wgpu_types::BufferUses::UNIFORM, + memory_flags: hal::MemoryFlags::PREFER_COHERENT, + }; + let local_buffer = unsafe { device.create_buffer(&local_buffer_desc).unwrap() }; + + let view_desc = hal::TextureViewDescriptor { + label: None, + format: texture_desc.format, + dimension: wgpu_types::TextureViewDimension::D2, + usage: wgpu_types::TextureUses::RESOURCE, + range: wgpu_types::ImageSubresourceRange::default(), + }; + let texture_view = unsafe { device.create_texture_view(&texture, &view_desc).unwrap() }; + + let global_group = { + // SAFETY: This is the same size that was specified for buffer creation. + let global_buffer_binding = hal::BufferBinding::new_unchecked( + &global_buffer, + 0, + NonZeroU64::new(global_buffer_desc.size), + ); + let texture_binding = hal::TextureBinding { + view: &texture_view, + usage: wgpu_types::TextureUses::RESOURCE, + }; + let global_group_desc = hal::BindGroupDescriptor { + label: Some("global"), + layout: &global_group_layout, + buffers: &[global_buffer_binding], + samplers: &[&sampler], + textures: &[texture_binding], + acceleration_structures: &[], + external_textures: &[], + entries: &[ + hal::BindGroupEntry { + binding: 0, + resource_index: 0, + count: 1, + }, + hal::BindGroupEntry { + binding: 1, + resource_index: 0, + count: 1, + }, + hal::BindGroupEntry { + binding: 2, + resource_index: 0, + count: 1, + }, + ], + }; + unsafe { device.create_bind_group(&global_group_desc).unwrap() } + }; + + let local_group = { + // SAFETY: The size must fit within the buffer. + let local_buffer_binding = hal::BufferBinding::new_unchecked( + &local_buffer, + 0, + wgpu_types::BufferSize::new(size_of::() as _), + ); + let local_group_desc = hal::BindGroupDescriptor { + label: Some("local"), + layout: &local_group_layout, + buffers: &[local_buffer_binding], + samplers: &[], + textures: &[], + acceleration_structures: &[], + external_textures: &[], + entries: &[hal::BindGroupEntry { + binding: 0, + resource_index: 0, + count: 1, + }], + }; + unsafe { device.create_bind_group(&local_group_desc).unwrap() } + }; + + let init_fence_value = 1; + let fence = unsafe { + let mut fence = device.create_fence().unwrap(); + let init_cmd = cmd_encoder.end_encoding().unwrap(); + queue + .submit(&[&init_cmd], &[], (&mut fence, init_fence_value)) + .unwrap(); + device.wait(&fence, init_fence_value, None).unwrap(); + device.destroy_buffer(staging_buffer); + cmd_encoder.reset_all(iter::once(init_cmd)); + fence + }; + + Ok(Example { + instance, + surface, + surface_format: surface_config.format, + adapter, + device, + queue, + pipeline_layout, + shader, + pipeline, + global_group, + local_group, + global_group_layout, + local_group_layout, + bunnies: Vec::new(), + local_buffer, + local_alignment, + global_buffer, + sampler, + texture, + texture_view, + contexts: vec![ExecutionContext { + encoder: cmd_encoder, + fence, + fence_value: init_fence_value + 1, + used_views: Vec::new(), + used_cmd_bufs: Vec::new(), + frames_recorded: 0, + }], + context_index: 0, + extent: [window_size.0, window_size.1], + start: Instant::now(), + }) + } + + fn is_empty(&self) -> bool { + self.bunnies.is_empty() + } + + fn exit(mut self) { + unsafe { + { + let ctx = &mut self.contexts[self.context_index]; + self.queue + .submit(&[], &[], (&mut ctx.fence, ctx.fence_value)) + .unwrap(); + } + + for mut ctx in self.contexts { + ctx.wait_and_clear(&self.device); + drop(ctx.encoder); + self.device.destroy_fence(ctx.fence); + } + + self.device.destroy_bind_group(self.local_group); + self.device.destroy_bind_group(self.global_group); + self.device.destroy_buffer(self.local_buffer); + self.device.destroy_buffer(self.global_buffer); + self.device.destroy_texture_view(self.texture_view); + self.device.destroy_texture(self.texture); + self.device.destroy_sampler(self.sampler); + self.device.destroy_shader_module(self.shader); + self.device.destroy_render_pipeline(self.pipeline); + self.device + .destroy_bind_group_layout(self.local_group_layout); + self.device + .destroy_bind_group_layout(self.global_group_layout); + self.device.destroy_pipeline_layout(self.pipeline_layout); + + self.surface.unconfigure(&self.device); + drop(self.queue); + drop(self.device); + drop(self.surface); + drop(self.adapter); + } + } + + fn update(&mut self, event: winit::event::WindowEvent) { + if let winit::event::WindowEvent::KeyboardInput { + event: + KeyEvent { + logical_key: Key::Named(NamedKey::Space), + state: ElementState::Pressed, + .. + }, + .. + } = event + { + let spawn_count = 64 + self.bunnies.len() / 2; + let elapsed = self.start.elapsed(); + let color = elapsed.as_nanos() as u32; + println!( + "Spawning {} bunnies, total at {}", + spawn_count, + self.bunnies.len() + spawn_count + ); + for i in 0..spawn_count { + let random = ((elapsed.as_nanos() * (i + 1) as u128) & 0xFF) as f32 / 255.0; + let speed = random * MAX_VELOCITY - (MAX_VELOCITY * 0.5); + self.bunnies.push(Locals { + position: [0.0, 0.5 * (self.extent[1] as f32)], + velocity: [speed, 0.0], + color, + _pad: 0, + }); + } + } + } + + fn render(&mut self) { + let delta = 0.01; + for bunny in self.bunnies.iter_mut() { + bunny.position[0] += bunny.velocity[0] * delta; + bunny.position[1] += bunny.velocity[1] * delta; + bunny.velocity[1] += GRAVITY * delta; + if (bunny.velocity[0] > 0.0 + && bunny.position[0] + 0.5 * BUNNY_SIZE > self.extent[0] as f32) + || (bunny.velocity[0] < 0.0 && bunny.position[0] - 0.5 * BUNNY_SIZE < 0.0) + { + bunny.velocity[0] *= -1.0; + } + if bunny.velocity[1] < 0.0 && bunny.position[1] < 0.5 * BUNNY_SIZE { + bunny.velocity[1] *= -1.0; + } + } + + if !self.bunnies.is_empty() { + let size = self.bunnies.len() * self.local_alignment as usize; + unsafe { + let mapping = self + .device + .map_buffer(&self.local_buffer, 0..size as wgpu_types::BufferAddress) + .unwrap(); + ptr::copy_nonoverlapping( + self.bunnies.as_ptr() as *const u8, + mapping.ptr.as_ptr(), + size, + ); + assert!(mapping.is_coherent); + self.device.unmap_buffer(&self.local_buffer); + } + } + + let ctx = &mut self.contexts[self.context_index]; + + let surface_tex = unsafe { + self.surface + .acquire_texture(None, &ctx.fence) + .unwrap() + .texture + }; + + let target_barrier0 = hal::TextureBarrier { + texture: surface_tex.borrow(), + range: wgpu_types::ImageSubresourceRange::default(), + usage: hal::StateTransition { + from: wgpu_types::TextureUses::UNINITIALIZED, + to: wgpu_types::TextureUses::COLOR_TARGET, + }, + }; + unsafe { + ctx.encoder.begin_encoding(Some("frame")).unwrap(); + ctx.encoder.transition_textures(iter::once(target_barrier0)); + } + + let surface_view_desc = hal::TextureViewDescriptor { + label: None, + format: self.surface_format, + dimension: wgpu_types::TextureViewDimension::D2, + usage: wgpu_types::TextureUses::COLOR_TARGET, + range: wgpu_types::ImageSubresourceRange::default(), + }; + let surface_tex_view = unsafe { + self.device + .create_texture_view(surface_tex.borrow(), &surface_view_desc) + .unwrap() + }; + let pass_desc = hal::RenderPassDescriptor { + label: None, + extent: wgpu_types::Extent3d { + width: self.extent[0], + height: self.extent[1], + depth_or_array_layers: 1, + }, + sample_count: 1, + color_attachments: &[Some(hal::ColorAttachment { + target: hal::Attachment { + view: &surface_tex_view, + usage: wgpu_types::TextureUses::COLOR_TARGET, + }, + depth_slice: None, + resolve_target: None, + ops: hal::AttachmentOps::STORE | hal::AttachmentOps::LOAD_CLEAR, + clear_value: wgpu_types::Color { + r: 0.1, + g: 0.2, + b: 0.3, + a: 1.0, + }, + })], + depth_stencil_attachment: None, + multiview_mask: None, + timestamp_writes: None, + occlusion_query_set: None, + }; + unsafe { + ctx.encoder.begin_render_pass(&pass_desc).unwrap(); + ctx.encoder.set_render_pipeline(&self.pipeline); + ctx.encoder + .set_bind_group(&self.pipeline_layout, 0, &self.global_group, &[]); + } + + for i in 0..self.bunnies.len() { + let offset = (i as wgpu_types::DynamicOffset) + * (self.local_alignment as wgpu_types::DynamicOffset); + unsafe { + ctx.encoder + .set_bind_group(&self.pipeline_layout, 1, &self.local_group, &[offset]); + ctx.encoder.draw(0, 4, 0, 1); + } + } + + ctx.frames_recorded += 1; + + let target_barrier1 = hal::TextureBarrier { + texture: surface_tex.borrow(), + range: wgpu_types::ImageSubresourceRange::default(), + usage: hal::StateTransition { + from: wgpu_types::TextureUses::COLOR_TARGET, + to: wgpu_types::TextureUses::PRESENT, + }, + }; + unsafe { + ctx.encoder.end_render_pass(); + ctx.encoder.transition_textures(iter::once(target_barrier1)); + } + + unsafe { + let cmd_buf = ctx.encoder.end_encoding().unwrap(); + self.queue + .submit( + &[&cmd_buf], + &[&surface_tex], + (&mut ctx.fence, ctx.fence_value), + ) + .unwrap(); + self.queue.present(&self.surface, surface_tex).unwrap(); + ctx.used_cmd_bufs.push(cmd_buf); + ctx.used_views.push(surface_tex_view); + }; + + log::debug!("Context switch from {}", self.context_index); + let old_fence_value = ctx.fence_value; + if self.contexts.len() == 1 { + let hal_desc = hal::CommandEncoderDescriptor { + label: None, + queue: &self.queue, + }; + self.contexts.push(unsafe { + ExecutionContext { + encoder: self.device.create_command_encoder(&hal_desc).unwrap(), + fence: self.device.create_fence().unwrap(), + fence_value: 0, + used_views: Vec::new(), + used_cmd_bufs: Vec::new(), + frames_recorded: 0, + } + }); + } + self.context_index = (self.context_index + 1) % self.contexts.len(); + let next = &mut self.contexts[self.context_index]; + unsafe { + next.wait_and_clear(&self.device); + } + next.fence_value = old_fence_value + 1; + } +} + +cfg_if::cfg_if! { + // Apple + Metal + if #[cfg(all(target_vendor = "apple", feature = "metal"))] { + type Api = hal::api::Metal; + } + // Wasm + Vulkan + else if #[cfg(all(not(target_arch = "wasm32"), feature = "vulkan"))] { + type Api = hal::api::Vulkan; + } + // Windows + DX12 + else if #[cfg(all(windows, feature = "dx12"))] { + type Api = hal::api::Dx12; + } + // Anything + GLES + else if #[cfg(feature = "gles")] { + type Api = hal::api::Gles; + } + // Fallback + else { + type Api = hal::api::Noop; + } +} + +struct App { + example: Option>, + window: Option, + last_frame_inst: Instant, + frame_count: u32, + accum_time: f32, +} + +impl ApplicationHandler for App { + fn resumed(&mut self, event_loop: &ActiveEventLoop) { + if self.window.is_some() { + return; + } + let window = event_loop + .create_window(Window::default_attributes().with_title("hal-bunnymark")) + .unwrap(); + let example = Example::::init(&window).expect("Selected backend is not supported"); + self.window = Some(window); + self.example = Some(example); + println!("Press space to spawn bunnies."); + self.window.as_ref().unwrap().request_redraw(); + } + + fn exiting(&mut self, _event_loop: &ActiveEventLoop) { + self.example.take().unwrap().exit(); + } + + fn window_event( + &mut self, + event_loop: &ActiveEventLoop, + _window_id: winit::window::WindowId, + event: WindowEvent, + ) { + event_loop.set_control_flow(ControlFlow::Poll); + + match event { + WindowEvent::KeyboardInput { + event: + KeyEvent { + logical_key: Key::Named(NamedKey::Escape), + state: ElementState::Pressed, + .. + }, + .. + } + | WindowEvent::CloseRequested => event_loop.exit(), + WindowEvent::RedrawRequested => { + let ex = self.example.as_mut().unwrap(); + { + self.accum_time += self.last_frame_inst.elapsed().as_secs_f32(); + self.last_frame_inst = Instant::now(); + self.frame_count += 1; + if self.frame_count == 100 && !ex.is_empty() { + println!( + "Avg frame time {}ms", + self.accum_time * 1000.0 / self.frame_count as f32 + ); + self.accum_time = 0.0; + self.frame_count = 0; + } + } + ex.render(); + self.window.as_ref().unwrap().request_redraw(); + } + _ => { + self.example.as_mut().unwrap().update(event); + } + } + } +} + +fn main() { + env_logger::init(); + + let event_loop = winit::event_loop::EventLoop::new().unwrap(); + let mut app = App { + example: None, + window: None, + last_frame_inst: Instant::now(), + frame_count: 0, + accum_time: 0.0, + }; + event_loop.run_app(&mut app).unwrap(); +} diff --git a/third_party/wgpu-hal-29.0.4-patched/examples/halmark/shader.wgsl b/third_party/wgpu-hal-29.0.4-patched/examples/halmark/shader.wgsl new file mode 100644 index 00000000..ffa72645 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/examples/halmark/shader.wgsl @@ -0,0 +1,50 @@ +struct Globals { + mvp: mat4x4, + size: vec2, + _pad0: u32, + _pad1: u32, +}; + +struct Locals { + position: vec2, + velocity: vec2, + color: u32, + _pad0: u32, + _pad1: u32, + _pad2: u32, +}; + +@group(0) +@binding(0) +var globals: Globals; + +@group(1) +@binding(0) +var locals: Locals; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) tex_coords: vec2, + @location(1) color: vec4, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vi: u32) -> VertexOutput { + let tc = vec2(f32(vi & 1u), 0.5 * f32(vi & 2u)); + let offset = vec2(tc.x * globals.size.x, tc.y * globals.size.y); + let pos = globals.mvp * vec4(locals.position + offset, 0.0, 1.0); + let color = vec4((vec4(locals.color) >> vec4(0u, 8u, 16u, 24u)) & vec4(255u)) / 255.0; + return VertexOutput(pos, tc, color); +} + +@group(0) +@binding(1) +var tex: texture_2d; +@group(0) +@binding(2) +var sam: sampler; + +@fragment +fn fs_main(vertex: VertexOutput) -> @location(0) vec4 { + return vertex.color * textureSampleLevel(tex, sam, vertex.tex_coords, 0.0); +} diff --git a/third_party/wgpu-hal-29.0.4-patched/examples/raw-gles.em.html b/third_party/wgpu-hal-29.0.4-patched/examples/raw-gles.em.html new file mode 100644 index 00000000..f3587e85 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/examples/raw-gles.em.html @@ -0,0 +1,16 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/third_party/wgpu-hal-29.0.4-patched/examples/raw-gles.rs b/third_party/wgpu-hal-29.0.4-patched/examples/raw-gles.rs new file mode 100644 index 00000000..b32398c0 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/examples/raw-gles.rs @@ -0,0 +1,396 @@ +//! This example shows interop with raw GLES contexts - +//! the ability to hook up wgpu-hal to an existing context and draw into it. +//! +//! Emscripten build: +//! 1. install emsdk +//! 2. build this example with cargo: +//! EMCC_CFLAGS="-g -s ERROR_ON_UNDEFINED_SYMBOLS=0 --no-entry -s FULL_ES3=1" cargo build --example raw-gles --target wasm32-unknown-emscripten +//! 3. copy raw-gles.em.html into target directory and open it in browser: +//! cp wgpu-hal/examples/raw-gles.em.html target/wasm32-unknown-emscripten/debug/examples + +extern crate wgpu_hal as hal; + +#[cfg(not(any( + target_arch = "wasm32", + target_os = "ios", + target_os = "visionos", + target_env = "ohos" +)))] +fn main() { + use std::ffi::CString; + + use glutin::{ + config::GlConfig as _, + context::{NotCurrentGlContext as _, PossiblyCurrentGlContext as _, Version}, + display::{GetGlDisplay as _, GlDisplay as _}, + surface::GlSurface as _, + }; + use glutin_winit::GlWindow as _; + use raw_window_handle::HasWindowHandle as _; + use winit::{ + application::ApplicationHandler, + event::{KeyEvent, WindowEvent}, + event_loop::{ActiveEventLoop, ControlFlow, EventLoop}, + keyboard::{Key, NamedKey}, + window::Window, + }; + + env_logger::init(); + println!("Initializing external GL context"); + + /// Find the config with the maximum number of samples, so our triangle will be + /// smooth. + fn gl_config_picker( + configs: Box + '_>, + ) -> glutin::config::Config { + configs + .reduce(|accum, config| { + if config.num_samples() > accum.num_samples() { + config + } else { + accum + } + }) + .expect("Failed to find a matching config") + } + + struct App { + gl_config: Option, + not_current_gl_context: Option, + state: Option<( + glutin::context::PossiblyCurrentContext, + glutin::surface::Surface, + Window, + )>, + exposed: Option>, + } + + impl App { + fn new() -> Self { + Self { + gl_config: None, + not_current_gl_context: None, + state: None, + exposed: None, + } + } + } + + impl ApplicationHandler for App { + fn resumed(&mut self, event_loop: &ActiveEventLoop) { + // First call: create display, pick config, create GL context. + if self.gl_config.is_none() { + // Only Windows requires the window to be present before creating the display. + // Other platforms don't really need one. + let window_attributes = cfg!(windows).then(|| { + Window::default_attributes() + .with_title("wgpu raw GLES example (press Escape to exit)") + }); + + let template = glutin::config::ConfigTemplateBuilder::new(); + + let display_builder = + glutin_winit::DisplayBuilder::new().with_window_attributes(window_attributes); + + let (window, gl_config) = display_builder + .build(event_loop, template, gl_config_picker) + .expect("Failed to build window and config from display"); + + println!("Picked a config with {} samples", gl_config.num_samples()); + + let raw_window_handle = window + .as_ref() + .and_then(|window| window.window_handle().ok()) + .map(|handle| handle.as_raw()); + + let gl_display = gl_config.display(); + + // Glutin tries to create an OpenGL context by default. + // Force it to use any version of GLES. + let context_attributes = glutin::context::ContextAttributesBuilder::new() + // wgpu expects GLES 3.0+. + .with_context_api(glutin::context::ContextApi::Gles(Some(Version::new(3, 0)))) + .build(raw_window_handle); + + let gl_context = unsafe { + gl_display + .create_context(&gl_config, &context_attributes) + .expect("failed to create context") + }; + + self.not_current_gl_context = Some(gl_context); + self.gl_config = Some(gl_config); + + // On Windows, the window was already created by DisplayBuilder. + if let Some(window) = window { + self.create_surface(event_loop, window); + return; + } + } + + // Create window + surface (non-Windows first call, or Android re-resume). + if self.state.is_none() { + let gl_config = self.gl_config.as_ref().unwrap(); + let window = glutin_winit::finalize_window( + event_loop, + Window::default_attributes() + .with_title("wgpu raw GLES example (press Escape to exit)"), + gl_config, + ) + .unwrap(); + self.create_surface(event_loop, window); + } + } + + fn suspended(&mut self, _event_loop: &ActiveEventLoop) { + // This event is only raised on Android, where the backing NativeWindow for a GL + // Surface can appear and disappear at any moment. + println!("Android window removed"); + + // Destroy the GL Surface and un-current the GL Context before ndk-glue releases + // the window back to the system. + if let Some((gl_context, ..)) = self.state.take() { + assert!(self + .not_current_gl_context + .replace(gl_context.make_not_current().unwrap()) + .is_none()); + } + } + + fn window_event( + &mut self, + event_loop: &ActiveEventLoop, + _window_id: winit::window::WindowId, + event: WindowEvent, + ) { + event_loop.set_control_flow(ControlFlow::Wait); + + match event { + WindowEvent::CloseRequested + | WindowEvent::KeyboardInput { + event: + KeyEvent { + logical_key: Key::Named(NamedKey::Escape), + .. + }, + .. + } => event_loop.exit(), + WindowEvent::Resized(size) => { + if size.width != 0 && size.height != 0 { + // Some platforms like EGL require resizing GL surface to update the size. + // Notable platforms here are Wayland and macOS, other don't require it + // and the function is no-op, but it's wise to resize it for portability + // reasons. + if let Some((gl_context, gl_surface, window)) = &self.state { + window.resize_surface(gl_surface, gl_context); + } + } + } + WindowEvent::RedrawRequested => { + if let (Some(exposed), Some((gl_context, gl_surface, window))) = + (&self.exposed, &self.state) + { + let inner_size = window.inner_size(); + + fill_screen(exposed, inner_size.width, inner_size.height); + + println!("Showing the window"); + gl_surface + .swap_buffers(gl_context) + .expect("Failed to swap buffers"); + } + } + _ => (), + } + } + } + + impl App { + fn create_surface(&mut self, _event_loop: &ActiveEventLoop, window: Window) { + let gl_config = self.gl_config.as_ref().unwrap(); + + let attrs = window + .build_surface_attributes(Default::default()) + .expect("Failed to build surface attributes"); + let gl_surface = unsafe { + gl_config + .display() + .create_window_surface(gl_config, &attrs) + .expect("Cannot create GL WindowSurface") + }; + + // Make it current. + let gl_context = self + .not_current_gl_context + .take() + .unwrap() + .make_current(&gl_surface) + .expect("GL context cannot be made current with WindowSurface"); + + // The context needs to be current for the Renderer to set up shaders and + // buffers. It also performs function loading, which needs a current context on + // WGL. + println!("Hooking up to wgpu-hal"); + self.exposed.get_or_insert_with(|| { + unsafe { + ::Adapter::new_external( + |name| { + // XXX: On WGL this should only be called after the context was + // made current + gl_config + .display() + .get_proc_address(&CString::new(name).expect(name)) + }, + wgpu_types::GlBackendOptions::default(), + ) + } + .expect("GL adapter can't be initialized") + }); + + window.request_redraw(); + self.state = Some((gl_context, gl_surface, window)); + } + } + + let event_loop = EventLoop::new().unwrap(); + let mut app = App::new(); + event_loop + .run_app(&mut app) + .expect("Couldn't run event loop"); +} + +#[cfg(target_os = "emscripten")] +fn main() { + env_logger::init(); + + println!("Initializing external GL context"); + let egl = khronos_egl::Instance::new(khronos_egl::Static); + let display = unsafe { egl.get_display(khronos_egl::DEFAULT_DISPLAY) }.unwrap(); + egl.initialize(display) + .expect("unable to initialize display"); + + let attributes = [ + khronos_egl::RED_SIZE, + 8, + khronos_egl::GREEN_SIZE, + 8, + khronos_egl::BLUE_SIZE, + 8, + khronos_egl::NONE, + ]; + + let config = egl + .choose_first_config(display, &attributes) + .unwrap() + .expect("unable to choose config"); + let surface = unsafe { + let window = std::ptr::null_mut::(); + egl.create_window_surface(display, config, window, None) + } + .expect("unable to create surface"); + + let context_attributes = [khronos_egl::CONTEXT_CLIENT_VERSION, 3, khronos_egl::NONE]; + + let gl_context = egl + .create_context(display, config, None, &context_attributes) + .expect("unable to create context"); + egl.make_current(display, Some(surface), Some(surface), Some(gl_context)) + .expect("can't make context current"); + + println!("Hooking up to wgpu-hal"); + let exposed = unsafe { + ::Adapter::new_external(|name| { + egl.get_proc_address(name) + .map_or(std::ptr::null(), |p| p as *const _) + }) + } + .expect("GL adapter can't be initialized"); + + fill_screen(&exposed, 640, 400); +} + +#[cfg(any( + all(target_arch = "wasm32", not(target_os = "emscripten")), + target_os = "ios", + target_os = "visionos", + target_env = "ohos" +))] +fn main() { + eprintln!("This example is not supported on this platform") +} + +#[cfg(not(any( + all(target_arch = "wasm32", not(target_os = "emscripten")), + target_os = "ios", + target_os = "visionos" +)))] +fn fill_screen(exposed: &hal::ExposedAdapter, width: u32, height: u32) { + use hal::{Adapter as _, CommandEncoder as _, Device as _, Queue as _}; + + let od = unsafe { + exposed.adapter.open( + wgpu_types::Features::empty(), + &wgpu_types::Limits::downlevel_defaults(), + &wgpu_types::MemoryHints::default(), + ) + } + .unwrap(); + + let format = wgpu_types::TextureFormat::Rgba8UnormSrgb; + let texture = ::Texture::default_framebuffer(format); + let view = unsafe { + od.device + .create_texture_view( + &texture, + &hal::TextureViewDescriptor { + label: None, + format, + dimension: wgpu_types::TextureViewDimension::D2, + usage: wgpu_types::TextureUses::COLOR_TARGET, + range: wgpu_types::ImageSubresourceRange::default(), + }, + ) + .unwrap() + }; + + println!("Filling the screen"); + let mut encoder = unsafe { + od.device + .create_command_encoder(&hal::CommandEncoderDescriptor { + label: None, + queue: &od.queue, + }) + .unwrap() + }; + let mut fence = unsafe { od.device.create_fence().unwrap() }; + let rp_desc = hal::RenderPassDescriptor { + label: None, + extent: wgpu_types::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + sample_count: 1, + color_attachments: &[Some(hal::ColorAttachment { + target: hal::Attachment { + view: &view, + usage: wgpu_types::TextureUses::COLOR_TARGET, + }, + depth_slice: None, + resolve_target: None, + ops: hal::AttachmentOps::STORE | hal::AttachmentOps::LOAD_CLEAR, + clear_value: wgpu_types::Color::BLUE, + })], + depth_stencil_attachment: None, + multiview_mask: None, + timestamp_writes: None, + occlusion_query_set: None, + }; + unsafe { + encoder.begin_encoding(None).unwrap(); + encoder.begin_render_pass(&rp_desc).unwrap(); + encoder.end_render_pass(); + let cmd_buf = encoder.end_encoding().unwrap(); + od.queue.submit(&[&cmd_buf], &[], (&mut fence, 0)).unwrap(); + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/examples/ray-traced-triangle/main.rs b/third_party/wgpu-hal-29.0.4-patched/examples/ray-traced-triangle/main.rs new file mode 100644 index 00000000..b486520e --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/examples/ray-traced-triangle/main.rs @@ -0,0 +1,1223 @@ +extern crate wgpu_hal as hal; + +use hal::{ + Adapter as _, CommandEncoder as _, Device as _, Instance as _, Queue as _, Surface as _, +}; +use raw_window_handle::{HasDisplayHandle, HasWindowHandle}; + +use glam::{Affine3A, Mat4, Vec3}; +use std::{ + borrow::{Borrow, Cow}, + iter, ptr, + time::Instant, +}; +use wgpu_types::Dx12BackendOptions; +use winit::{ + application::ApplicationHandler, + event_loop::{ActiveEventLoop, ControlFlow}, + window::{Window, WindowButtons}, +}; + +const DESIRED_MAX_LATENCY: u32 = 2; + +/// [D3D12_RAYTRACING_INSTANCE_DESC](https://microsoft.github.io/DirectX-Specs/d3d/Raytracing.html#d3d12_raytracing_instance_desc) +/// [VkAccelerationStructureInstanceKHR](https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkAccelerationStructureInstanceKHR.html) +#[derive(Clone)] +#[repr(C)] +struct AccelerationStructureInstance { + transform: [f32; 12], + custom_index_and_mask: u32, + shader_binding_table_record_offset_and_flags: u32, + acceleration_structure_reference: u64, +} + +impl std::fmt::Debug for AccelerationStructureInstance { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Instance") + .field("transform", &self.transform) + .field("custom_data()", &self.custom_index()) + .field("mask()", &self.mask()) + .field( + "shader_binding_table_record_offset()", + &self.shader_binding_table_record_offset(), + ) + .field("flags()", &self.flags()) + .field( + "acceleration_structure_reference", + &self.acceleration_structure_reference, + ) + .finish() + } +} + +#[allow(dead_code)] +impl AccelerationStructureInstance { + const LOW_24_MASK: u32 = 0x00ff_ffff; + const MAX_U24: u32 = (1u32 << 24u32) - 1u32; + + #[inline] + fn affine_to_rows(mat: &Affine3A) -> [f32; 12] { + let row_0 = mat.matrix3.row(0); + let row_1 = mat.matrix3.row(1); + let row_2 = mat.matrix3.row(2); + let translation = mat.translation; + [ + row_0.x, + row_0.y, + row_0.z, + translation.x, + row_1.x, + row_1.y, + row_1.z, + translation.y, + row_2.x, + row_2.y, + row_2.z, + translation.z, + ] + } + + #[inline] + fn rows_to_affine(rows: &[f32; 12]) -> Affine3A { + Affine3A::from_cols_array(&[ + rows[0], rows[3], rows[6], rows[9], rows[1], rows[4], rows[7], rows[10], rows[2], + rows[5], rows[8], rows[11], + ]) + } + + pub fn transform_as_affine(&self) -> Affine3A { + Self::rows_to_affine(&self.transform) + } + pub fn set_transform(&mut self, transform: &Affine3A) { + self.transform = Self::affine_to_rows(transform); + } + + pub fn custom_index(&self) -> u32 { + self.custom_index_and_mask & Self::LOW_24_MASK + } + + pub fn mask(&self) -> u8 { + (self.custom_index_and_mask >> 24) as u8 + } + + pub fn shader_binding_table_record_offset(&self) -> u32 { + self.shader_binding_table_record_offset_and_flags & Self::LOW_24_MASK + } + + pub fn flags(&self) -> u8 { + (self.shader_binding_table_record_offset_and_flags >> 24) as u8 + } + + pub fn set_custom_index(&mut self, custom_index: u32) { + debug_assert!( + custom_index <= Self::MAX_U24, + "custom_index uses more than 24 bits! {custom_index} > {}", + Self::MAX_U24 + ); + self.custom_index_and_mask = + (custom_index & Self::LOW_24_MASK) | (self.custom_index_and_mask & !Self::LOW_24_MASK) + } + + pub fn set_mask(&mut self, mask: u8) { + self.custom_index_and_mask = + (self.custom_index_and_mask & Self::LOW_24_MASK) | (u32::from(mask) << 24) + } + + pub fn set_shader_binding_table_record_offset( + &mut self, + shader_binding_table_record_offset: u32, + ) { + debug_assert!( + shader_binding_table_record_offset <= Self::MAX_U24, + "shader_binding_table_record_offset uses more than 24 bits! {} > {}", + shader_binding_table_record_offset, + Self::MAX_U24 + ); + self.shader_binding_table_record_offset_and_flags = (shader_binding_table_record_offset + & Self::LOW_24_MASK) + | (self.shader_binding_table_record_offset_and_flags & !Self::LOW_24_MASK) + } + + pub fn set_flags(&mut self, flags: u8) { + self.shader_binding_table_record_offset_and_flags = + (self.shader_binding_table_record_offset_and_flags & Self::LOW_24_MASK) + | (u32::from(flags) << 24) + } + + pub fn new( + transform: &Affine3A, + custom_index: u32, + mask: u8, + shader_binding_table_record_offset: u32, + flags: u8, + acceleration_structure_reference: u64, + ) -> Self { + debug_assert!( + custom_index <= Self::MAX_U24, + "custom_index uses more than 24 bits! {custom_index} > {}", + Self::MAX_U24 + ); + debug_assert!( + shader_binding_table_record_offset <= Self::MAX_U24, + "shader_binding_table_record_offset uses more than 24 bits! {} > {}", + shader_binding_table_record_offset, + Self::MAX_U24 + ); + AccelerationStructureInstance { + transform: Self::affine_to_rows(transform), + custom_index_and_mask: (custom_index & Self::MAX_U24) | (u32::from(mask) << 24), + shader_binding_table_record_offset_and_flags: (shader_binding_table_record_offset + & Self::MAX_U24) + | (u32::from(flags) << 24), + acceleration_structure_reference, + } + } +} + +struct ExecutionContext { + encoder: A::CommandEncoder, + fence: A::Fence, + fence_value: hal::FenceValue, + used_views: Vec, + used_cmd_bufs: Vec, + frames_recorded: usize, +} + +impl ExecutionContext { + unsafe fn wait_and_clear(&mut self, device: &A::Device) { + device.wait(&self.fence, self.fence_value, None).unwrap(); + self.encoder.reset_all(self.used_cmd_bufs.drain(..)); + for view in self.used_views.drain(..) { + device.destroy_texture_view(view); + } + self.frames_recorded = 0; + } +} + +#[allow(dead_code)] +struct Example { + instance: A::Instance, + adapter: A::Adapter, + surface: A::Surface, + surface_format: wgpu_types::TextureFormat, + device: A::Device, + queue: A::Queue, + + contexts: Vec>, + context_index: usize, + extent: [u32; 2], + start: Instant, + pipeline: A::ComputePipeline, + bind_group: A::BindGroup, + bgl: A::BindGroupLayout, + shader_module: A::ShaderModule, + texture_view: A::TextureView, + uniform_buffer: A::Buffer, + pipeline_layout: A::PipelineLayout, + vertices_buffer: A::Buffer, + indices_buffer: Option, + texture: A::Texture, + instances: [AccelerationStructureInstance; 3], + instances_buffer: A::Buffer, + blas: A::AccelerationStructure, + tlas: A::AccelerationStructure, + scratch_buffer: A::Buffer, + time: f32, +} + +impl Example { + fn init(window: &winit::window::Window) -> Result> { + let mut index_buffer = false; + + for arg in std::env::args() { + if arg == "index_buffer" { + index_buffer = true; + } + } + + if index_buffer { + log::info!("using index buffer") + } + + // The Instance can be initialized with the DisplayHandle from the EventLoop as well + let raw_display_handle = window.display_handle()?; + + let instance_desc = hal::InstanceDescriptor { + name: "example", + flags: wgpu_types::InstanceFlags::default(), + memory_budget_thresholds: wgpu_types::MemoryBudgetThresholds::default(), + backend_options: wgpu_types::BackendOptions { + dx12: Dx12BackendOptions { + shader_compiler: wgpu_types::Dx12Compiler::default_dynamic_dxc(), + ..Default::default() + }, + ..Default::default() + }, + telemetry: None, + display: Some(raw_display_handle), + }; + let instance = unsafe { A::Instance::init(&instance_desc)? }; + let surface = { + let raw_window_handle = window.window_handle()?.as_raw(); + + unsafe { + instance + .create_surface(raw_display_handle.as_raw(), raw_window_handle) + .unwrap() + } + }; + + let (adapter, features) = unsafe { + let mut adapters = instance.enumerate_adapters(Some(&surface)); + if adapters.is_empty() { + panic!("No adapters found"); + } + let exposed = adapters.swap_remove(0); + dbg!(exposed.features); + (exposed.adapter, exposed.features) + }; + let surface_caps = unsafe { adapter.surface_capabilities(&surface) } + .expect("Surface doesn't support presentation"); + log::info!("Surface caps: {surface_caps:#?}"); + + let hal::OpenDevice { device, queue } = unsafe { + adapter + .open( + features, + &wgpu_types::Limits::default(), + &wgpu_types::MemoryHints::Performance, + ) + .unwrap() + }; + + let window_size: (u32, u32) = window.inner_size().into(); + dbg!(&surface_caps.formats); + let surface_format = if surface_caps + .formats + .contains(&wgpu_types::TextureFormat::Rgba8Unorm) + { + wgpu_types::TextureFormat::Rgba8Unorm + } else { + *surface_caps.formats.first().unwrap() + }; + let surface_config = hal::SurfaceConfiguration { + maximum_frame_latency: DESIRED_MAX_LATENCY + .max(*surface_caps.maximum_frame_latency.start()) + .min(*surface_caps.maximum_frame_latency.end()), + present_mode: wgpu_types::PresentMode::Fifo, + composite_alpha_mode: wgpu_types::CompositeAlphaMode::Opaque, + format: surface_format, + extent: wgpu_types::Extent3d { + width: window_size.0, + height: window_size.1, + depth_or_array_layers: 1, + }, + usage: wgpu_types::TextureUses::COLOR_TARGET | wgpu_types::TextureUses::COPY_DST, + view_formats: vec![surface_format], + }; + unsafe { + surface.configure(&device, &surface_config).unwrap(); + }; + + #[allow(dead_code)] + struct Uniforms { + view_inverse: glam::Mat4, + proj_inverse: glam::Mat4, + } + + let bgl_desc = hal::BindGroupLayoutDescriptor { + label: None, + flags: hal::BindGroupLayoutFlags::empty(), + entries: &[ + wgpu_types::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu_types::ShaderStages::COMPUTE, + ty: wgpu_types::BindingType::Buffer { + ty: wgpu_types::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: wgpu_types::BufferSize::new(size_of::() as _), + }, + count: None, + }, + wgpu_types::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu_types::ShaderStages::COMPUTE, + ty: wgpu_types::BindingType::StorageTexture { + access: wgpu_types::StorageTextureAccess::WriteOnly, + format: wgpu_types::TextureFormat::Rgba8Unorm, + view_dimension: wgpu_types::TextureViewDimension::D2, + }, + count: None, + }, + wgpu_types::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu_types::ShaderStages::COMPUTE, + ty: wgpu_types::BindingType::AccelerationStructure { + vertex_return: false, + }, + count: None, + }, + ], + }; + + let bgl = unsafe { device.create_bind_group_layout(&bgl_desc).unwrap() }; + + let naga_shader = { + let shader_file = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("examples") + .join("ray-traced-triangle") + .join("shader.wgsl"); + let source = std::fs::read_to_string(shader_file).unwrap(); + let module = naga::front::wgsl::Frontend::new().parse(&source).unwrap(); + let info = naga::valid::Validator::new( + naga::valid::ValidationFlags::all(), + naga::valid::Capabilities::RAY_QUERY, + ) + .validate(&module) + .unwrap(); + hal::NagaShader { + module: Cow::Owned(module), + info, + debug_source: None, + } + }; + let shader_desc = hal::ShaderModuleDescriptor { + label: None, + runtime_checks: wgpu_types::ShaderRuntimeChecks::checked(), + }; + let shader_module = unsafe { + device + .create_shader_module(&shader_desc, hal::ShaderInput::Naga(naga_shader)) + .unwrap() + }; + + let pipeline_layout_desc = hal::PipelineLayoutDescriptor { + label: None, + flags: hal::PipelineLayoutFlags::empty(), + bind_group_layouts: &[Some(&bgl)], + immediate_size: 0, + }; + let pipeline_layout = unsafe { + device + .create_pipeline_layout(&pipeline_layout_desc) + .unwrap() + }; + + let pipeline = unsafe { + device.create_compute_pipeline(&hal::ComputePipelineDescriptor { + label: Some("pipeline"), + layout: &pipeline_layout, + stage: hal::ProgrammableStage { + module: &shader_module, + entry_point: "main", + constants: &Default::default(), + zero_initialize_workgroup_memory: true, + }, + cache: None, + }) + } + .unwrap(); + + let vertices: [f32; 9] = [1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 0.0, -1.0, 0.0]; + + let vertices_size_in_bytes = vertices.len() * 4; + + let indices: [u32; 3] = [0, 1, 2]; + + let indices_size_in_bytes = indices.len() * 4; + + let vertices_buffer = unsafe { + let vertices_buffer = device + .create_buffer(&hal::BufferDescriptor { + label: Some("vertices buffer"), + size: vertices_size_in_bytes as u64, + usage: wgpu_types::BufferUses::MAP_WRITE + | wgpu_types::BufferUses::BOTTOM_LEVEL_ACCELERATION_STRUCTURE_INPUT, + memory_flags: hal::MemoryFlags::TRANSIENT | hal::MemoryFlags::PREFER_COHERENT, + }) + .unwrap(); + + let mapping = device + .map_buffer(&vertices_buffer, 0..vertices_size_in_bytes as u64) + .unwrap(); + ptr::copy_nonoverlapping( + vertices.as_ptr() as *const u8, + mapping.ptr.as_ptr(), + vertices_size_in_bytes, + ); + device.unmap_buffer(&vertices_buffer); + assert!(mapping.is_coherent); + + vertices_buffer + }; + + let indices_buffer = if index_buffer { + unsafe { + let indices_buffer = device + .create_buffer(&hal::BufferDescriptor { + label: Some("indices buffer"), + size: indices_size_in_bytes as u64, + usage: wgpu_types::BufferUses::MAP_WRITE + | wgpu_types::BufferUses::BOTTOM_LEVEL_ACCELERATION_STRUCTURE_INPUT, + memory_flags: hal::MemoryFlags::TRANSIENT + | hal::MemoryFlags::PREFER_COHERENT, + }) + .unwrap(); + + let mapping = device + .map_buffer(&indices_buffer, 0..indices_size_in_bytes as u64) + .unwrap(); + ptr::copy_nonoverlapping( + indices.as_ptr() as *const u8, + mapping.ptr.as_ptr(), + indices_size_in_bytes, + ); + device.unmap_buffer(&indices_buffer); + assert!(mapping.is_coherent); + + Some((indices_buffer, indices.len())) + } + } else { + None + }; + + let blas_triangles = vec![hal::AccelerationStructureTriangles { + vertex_buffer: Some(&vertices_buffer), + first_vertex: 0, + vertex_format: wgpu_types::VertexFormat::Float32x3, + // each vertex is 3 floats, and floats are stored raw in the array + vertex_count: vertices.len() as u32 / 3, + vertex_stride: 3 * 4, + indices: indices_buffer.as_ref().map(|(buf, len)| { + hal::AccelerationStructureTriangleIndices { + buffer: Some(buf), + format: wgpu_types::IndexFormat::Uint32, + offset: 0, + count: *len as u32, + } + }), + + transform: None, + flags: hal::AccelerationStructureGeometryFlags::OPAQUE, + }]; + let blas_entries = hal::AccelerationStructureEntries::Triangles(blas_triangles); + + let blas_sizes = unsafe { + device.get_acceleration_structure_build_sizes( + &hal::GetAccelerationStructureBuildSizesDescriptor { + entries: &blas_entries, + flags: hal::AccelerationStructureBuildFlags::PREFER_FAST_TRACE, + }, + ) + }; + + let blas = unsafe { + device.create_acceleration_structure(&hal::AccelerationStructureDescriptor { + label: Some("blas"), + size: blas_sizes.acceleration_structure_size, + format: hal::AccelerationStructureFormat::BottomLevel, + allow_compaction: false, + }) + } + .unwrap(); + + let instances = [ + AccelerationStructureInstance::new( + &Affine3A::from_translation(Vec3 { + x: 0.0, + y: 0.0, + z: 0.0, + }), + 0, + 0xff, + 0, + 0, + unsafe { device.get_acceleration_structure_device_address(&blas) }, + ), + AccelerationStructureInstance::new( + &Affine3A::from_translation(Vec3 { + x: -1.0, + y: -1.0, + z: -2.0, + }), + 0, + 0xff, + 0, + 0, + unsafe { device.get_acceleration_structure_device_address(&blas) }, + ), + AccelerationStructureInstance::new( + &Affine3A::from_translation(Vec3 { + x: 1.0, + y: -1.0, + z: -2.0, + }), + 0, + 0xff, + 0, + 0, + unsafe { device.get_acceleration_structure_device_address(&blas) }, + ), + ]; + + let instances_buffer_size = instances.len() * size_of::(); + + let instances_buffer = unsafe { + let instances_buffer = device + .create_buffer(&hal::BufferDescriptor { + label: Some("instances_buffer"), + size: instances_buffer_size as u64, + usage: wgpu_types::BufferUses::MAP_WRITE + | wgpu_types::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT, + memory_flags: hal::MemoryFlags::TRANSIENT | hal::MemoryFlags::PREFER_COHERENT, + }) + .unwrap(); + + let mapping = device + .map_buffer(&instances_buffer, 0..instances_buffer_size as u64) + .unwrap(); + ptr::copy_nonoverlapping( + instances.as_ptr() as *const u8, + mapping.ptr.as_ptr(), + instances_buffer_size, + ); + device.unmap_buffer(&instances_buffer); + assert!(mapping.is_coherent); + + instances_buffer + }; + + let mut tlas_entries = + hal::AccelerationStructureEntries::Instances(hal::AccelerationStructureInstances { + buffer: Some(&instances_buffer), + count: 3, + offset: 0, + }); + + let tlas_flags = hal::AccelerationStructureBuildFlags::PREFER_FAST_TRACE + | hal::AccelerationStructureBuildFlags::ALLOW_UPDATE; + + let tlas_sizes = unsafe { + device.get_acceleration_structure_build_sizes( + &hal::GetAccelerationStructureBuildSizesDescriptor { + entries: &tlas_entries, + flags: tlas_flags, + }, + ) + }; + + let tlas = unsafe { + device.create_acceleration_structure(&hal::AccelerationStructureDescriptor { + label: Some("tlas"), + size: tlas_sizes.acceleration_structure_size, + format: hal::AccelerationStructureFormat::TopLevel, + allow_compaction: false, + }) + } + .unwrap(); + + let uniforms = { + let view = Mat4::look_at_rh(Vec3::new(0.0, 0.0, 2.5), Vec3::ZERO, Vec3::Y); + let proj = Mat4::perspective_rh(59.0_f32.to_radians(), 1.0, 0.001, 1000.0); + + Uniforms { + view_inverse: view.inverse(), + proj_inverse: proj.inverse(), + } + }; + + let uniforms_size = size_of::(); + + let uniform_buffer = unsafe { + let uniform_buffer = device + .create_buffer(&hal::BufferDescriptor { + label: Some("uniform buffer"), + size: uniforms_size as u64, + usage: wgpu_types::BufferUses::MAP_WRITE | wgpu_types::BufferUses::UNIFORM, + memory_flags: hal::MemoryFlags::PREFER_COHERENT, + }) + .unwrap(); + + let mapping = device + .map_buffer(&uniform_buffer, 0..uniforms_size as u64) + .unwrap(); + ptr::copy_nonoverlapping( + &uniforms as *const Uniforms as *const u8, + mapping.ptr.as_ptr(), + uniforms_size, + ); + device.unmap_buffer(&uniform_buffer); + assert!(mapping.is_coherent); + uniform_buffer + }; + + let texture_desc = hal::TextureDescriptor { + label: None, + size: wgpu_types::Extent3d { + width: 512, + height: 512, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu_types::TextureDimension::D2, + format: wgpu_types::TextureFormat::Rgba8Unorm, + usage: wgpu_types::TextureUses::STORAGE_READ_WRITE | wgpu_types::TextureUses::COPY_SRC, + memory_flags: hal::MemoryFlags::empty(), + view_formats: vec![wgpu_types::TextureFormat::Rgba8Unorm], + }; + let texture = unsafe { device.create_texture(&texture_desc).unwrap() }; + + let view_desc = hal::TextureViewDescriptor { + label: None, + format: texture_desc.format, + dimension: wgpu_types::TextureViewDimension::D2, + usage: wgpu_types::TextureUses::STORAGE_READ_WRITE | wgpu_types::TextureUses::COPY_SRC, + range: wgpu_types::ImageSubresourceRange::default(), + }; + let texture_view = unsafe { device.create_texture_view(&texture, &view_desc).unwrap() }; + + let bind_group = { + let buffer_binding = unsafe { + // SAFETY: The size matches the buffer allocation. + hal::BufferBinding::new_unchecked( + &uniform_buffer, + 0, + wgpu_types::BufferSize::new_unchecked(uniforms_size as u64), + ) + }; + let texture_binding = hal::TextureBinding { + view: &texture_view, + usage: wgpu_types::TextureUses::STORAGE_READ_WRITE, + }; + let group_desc = hal::BindGroupDescriptor { + label: Some("bind group"), + layout: &bgl, + buffers: &[buffer_binding], + samplers: &[], + textures: &[texture_binding], + acceleration_structures: &[&tlas], + external_textures: &[], + entries: &[ + hal::BindGroupEntry { + binding: 0, + resource_index: 0, + count: 1, + }, + hal::BindGroupEntry { + binding: 1, + resource_index: 0, + count: 1, + }, + hal::BindGroupEntry { + binding: 2, + resource_index: 0, + count: 1, + }, + ], + }; + unsafe { device.create_bind_group(&group_desc).unwrap() } + }; + + let scratch_buffer = unsafe { + device + .create_buffer(&hal::BufferDescriptor { + label: Some("scratch buffer"), + size: blas_sizes + .build_scratch_size + .max(tlas_sizes.build_scratch_size), + usage: wgpu_types::BufferUses::ACCELERATION_STRUCTURE_SCRATCH, + memory_flags: hal::MemoryFlags::empty(), + }) + .unwrap() + }; + + if let hal::AccelerationStructureEntries::Instances(ref mut i) = tlas_entries { + assert!( + instances.len() <= i.count as usize, + "Tlas allocation to small" + ); + } + + let cmd_encoder_desc = hal::CommandEncoderDescriptor { + label: None, + queue: &queue, + }; + let mut cmd_encoder = unsafe { device.create_command_encoder(&cmd_encoder_desc).unwrap() }; + + unsafe { cmd_encoder.begin_encoding(Some("init")).unwrap() }; + + unsafe { + cmd_encoder.place_acceleration_structure_barrier(hal::AccelerationStructureBarrier { + usage: hal::StateTransition { + from: hal::AccelerationStructureUses::empty(), + to: hal::AccelerationStructureUses::BUILD_OUTPUT, + }, + }); + + cmd_encoder.build_acceleration_structures( + 1, + [hal::BuildAccelerationStructureDescriptor { + mode: hal::AccelerationStructureBuildMode::Build, + flags: hal::AccelerationStructureBuildFlags::PREFER_FAST_TRACE, + destination_acceleration_structure: &blas, + scratch_buffer: &scratch_buffer, + entries: &blas_entries, + source_acceleration_structure: None, + scratch_buffer_offset: 0, + }], + ); + + let scratch_buffer_barrier = hal::BufferBarrier { + buffer: &scratch_buffer, + usage: hal::StateTransition { + from: wgpu_types::BufferUses::BOTTOM_LEVEL_ACCELERATION_STRUCTURE_INPUT, + to: wgpu_types::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT, + }, + }; + cmd_encoder.transition_buffers(iter::once(scratch_buffer_barrier)); + + cmd_encoder.place_acceleration_structure_barrier(hal::AccelerationStructureBarrier { + usage: hal::StateTransition { + from: hal::AccelerationStructureUses::BUILD_OUTPUT, + to: hal::AccelerationStructureUses::BUILD_INPUT, + }, + }); + + cmd_encoder.build_acceleration_structures( + 1, + [hal::BuildAccelerationStructureDescriptor { + mode: hal::AccelerationStructureBuildMode::Build, + flags: tlas_flags, + destination_acceleration_structure: &tlas, + scratch_buffer: &scratch_buffer, + entries: &tlas_entries, + source_acceleration_structure: None, + scratch_buffer_offset: 0, + }], + ); + + cmd_encoder.place_acceleration_structure_barrier(hal::AccelerationStructureBarrier { + usage: hal::StateTransition { + from: hal::AccelerationStructureUses::BUILD_OUTPUT, + to: hal::AccelerationStructureUses::SHADER_INPUT, + }, + }); + + let texture_barrier = hal::TextureBarrier { + texture: &texture, + range: wgpu_types::ImageSubresourceRange::default(), + usage: hal::StateTransition { + from: wgpu_types::TextureUses::UNINITIALIZED, + to: wgpu_types::TextureUses::STORAGE_READ_WRITE, + }, + }; + + cmd_encoder.transition_textures(iter::once(texture_barrier)); + } + + let init_fence_value = 1; + let fence = unsafe { + let mut fence = device.create_fence().unwrap(); + let init_cmd = cmd_encoder.end_encoding().unwrap(); + queue + .submit(&[&init_cmd], &[], (&mut fence, init_fence_value)) + .unwrap(); + device.wait(&fence, init_fence_value, None).unwrap(); + cmd_encoder.reset_all(iter::once(init_cmd)); + fence + }; + + Ok(Self { + instance, + adapter, + surface, + surface_format: surface_config.format, + device, + queue, + pipeline, + contexts: vec![ExecutionContext { + encoder: cmd_encoder, + fence, + fence_value: init_fence_value + 1, + used_views: Vec::new(), + used_cmd_bufs: Vec::new(), + frames_recorded: 0, + }], + context_index: 0, + extent: [window_size.0, window_size.1], + start: Instant::now(), + pipeline_layout, + bind_group, + texture, + instances, + instances_buffer, + blas, + tlas, + scratch_buffer, + time: 0.0, + indices_buffer: indices_buffer.map(|(buf, _)| buf), + vertices_buffer, + uniform_buffer, + texture_view, + bgl, + shader_module, + }) + } + + fn update(&mut self, _event: winit::event::WindowEvent) {} + + fn render(&mut self) { + let ctx = &mut self.contexts[self.context_index]; + + let surface_tex = unsafe { + self.surface + .acquire_texture(None, &ctx.fence) + .unwrap() + .texture + }; + + let target_barrier0 = hal::TextureBarrier { + texture: surface_tex.borrow(), + range: wgpu_types::ImageSubresourceRange::default(), + usage: hal::StateTransition { + from: wgpu_types::TextureUses::UNINITIALIZED, + to: wgpu_types::TextureUses::COPY_DST, + }, + }; + + let instances_buffer_size = + self.instances.len() * size_of::(); + + let tlas_flags = hal::AccelerationStructureBuildFlags::PREFER_FAST_TRACE + | hal::AccelerationStructureBuildFlags::ALLOW_UPDATE; + + self.time += 1.0 / 60.0; + + self.instances[0].set_transform(&Affine3A::from_rotation_y(self.time)); + + unsafe { + let mapping = self + .device + .map_buffer(&self.instances_buffer, 0..instances_buffer_size as u64) + .unwrap(); + ptr::copy_nonoverlapping( + self.instances.as_ptr() as *const u8, + mapping.ptr.as_ptr(), + instances_buffer_size, + ); + self.device.unmap_buffer(&self.instances_buffer); + assert!(mapping.is_coherent); + } + + unsafe { + ctx.encoder.begin_encoding(Some("frame")).unwrap(); + + let instances = hal::AccelerationStructureInstances { + buffer: Some(&self.instances_buffer), + count: self.instances.len() as u32, + offset: 0, + }; + + ctx.encoder + .place_acceleration_structure_barrier(hal::AccelerationStructureBarrier { + usage: hal::StateTransition { + from: hal::AccelerationStructureUses::SHADER_INPUT, + to: hal::AccelerationStructureUses::BUILD_INPUT, + }, + }); + + ctx.encoder.build_acceleration_structures( + 1, + [hal::BuildAccelerationStructureDescriptor { + mode: hal::AccelerationStructureBuildMode::Update, + flags: tlas_flags, + destination_acceleration_structure: &self.tlas, + scratch_buffer: &self.scratch_buffer, + entries: &hal::AccelerationStructureEntries::Instances(instances), + source_acceleration_structure: Some(&self.tlas), + scratch_buffer_offset: 0, + }], + ); + + ctx.encoder + .place_acceleration_structure_barrier(hal::AccelerationStructureBarrier { + usage: hal::StateTransition { + from: hal::AccelerationStructureUses::BUILD_OUTPUT, + to: hal::AccelerationStructureUses::SHADER_INPUT, + }, + }); + + let scratch_buffer_barrier = hal::BufferBarrier { + buffer: &self.scratch_buffer, + usage: hal::StateTransition { + from: wgpu_types::BufferUses::BOTTOM_LEVEL_ACCELERATION_STRUCTURE_INPUT, + to: wgpu_types::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT, + }, + }; + ctx.encoder + .transition_buffers(iter::once(scratch_buffer_barrier)); + + ctx.encoder.transition_textures(iter::once(target_barrier0)); + } + + let surface_view_desc = hal::TextureViewDescriptor { + label: None, + format: self.surface_format, + dimension: wgpu_types::TextureViewDimension::D2, + usage: wgpu_types::TextureUses::COPY_DST, + range: wgpu_types::ImageSubresourceRange::default(), + }; + let surface_tex_view = unsafe { + self.device + .create_texture_view(surface_tex.borrow(), &surface_view_desc) + .unwrap() + }; + unsafe { + ctx.encoder.begin_compute_pass(&hal::ComputePassDescriptor { + label: None, + timestamp_writes: None, + }); + ctx.encoder.set_compute_pipeline(&self.pipeline); + ctx.encoder + .set_bind_group(&self.pipeline_layout, 0, &self.bind_group, &[]); + ctx.encoder.dispatch([512 / 8, 512 / 8, 1]); + } + + ctx.frames_recorded += 1; + + let target_barrier1 = hal::TextureBarrier { + texture: surface_tex.borrow(), + range: wgpu_types::ImageSubresourceRange::default(), + usage: hal::StateTransition { + from: wgpu_types::TextureUses::COPY_DST, + to: wgpu_types::TextureUses::PRESENT, + }, + }; + let target_barrier2 = hal::TextureBarrier { + texture: &self.texture, + range: wgpu_types::ImageSubresourceRange::default(), + usage: hal::StateTransition { + from: wgpu_types::TextureUses::STORAGE_READ_WRITE, + to: wgpu_types::TextureUses::COPY_SRC, + }, + }; + let target_barrier3 = hal::TextureBarrier { + texture: &self.texture, + range: wgpu_types::ImageSubresourceRange::default(), + usage: hal::StateTransition { + from: wgpu_types::TextureUses::COPY_SRC, + to: wgpu_types::TextureUses::STORAGE_READ_WRITE, + }, + }; + unsafe { + ctx.encoder.end_compute_pass(); + ctx.encoder.transition_textures(iter::once(target_barrier2)); + ctx.encoder.copy_texture_to_texture( + &self.texture, + wgpu_types::TextureUses::COPY_SRC, + surface_tex.borrow(), + std::iter::once(hal::TextureCopy { + src_base: hal::TextureCopyBase { + mip_level: 0, + array_layer: 0, + origin: wgpu_types::Origin3d::ZERO, + aspect: hal::FormatAspects::COLOR, + }, + dst_base: hal::TextureCopyBase { + mip_level: 0, + array_layer: 0, + origin: wgpu_types::Origin3d::ZERO, + aspect: hal::FormatAspects::COLOR, + }, + size: hal::CopyExtent { + width: 512, + height: 512, + depth: 1, + }, + }), + ); + ctx.encoder.transition_textures(iter::once(target_barrier1)); + ctx.encoder.transition_textures(iter::once(target_barrier3)); + } + + unsafe { + let cmd_buf = ctx.encoder.end_encoding().unwrap(); + self.queue + .submit( + &[&cmd_buf], + &[&surface_tex], + (&mut ctx.fence, ctx.fence_value), + ) + .unwrap(); + self.queue.present(&self.surface, surface_tex).unwrap(); + ctx.used_cmd_bufs.push(cmd_buf); + ctx.used_views.push(surface_tex_view); + }; + + log::info!("Context switch from {}", self.context_index); + let old_fence_value = ctx.fence_value; + if self.contexts.len() == 1 { + let hal_desc = hal::CommandEncoderDescriptor { + label: None, + queue: &self.queue, + }; + self.contexts.push(unsafe { + ExecutionContext { + encoder: self.device.create_command_encoder(&hal_desc).unwrap(), + fence: self.device.create_fence().unwrap(), + fence_value: 0, + used_views: Vec::new(), + used_cmd_bufs: Vec::new(), + frames_recorded: 0, + } + }); + } + self.context_index = (self.context_index + 1) % self.contexts.len(); + let next = &mut self.contexts[self.context_index]; + unsafe { + next.wait_and_clear(&self.device); + } + next.fence_value = old_fence_value + 1; + } + + fn exit(mut self) { + unsafe { + { + let ctx = &mut self.contexts[self.context_index]; + self.queue + .submit(&[], &[], (&mut ctx.fence, ctx.fence_value)) + .unwrap(); + } + + for mut ctx in self.contexts { + ctx.wait_and_clear(&self.device); + drop(ctx.encoder); + self.device.destroy_fence(ctx.fence); + } + + self.device.destroy_bind_group(self.bind_group); + self.device.destroy_buffer(self.scratch_buffer); + self.device.destroy_buffer(self.instances_buffer); + if let Some(buffer) = self.indices_buffer { + self.device.destroy_buffer(buffer); + } + self.device.destroy_buffer(self.vertices_buffer); + self.device.destroy_buffer(self.uniform_buffer); + self.device.destroy_acceleration_structure(self.tlas); + self.device.destroy_acceleration_structure(self.blas); + self.device.destroy_texture_view(self.texture_view); + self.device.destroy_texture(self.texture); + self.device.destroy_compute_pipeline(self.pipeline); + self.device.destroy_pipeline_layout(self.pipeline_layout); + self.device.destroy_bind_group_layout(self.bgl); + self.device.destroy_shader_module(self.shader_module); + + self.surface.unconfigure(&self.device); + drop(self.queue); + drop(self.device); + drop(self.surface); + drop(self.adapter); + } + } +} + +cfg_if::cfg_if! { + // Apple + Metal + if #[cfg(all(target_vendor = "apple", feature = "metal"))] { + type Api = hal::api::Metal; + } + // Wasm + Vulkan + else if #[cfg(all(not(target_arch = "wasm32"), feature = "vulkan"))] { + type Api = hal::api::Vulkan; + } + // Windows + DX12 + else if #[cfg(all(windows, feature = "dx12"))] { + type Api = hal::api::Dx12; + } + // Anything + GLES + else if #[cfg(feature = "gles")] { + type Api = hal::api::Gles; + } + // Fallback + else { + type Api = hal::api::Noop; + } +} + +struct App { + example: Option>, + window: Option, +} + +impl ApplicationHandler for App { + fn resumed(&mut self, event_loop: &ActiveEventLoop) { + if self.window.is_some() { + return; + } + let window = event_loop + .create_window( + Window::default_attributes() + .with_title("hal-ray-traced-triangle") + .with_inner_size(winit::dpi::PhysicalSize { + width: 512, + height: 512, + }) + .with_resizable(false) + .with_enabled_buttons(WindowButtons::CLOSE), + ) + .unwrap(); + let example = Example::::init(&window).expect("Selected backend is not supported"); + self.window = Some(window); + self.example = Some(example); + } + + fn exiting(&mut self, _event_loop: &ActiveEventLoop) { + self.example.take().unwrap().exit(); + } + + fn about_to_wait(&mut self, _event_loop: &ActiveEventLoop) { + if let Some(window) = &self.window { + window.request_redraw(); + } + } + + fn window_event( + &mut self, + event_loop: &ActiveEventLoop, + _window_id: winit::window::WindowId, + event: winit::event::WindowEvent, + ) { + event_loop.set_control_flow(ControlFlow::Poll); + + match event { + winit::event::WindowEvent::CloseRequested => { + event_loop.exit(); + } + winit::event::WindowEvent::KeyboardInput { event, .. } + if event.physical_key + == winit::keyboard::PhysicalKey::Code(winit::keyboard::KeyCode::Escape) => + { + event_loop.exit(); + } + winit::event::WindowEvent::RedrawRequested => { + let ex = self.example.as_mut().unwrap(); + ex.render(); + } + _ => { + self.example.as_mut().unwrap().update(event); + } + } + } +} + +fn main() { + env_logger::init(); + + let event_loop = winit::event_loop::EventLoop::new().unwrap(); + let mut app = App { + example: None, + window: None, + }; + event_loop.run_app(&mut app).unwrap(); +} diff --git a/third_party/wgpu-hal-29.0.4-patched/examples/ray-traced-triangle/shader.wgsl b/third_party/wgpu-hal-29.0.4-patched/examples/ray-traced-triangle/shader.wgsl new file mode 100644 index 00000000..9710ca65 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/examples/ray-traced-triangle/shader.wgsl @@ -0,0 +1,39 @@ +enable wgpu_ray_query; + +struct Uniforms { + view_inv: mat4x4, + proj_inv: mat4x4, +}; +@group(0) @binding(0) +var uniforms: Uniforms; + +@group(0) @binding(1) +var output: texture_storage_2d; + +@group(0) @binding(2) +var acc_struct: acceleration_structure; + +@compute @workgroup_size(8, 8) +fn main(@builtin(global_invocation_id) global_id: vec3) { + let target_size = textureDimensions(output); + + let pixel_center = vec2(global_id.xy) + vec2(0.5); + let in_uv = pixel_center / vec2(target_size.xy); + let d = in_uv * 2.0 - 1.0; + + let origin = (uniforms.view_inv * vec4(0.0, 0.0, 0.0, 1.0)).xyz; + let temp = uniforms.proj_inv * vec4(d.x, d.y, 1.0, 1.0); + let direction = (uniforms.view_inv * vec4(normalize(temp.xyz), 0.0)).xyz; + + var rq: ray_query; + rayQueryInitialize(&rq, acc_struct, RayDesc(0u, 0xFFu, 0.1, 200.0, origin, direction)); + rayQueryProceed(&rq); + + var color = vec4(0.0, 0.0, 0.0, 1.0); + let intersection = rayQueryGetCommittedIntersection(&rq); + if intersection.kind != RAY_QUERY_INTERSECTION_NONE { + color = vec4(intersection.barycentrics, 1.0 - intersection.barycentrics.x - intersection.barycentrics.y, 1.0); + } + + textureStore(output, global_id.xy, color); +} \ No newline at end of file diff --git a/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/conv.rs b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/conv.rs new file mode 100644 index 00000000..6e27081a --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/conv.rs @@ -0,0 +1,295 @@ +use alloc::string::String; +use std::{ffi::OsString, os::windows::ffi::OsStringExt}; + +use windows::Win32::Graphics::Dxgi; + +// Helper to convert DXGI adapter name to a normal string +pub fn map_adapter_name(name: [u16; 128]) -> String { + let len = name.iter().take_while(|&&c| c != 0).count(); + let name = OsString::from_wide(&name[..len]); + name.to_string_lossy().into_owned() +} + +pub fn map_texture_format_failable( + format: wgt::TextureFormat, +) -> Option { + use wgt::TextureFormat as Tf; + use Dxgi::Common::*; + + Some(match format { + Tf::R8Unorm => DXGI_FORMAT_R8_UNORM, + Tf::R8Snorm => DXGI_FORMAT_R8_SNORM, + Tf::R8Uint => DXGI_FORMAT_R8_UINT, + Tf::R8Sint => DXGI_FORMAT_R8_SINT, + Tf::R16Uint => DXGI_FORMAT_R16_UINT, + Tf::R16Sint => DXGI_FORMAT_R16_SINT, + Tf::R16Unorm => DXGI_FORMAT_R16_UNORM, + Tf::R16Snorm => DXGI_FORMAT_R16_SNORM, + Tf::R16Float => DXGI_FORMAT_R16_FLOAT, + Tf::Rg8Unorm => DXGI_FORMAT_R8G8_UNORM, + Tf::Rg8Snorm => DXGI_FORMAT_R8G8_SNORM, + Tf::Rg8Uint => DXGI_FORMAT_R8G8_UINT, + Tf::Rg8Sint => DXGI_FORMAT_R8G8_SINT, + Tf::Rg16Unorm => DXGI_FORMAT_R16G16_UNORM, + Tf::Rg16Snorm => DXGI_FORMAT_R16G16_SNORM, + Tf::R32Uint => DXGI_FORMAT_R32_UINT, + Tf::R32Sint => DXGI_FORMAT_R32_SINT, + Tf::R32Float => DXGI_FORMAT_R32_FLOAT, + Tf::Rg16Uint => DXGI_FORMAT_R16G16_UINT, + Tf::Rg16Sint => DXGI_FORMAT_R16G16_SINT, + Tf::Rg16Float => DXGI_FORMAT_R16G16_FLOAT, + Tf::Rgba8Unorm => DXGI_FORMAT_R8G8B8A8_UNORM, + Tf::Rgba8UnormSrgb => DXGI_FORMAT_R8G8B8A8_UNORM_SRGB, + Tf::Bgra8UnormSrgb => DXGI_FORMAT_B8G8R8A8_UNORM_SRGB, + Tf::Rgba8Snorm => DXGI_FORMAT_R8G8B8A8_SNORM, + Tf::Bgra8Unorm => DXGI_FORMAT_B8G8R8A8_UNORM, + Tf::Rgba8Uint => DXGI_FORMAT_R8G8B8A8_UINT, + Tf::Rgba8Sint => DXGI_FORMAT_R8G8B8A8_SINT, + Tf::Rgb9e5Ufloat => DXGI_FORMAT_R9G9B9E5_SHAREDEXP, + Tf::Rgb10a2Uint => DXGI_FORMAT_R10G10B10A2_UINT, + Tf::Rgb10a2Unorm => DXGI_FORMAT_R10G10B10A2_UNORM, + Tf::Rg11b10Ufloat => DXGI_FORMAT_R11G11B10_FLOAT, + Tf::R64Uint => DXGI_FORMAT_R32G32_UINT, // R64 emulated by R32G32 + Tf::Rg32Uint => DXGI_FORMAT_R32G32_UINT, + Tf::Rg32Sint => DXGI_FORMAT_R32G32_SINT, + Tf::Rg32Float => DXGI_FORMAT_R32G32_FLOAT, + Tf::Rgba16Uint => DXGI_FORMAT_R16G16B16A16_UINT, + Tf::Rgba16Sint => DXGI_FORMAT_R16G16B16A16_SINT, + Tf::Rgba16Unorm => DXGI_FORMAT_R16G16B16A16_UNORM, + Tf::Rgba16Snorm => DXGI_FORMAT_R16G16B16A16_SNORM, + Tf::Rgba16Float => DXGI_FORMAT_R16G16B16A16_FLOAT, + Tf::Rgba32Uint => DXGI_FORMAT_R32G32B32A32_UINT, + Tf::Rgba32Sint => DXGI_FORMAT_R32G32B32A32_SINT, + Tf::Rgba32Float => DXGI_FORMAT_R32G32B32A32_FLOAT, + Tf::Stencil8 => DXGI_FORMAT_D24_UNORM_S8_UINT, + Tf::Depth16Unorm => DXGI_FORMAT_D16_UNORM, + Tf::Depth24Plus => DXGI_FORMAT_D24_UNORM_S8_UINT, + Tf::Depth24PlusStencil8 => DXGI_FORMAT_D24_UNORM_S8_UINT, + Tf::Depth32Float => DXGI_FORMAT_D32_FLOAT, + Tf::Depth32FloatStencil8 => DXGI_FORMAT_D32_FLOAT_S8X24_UINT, + Tf::NV12 => DXGI_FORMAT_NV12, + Tf::P010 => DXGI_FORMAT_P010, + Tf::Bc1RgbaUnorm => DXGI_FORMAT_BC1_UNORM, + Tf::Bc1RgbaUnormSrgb => DXGI_FORMAT_BC1_UNORM_SRGB, + Tf::Bc2RgbaUnorm => DXGI_FORMAT_BC2_UNORM, + Tf::Bc2RgbaUnormSrgb => DXGI_FORMAT_BC2_UNORM_SRGB, + Tf::Bc3RgbaUnorm => DXGI_FORMAT_BC3_UNORM, + Tf::Bc3RgbaUnormSrgb => DXGI_FORMAT_BC3_UNORM_SRGB, + Tf::Bc4RUnorm => DXGI_FORMAT_BC4_UNORM, + Tf::Bc4RSnorm => DXGI_FORMAT_BC4_SNORM, + Tf::Bc5RgUnorm => DXGI_FORMAT_BC5_UNORM, + Tf::Bc5RgSnorm => DXGI_FORMAT_BC5_SNORM, + Tf::Bc6hRgbUfloat => DXGI_FORMAT_BC6H_UF16, + Tf::Bc6hRgbFloat => DXGI_FORMAT_BC6H_SF16, + Tf::Bc7RgbaUnorm => DXGI_FORMAT_BC7_UNORM, + Tf::Bc7RgbaUnormSrgb => DXGI_FORMAT_BC7_UNORM_SRGB, + Tf::Etc2Rgb8Unorm + | Tf::Etc2Rgb8UnormSrgb + | Tf::Etc2Rgb8A1Unorm + | Tf::Etc2Rgb8A1UnormSrgb + | Tf::Etc2Rgba8Unorm + | Tf::Etc2Rgba8UnormSrgb + | Tf::EacR11Unorm + | Tf::EacR11Snorm + | Tf::EacRg11Unorm + | Tf::EacRg11Snorm + | Tf::Astc { + block: _, + channel: _, + } => return None, + }) +} + +pub fn map_texture_format(format: wgt::TextureFormat) -> Dxgi::Common::DXGI_FORMAT { + match map_texture_format_failable(format) { + Some(f) => f, + None => unreachable!(), + } +} + +// Note: DXGI doesn't allow sRGB format on the swapchain, +// but creating RTV of swapchain buffers with sRGB works. +pub fn map_texture_format_nosrgb(format: wgt::TextureFormat) -> Dxgi::Common::DXGI_FORMAT { + match format { + wgt::TextureFormat::Bgra8UnormSrgb => Dxgi::Common::DXGI_FORMAT_B8G8R8A8_UNORM, + wgt::TextureFormat::Rgba8UnormSrgb => Dxgi::Common::DXGI_FORMAT_R8G8B8A8_UNORM, + _ => map_texture_format(format), + } +} + +// SRV and UAV can't use the depth or typeless formats +// see https://microsoft.github.io/DirectX-Specs/d3d/PlanarDepthStencilDDISpec.html#view-creation +pub fn map_texture_format_for_srv_uav( + format: wgt::TextureFormat, + aspect: crate::FormatAspects, +) -> Option { + Some(match (format, aspect) { + (wgt::TextureFormat::Depth16Unorm, crate::FormatAspects::DEPTH) => { + Dxgi::Common::DXGI_FORMAT_R16_UNORM + } + (wgt::TextureFormat::Depth32Float, crate::FormatAspects::DEPTH) => { + Dxgi::Common::DXGI_FORMAT_R32_FLOAT + } + (wgt::TextureFormat::Depth32FloatStencil8, crate::FormatAspects::DEPTH) => { + Dxgi::Common::DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS + } + ( + wgt::TextureFormat::Depth24Plus | wgt::TextureFormat::Depth24PlusStencil8, + crate::FormatAspects::DEPTH, + ) => Dxgi::Common::DXGI_FORMAT_R24_UNORM_X8_TYPELESS, + + (wgt::TextureFormat::Depth32FloatStencil8, crate::FormatAspects::STENCIL) => { + Dxgi::Common::DXGI_FORMAT_X32_TYPELESS_G8X24_UINT + } + ( + wgt::TextureFormat::Stencil8 | wgt::TextureFormat::Depth24PlusStencil8, + crate::FormatAspects::STENCIL, + ) => Dxgi::Common::DXGI_FORMAT_X24_TYPELESS_G8_UINT, + + (_, crate::FormatAspects::DEPTH) + | (_, crate::FormatAspects::STENCIL) + | (_, crate::FormatAspects::DEPTH_STENCIL) => return None, + + _ => map_texture_format(format), + }) +} + +// see https://microsoft.github.io/DirectX-Specs/d3d/PlanarDepthStencilDDISpec.html#planar-layout-for-staging-from-buffer +pub fn map_texture_format_for_copy( + format: wgt::TextureFormat, + aspect: crate::FormatAspects, +) -> Option { + Some(match (format, aspect) { + (wgt::TextureFormat::Depth16Unorm, crate::FormatAspects::DEPTH) => { + Dxgi::Common::DXGI_FORMAT_R16_UNORM + } + ( + wgt::TextureFormat::Depth32Float | wgt::TextureFormat::Depth32FloatStencil8, + crate::FormatAspects::DEPTH, + ) => Dxgi::Common::DXGI_FORMAT_R32_FLOAT, + + ( + wgt::TextureFormat::Stencil8 + | wgt::TextureFormat::Depth24PlusStencil8 + | wgt::TextureFormat::Depth32FloatStencil8, + crate::FormatAspects::STENCIL, + ) => Dxgi::Common::DXGI_FORMAT_R8_UINT, + + (format, crate::FormatAspects::COLOR) => map_texture_format(format), + + _ => return None, + }) +} + +pub fn map_texture_format_for_resource( + format: wgt::TextureFormat, + usage: wgt::TextureUses, + has_view_formats: bool, + casting_fully_typed_format_supported: bool, +) -> Dxgi::Common::DXGI_FORMAT { + use wgt::TextureFormat as Tf; + use Dxgi::Common::*; + + if casting_fully_typed_format_supported { + map_texture_format(format) + + // We might view this resource as srgb or non-srgb + } else if has_view_formats { + match format { + Tf::Rgba8Unorm | Tf::Rgba8UnormSrgb => DXGI_FORMAT_R8G8B8A8_TYPELESS, + Tf::Bgra8Unorm | Tf::Bgra8UnormSrgb => DXGI_FORMAT_B8G8R8A8_TYPELESS, + Tf::Bc1RgbaUnorm | Tf::Bc1RgbaUnormSrgb => DXGI_FORMAT_BC1_TYPELESS, + Tf::Bc2RgbaUnorm | Tf::Bc2RgbaUnormSrgb => DXGI_FORMAT_BC2_TYPELESS, + Tf::Bc3RgbaUnorm | Tf::Bc3RgbaUnormSrgb => DXGI_FORMAT_BC3_TYPELESS, + Tf::Bc7RgbaUnorm | Tf::Bc7RgbaUnormSrgb => DXGI_FORMAT_BC7_TYPELESS, + format => map_texture_format(format), + } + + // We might view this resource as SRV/UAV but also as DSV + } else if format.is_depth_stencil_format() + && usage.intersects( + wgt::TextureUses::RESOURCE + | wgt::TextureUses::STORAGE_READ_ONLY + | wgt::TextureUses::STORAGE_WRITE_ONLY + | wgt::TextureUses::STORAGE_READ_WRITE, + ) + { + match format { + Tf::Depth16Unorm => DXGI_FORMAT_R16_TYPELESS, + Tf::Depth32Float => DXGI_FORMAT_R32_TYPELESS, + Tf::Depth32FloatStencil8 => DXGI_FORMAT_R32G8X24_TYPELESS, + Tf::Stencil8 | Tf::Depth24Plus | Tf::Depth24PlusStencil8 => DXGI_FORMAT_R24G8_TYPELESS, + _ => unreachable!(), + } + } else { + map_texture_format(format) + } +} + +pub fn map_index_format(format: wgt::IndexFormat) -> Dxgi::Common::DXGI_FORMAT { + match format { + wgt::IndexFormat::Uint16 => Dxgi::Common::DXGI_FORMAT_R16_UINT, + wgt::IndexFormat::Uint32 => Dxgi::Common::DXGI_FORMAT_R32_UINT, + } +} + +pub fn map_vertex_format(format: wgt::VertexFormat) -> Dxgi::Common::DXGI_FORMAT { + use wgt::VertexFormat as Vf; + use Dxgi::Common::*; + + match format { + Vf::Unorm8 => DXGI_FORMAT_R8_UNORM, + Vf::Snorm8 => DXGI_FORMAT_R8_SNORM, + Vf::Uint8 => DXGI_FORMAT_R8_UINT, + Vf::Sint8 => DXGI_FORMAT_R8_SINT, + Vf::Unorm8x2 => DXGI_FORMAT_R8G8_UNORM, + Vf::Snorm8x2 => DXGI_FORMAT_R8G8_SNORM, + Vf::Uint8x2 => DXGI_FORMAT_R8G8_UINT, + Vf::Sint8x2 => DXGI_FORMAT_R8G8_SINT, + Vf::Unorm8x4 => DXGI_FORMAT_R8G8B8A8_UNORM, + Vf::Snorm8x4 => DXGI_FORMAT_R8G8B8A8_SNORM, + Vf::Uint8x4 => DXGI_FORMAT_R8G8B8A8_UINT, + Vf::Sint8x4 => DXGI_FORMAT_R8G8B8A8_SINT, + Vf::Unorm16 => DXGI_FORMAT_R16_UNORM, + Vf::Snorm16 => DXGI_FORMAT_R16_SNORM, + Vf::Uint16 => DXGI_FORMAT_R16_UINT, + Vf::Sint16 => DXGI_FORMAT_R16_SINT, + Vf::Float16 => DXGI_FORMAT_R16_FLOAT, + Vf::Unorm16x2 => DXGI_FORMAT_R16G16_UNORM, + Vf::Snorm16x2 => DXGI_FORMAT_R16G16_SNORM, + Vf::Uint16x2 => DXGI_FORMAT_R16G16_UINT, + Vf::Sint16x2 => DXGI_FORMAT_R16G16_SINT, + Vf::Float16x2 => DXGI_FORMAT_R16G16_FLOAT, + Vf::Unorm16x4 => DXGI_FORMAT_R16G16B16A16_UNORM, + Vf::Snorm16x4 => DXGI_FORMAT_R16G16B16A16_SNORM, + Vf::Uint16x4 => DXGI_FORMAT_R16G16B16A16_UINT, + Vf::Sint16x4 => DXGI_FORMAT_R16G16B16A16_SINT, + Vf::Float16x4 => DXGI_FORMAT_R16G16B16A16_FLOAT, + Vf::Uint32 => DXGI_FORMAT_R32_UINT, + Vf::Sint32 => DXGI_FORMAT_R32_SINT, + Vf::Float32 => DXGI_FORMAT_R32_FLOAT, + Vf::Uint32x2 => DXGI_FORMAT_R32G32_UINT, + Vf::Sint32x2 => DXGI_FORMAT_R32G32_SINT, + Vf::Float32x2 => DXGI_FORMAT_R32G32_FLOAT, + Vf::Uint32x3 => DXGI_FORMAT_R32G32B32_UINT, + Vf::Sint32x3 => DXGI_FORMAT_R32G32B32_SINT, + Vf::Float32x3 => DXGI_FORMAT_R32G32B32_FLOAT, + Vf::Uint32x4 => DXGI_FORMAT_R32G32B32A32_UINT, + Vf::Sint32x4 => DXGI_FORMAT_R32G32B32A32_SINT, + Vf::Float32x4 => DXGI_FORMAT_R32G32B32A32_FLOAT, + Vf::Unorm10_10_10_2 => DXGI_FORMAT_R10G10B10A2_UNORM, + Vf::Unorm8x4Bgra => DXGI_FORMAT_B8G8R8A8_UNORM, + Vf::Float64 | Vf::Float64x2 | Vf::Float64x3 | Vf::Float64x4 => unimplemented!(), + } +} + +pub fn map_acomposite_alpha_mode(mode: wgt::CompositeAlphaMode) -> Dxgi::Common::DXGI_ALPHA_MODE { + match mode { + wgt::CompositeAlphaMode::PreMultiplied => Dxgi::Common::DXGI_ALPHA_MODE_PREMULTIPLIED, + wgt::CompositeAlphaMode::PostMultiplied => Dxgi::Common::DXGI_ALPHA_MODE_STRAIGHT, + wgt::CompositeAlphaMode::Opaque => Dxgi::Common::DXGI_ALPHA_MODE_IGNORE, + wgt::CompositeAlphaMode::Auto | wgt::CompositeAlphaMode::Inherit => { + Dxgi::Common::DXGI_ALPHA_MODE_UNSPECIFIED + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/exception.rs b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/exception.rs new file mode 100644 index 00000000..ac7d55d8 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/exception.rs @@ -0,0 +1,97 @@ +use alloc::{borrow::Cow, string::String}; + +use parking_lot::Mutex; +use windows::Win32::{Foundation, System::Diagnostics::Debug}; + +// This is a mutex as opposed to an atomic as we need to completely +// lock everyone out until we have registered or unregistered the +// exception handler, otherwise really nasty races could happen. +// +// By routing all the registration through these functions we can guarantee +// there is either 1 or 0 exception handlers registered, not multiple. +static EXCEPTION_HANDLER_COUNT: Mutex = Mutex::new(0); + +pub fn register_exception_handler() { + let mut count_guard = EXCEPTION_HANDLER_COUNT.lock(); + if *count_guard == 0 { + unsafe { Debug::AddVectoredExceptionHandler(0, Some(output_debug_string_handler)) }; + } + *count_guard += 1; +} + +pub fn unregister_exception_handler() { + let mut count_guard = EXCEPTION_HANDLER_COUNT.lock(); + if *count_guard == 1 { + unsafe { Debug::RemoveVectoredExceptionHandler(output_debug_string_handler as *mut _) }; + } + *count_guard -= 1; +} + +const MESSAGE_PREFIXES: &[(&str, log::Level)] = &[ + ("CORRUPTION", log::Level::Error), + ("ERROR", log::Level::Error), + ("WARNING", log::Level::Warn), + // We intentionally suppress "INFO" messages down to debug + // so that users are not innundated with info messages from the runtime. + ("INFO", log::Level::Debug), + ("MESSAGE", log::Level::Trace), +]; + +unsafe extern "system" fn output_debug_string_handler( + exception_info: *mut Debug::EXCEPTION_POINTERS, +) -> i32 { + // See https://stackoverflow.com/a/41480827 + let record = unsafe { &*(*exception_info).ExceptionRecord }; + if record.NumberParameters != 2 { + return Debug::EXCEPTION_CONTINUE_SEARCH; + } + let message = match record.ExceptionCode { + Foundation::DBG_PRINTEXCEPTION_C => { + String::from_utf8_lossy(bytemuck::cast_slice(&record.ExceptionInformation)) + } + Foundation::DBG_PRINTEXCEPTION_WIDE_C => Cow::Owned(String::from_utf16_lossy( + bytemuck::cast_slice(&record.ExceptionInformation), + )), + _ => return Debug::EXCEPTION_CONTINUE_SEARCH, + }; + + let message = match message.strip_prefix("D3D12 ") { + Some(msg) => msg + .trim_end_matches("\n\0") + .trim_end_matches("[ STATE_CREATION WARNING #0: UNKNOWN]"), + None => return Debug::EXCEPTION_CONTINUE_SEARCH, + }; + + let (message, level) = match MESSAGE_PREFIXES + .iter() + .find(|&&(prefix, _)| message.starts_with(prefix)) + { + Some(&(prefix, level)) => (&message[prefix.len() + 2..], level), + None => (message, log::Level::Debug), + }; + + if level == log::Level::Warn && message.contains("#82") { + // This is are useless spammy warnings (#820, #821): + // "The application did not pass any clear value to resource creation" + return Debug::EXCEPTION_CONTINUE_SEARCH; + } + + if level == log::Level::Warn && message.contains("DRAW_EMPTY_SCISSOR_RECTANGLE") { + // This is normal, WebGPU allows passing empty scissor rectangles. + return Debug::EXCEPTION_CONTINUE_SEARCH; + } + + let _ = std::panic::catch_unwind(|| { + log::log!(level, "{message}"); + }); + + #[cfg(feature = "validation_canary")] + if cfg!(debug_assertions) && level == log::Level::Error { + use alloc::string::ToString as _; + + // Set canary and continue + crate::VALIDATION_CANARY.add(message.to_string()); + } + + Debug::EXCEPTION_CONTINUE_EXECUTION +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/factory.rs b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/factory.rs new file mode 100644 index 00000000..ac050bcd --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/factory.rs @@ -0,0 +1,181 @@ +use alloc::{string::String, vec::Vec}; +use core::ops::Deref; + +use windows::{core::Interface as _, Win32::Graphics::Dxgi}; + +use crate::dx12::DxgiLib; + +use super::result::HResult as _; + +// We can rely on the presence of DXGI 1.4 since D3D12 requires WDDM 2.0, Windows 10 (1507), and so does DXGI 1.4. + +fn should_keep_adapter(adapter: &Dxgi::IDXGIAdapter1) -> bool { + let desc = unsafe { adapter.GetDesc1() }.unwrap(); + + // The Intel Haswell family of iGPUs had support for the D3D12 API but it was later + // removed due to a security vulnerability. + // + // We are explicitly filtering out all the devices in the family because we are now + // getting reports of device loss at a later time than at device creation time (`D3D12CreateDevice`). + // + // See https://www.intel.com/content/www/us/en/support/articles/000057520/graphics.html + // This list of device IDs is from https://dgpu-docs.intel.com/devices/hardware-table.html + let haswell_device_ids = [ + 0x0422, 0x0426, 0x042A, 0x042B, 0x042E, 0x0C22, 0x0C26, 0x0C2A, 0x0C2B, 0x0C2E, 0x0A22, + 0x0A2A, 0x0A2B, 0x0D2A, 0x0D2B, 0x0D2E, 0x0A26, 0x0A2E, 0x0D22, 0x0D26, 0x0412, 0x0416, + 0x0D12, 0x041A, 0x041B, 0x0C12, 0x0C16, 0x0C1A, 0x0C1B, 0x0C1E, 0x0A12, 0x0A1A, 0x0A1B, + 0x0D16, 0x0D1A, 0x0D1B, 0x0D1E, 0x041E, 0x0A16, 0x0A1E, 0x0402, 0x0406, 0x040A, 0x040B, + 0x040E, 0x0C02, 0x0C06, 0x0C0A, 0x0C0B, 0x0C0E, 0x0A02, 0x0A06, 0x0A0A, 0x0A0B, 0x0A0E, + 0x0D02, 0x0D06, 0x0D0A, 0x0D0B, 0x0D0E, + ]; + if desc.VendorId == 0x8086 && haswell_device_ids.contains(&desc.DeviceId) { + return false; + } + + // If run completely headless, windows will show two different WARP adapters, one + // which is lying about being an integrated card. This is so that programs + // that ignore software adapters will actually run on headless/gpu-less machines. + // + // We don't want that and discourage that kind of filtering anyway, so we skip the integrated WARP. + if desc.VendorId == 5140 + && !Dxgi::DXGI_ADAPTER_FLAG(desc.Flags as i32).contains(Dxgi::DXGI_ADAPTER_FLAG_SOFTWARE) + { + let adapter_name = super::conv::map_adapter_name(desc.Description); + if adapter_name.contains("Microsoft Basic Render Driver") { + return false; + } + } + + true +} + +#[derive(Clone, Debug)] +pub enum DxgiAdapter { + /// Provided by DXGI 1.4 + Adapter3(Dxgi::IDXGIAdapter3), + /// Provided by DXGI 1.6 + Adapter4(Dxgi::IDXGIAdapter4), +} + +impl DxgiAdapter { + pub fn query_video_memory_info( + &self, + group: Dxgi::DXGI_MEMORY_SEGMENT_GROUP, + ) -> Result { + let mut info = Dxgi::DXGI_QUERY_VIDEO_MEMORY_INFO::default(); + unsafe { self.QueryVideoMemoryInfo(0, group, &mut info) } + .into_device_result("QueryVideoMemoryInfo")?; + Ok(info) + } +} + +impl Deref for DxgiAdapter { + type Target = Dxgi::IDXGIAdapter3; + + fn deref(&self) -> &Self::Target { + match self { + DxgiAdapter::Adapter3(a) => a, + DxgiAdapter::Adapter4(a) => a, + } + } +} + +pub fn enumerate_adapters(factory: DxgiFactory) -> Vec { + let mut adapters = Vec::with_capacity(8); + + for cur_index in 0.. { + profiling::scope!("IDXGIFactory1::EnumAdapters1"); + let adapter1: Dxgi::IDXGIAdapter1 = match unsafe { factory.EnumAdapters1(cur_index) } { + Ok(a) => a, + Err(e) if e.code() == Dxgi::DXGI_ERROR_NOT_FOUND => break, + Err(e) => { + log::error!("Failed enumerating adapters: {e}"); + break; + } + }; + + if !should_keep_adapter(&adapter1) { + continue; + } + + if let Ok(adapter4) = adapter1.cast::() { + adapters.push(DxgiAdapter::Adapter4(adapter4)); + } else { + let adapter3 = adapter1.cast::().unwrap(); + adapters.push(DxgiAdapter::Adapter3(adapter3)); + } + } + + adapters +} + +#[derive(Clone, Debug)] +pub enum DxgiFactory { + /// Provided by DXGI 1.4 + Factory4(Dxgi::IDXGIFactory4), + /// Provided by DXGI 1.5 + Factory5(Dxgi::IDXGIFactory5), + /// Provided by DXGI 1.6 + Factory6(Dxgi::IDXGIFactory6), +} + +impl Deref for DxgiFactory { + type Target = Dxgi::IDXGIFactory4; + + fn deref(&self) -> &Self::Target { + match self { + DxgiFactory::Factory4(f) => f, + DxgiFactory::Factory5(f) => f, + DxgiFactory::Factory6(f) => f, + } + } +} + +impl DxgiFactory { + pub fn as_factory5(&self) -> Option<&Dxgi::IDXGIFactory5> { + match self { + Self::Factory4(_) => None, + Self::Factory5(f) => Some(f), + Self::Factory6(f) => Some(f), + } + } +} + +pub fn create_factory( + instance_flags: wgt::InstanceFlags, +) -> Result<(DxgiLib, DxgiFactory), crate::InstanceError> { + let lib_dxgi = DxgiLib::new().map_err(|e| { + crate::InstanceError::with_source(String::from("failed to load dxgi.dll"), e) + })?; + + let mut factory_flags = Dxgi::DXGI_CREATE_FACTORY_FLAGS::default(); + + if instance_flags.contains(wgt::InstanceFlags::VALIDATION) { + // The `DXGI_CREATE_FACTORY_DEBUG` flag is only allowed to be passed to + // `CreateDXGIFactory2` if the debug interface is actually available. So + // we check for whether it exists first. + if let Ok(Some(_)) = lib_dxgi.debug_interface1() { + factory_flags |= Dxgi::DXGI_CREATE_FACTORY_DEBUG; + } + } + + let factory4 = match lib_dxgi.create_factory4(factory_flags) { + Ok(factory) => factory, + Err(err) => { + return Err(crate::InstanceError::with_source( + String::from("IDXGIFactory4 creation failed"), + err, + )); + } + }; + + if let Ok(factory6) = factory4.cast::() { + return Ok((lib_dxgi, DxgiFactory::Factory6(factory6))); + } + + if let Ok(factory5) = factory4.cast::() { + return Ok((lib_dxgi, DxgiFactory::Factory5(factory5))); + } + + Ok((lib_dxgi, DxgiFactory::Factory4(factory4))) +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/mod.rs b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/mod.rs new file mode 100644 index 00000000..5cc6af6e --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/mod.rs @@ -0,0 +1,6 @@ +pub mod conv; +pub mod exception; +pub mod factory; +pub mod name; +pub mod result; +pub mod time; diff --git a/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/name.rs b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/name.rs new file mode 100644 index 00000000..d120e4ed --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/name.rs @@ -0,0 +1,24 @@ +use windows::Win32::Graphics::Direct3D12::ID3D12Object; + +use crate::auxil::dxgi::result::HResult; + +/// Helper trait for setting the name of a D3D12 object. +/// +/// This is implemented on all types that can be converted to an [`ID3D12Object`]. +pub trait ObjectExt { + fn set_name(&self, name: &str) -> Result<(), crate::DeviceError>; +} + +impl ObjectExt for T +where + // Windows impls `From` for all parent interfaces, so we can use that to convert to ID3D12Object. + // + // This includes implementations for references. + for<'a> &'a ID3D12Object: From<&'a T>, +{ + fn set_name(&self, name: &str) -> Result<(), crate::DeviceError> { + let name = windows::core::HSTRING::from(name); + let object: &ID3D12Object = self.into(); + unsafe { object.SetName(&name).into_device_result("SetName") } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/result.rs b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/result.rs new file mode 100644 index 00000000..f3d77e27 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/result.rs @@ -0,0 +1,28 @@ +use windows::Win32::{Foundation, Graphics::Dxgi}; + +pub(crate) trait HResult { + fn into_device_result(self, description: &str) -> Result; +} +impl HResult for windows::core::Result { + fn into_device_result(self, description: &str) -> Result { + #![allow(unreachable_code)] + + self.map_err(|err| { + log::error!("{description} failed: {err}"); + + match err.code() { + Foundation::E_OUTOFMEMORY => crate::DeviceError::OutOfMemory, + Dxgi::DXGI_ERROR_DEVICE_RESET | Dxgi::DXGI_ERROR_DEVICE_REMOVED => { + #[cfg(feature = "device_lost_panic")] + panic!("{description} failed: Device lost ({err})"); + crate::DeviceError::Lost + } + _ => { + #[cfg(feature = "internal_error_panic")] + panic!("{description} failed: {err}"); + crate::DeviceError::Unexpected + } + } + }) + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/time.rs b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/time.rs new file mode 100644 index 00000000..6b4c60fc --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/auxil/dxgi/time.rs @@ -0,0 +1,97 @@ +#![allow(dead_code)] // IPresentationManager is unused currently + +use windows::Win32::System::Performance::{QueryPerformanceCounter, QueryPerformanceFrequency}; + +pub enum PresentationTimer { + /// DXGI uses [`QueryPerformanceCounter()`] + Dxgi { + /// How many ticks of QPC per second + frequency: u64, + }, + /// [`IPresentationManager`] uses [`QueryInterruptTimePrecise()`] + /// + /// [`IPresentationManager`]: https://microsoft.github.io/windows-docs-rs/doc/windows/Win32/Graphics/CompositionSwapchain/struct.IPresentationManager.html + /// [`QueryInterruptTimePrecise()`]: https://microsoft.github.io/windows-docs-rs/doc/windows/Win32/System/WindowsProgramming/fn.QueryInterruptTimePrecise.html + #[allow(non_snake_case)] + IPresentationManager { + fnQueryInterruptTimePrecise: unsafe extern "system" fn(*mut u64), + }, +} + +impl core::fmt::Debug for PresentationTimer { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match *self { + Self::Dxgi { frequency } => f + .debug_struct("DXGI") + .field("frequency", &frequency) + .finish(), + Self::IPresentationManager { + fnQueryInterruptTimePrecise, + } => f + .debug_struct("IPresentationManager") + .field( + "QueryInterruptTimePrecise", + &(fnQueryInterruptTimePrecise as usize), + ) + .finish(), + } + } +} + +impl PresentationTimer { + /// Create a presentation timer using QueryPerformanceFrequency (what DXGI uses for presentation times) + pub fn new_dxgi() -> Self { + let mut frequency = 0; + unsafe { QueryPerformanceFrequency(&mut frequency) }.unwrap(); + + Self::Dxgi { + frequency: frequency + .try_into() + .expect("Frequency should not be negative"), + } + } + + /// Create a presentation timer using QueryInterruptTimePrecise (what IPresentationManager uses for presentation times) + /// + /// Panics if QueryInterruptTimePrecise isn't found (below Win10) + pub fn new_ipresentation_manager() -> Self { + // We need to load this explicitly, as QueryInterruptTimePrecise is only available on Windows 10+ + // + // Docs say it's in kernel32.dll, but it's actually in kernelbase.dll. + // api-ms-win-core-realtime-l1-1-1.dll + let kernelbase = + libloading::os::windows::Library::open_already_loaded("kernelbase.dll").unwrap(); + // No concerns about lifetimes here as kernelbase is always there. + let ptr = unsafe { + kernelbase + .get(c"QueryInterruptTimePrecise".to_bytes()) + .unwrap() + }; + Self::IPresentationManager { + fnQueryInterruptTimePrecise: *ptr, + } + } + + /// Gets the current time in nanoseconds. + pub fn get_timestamp_ns(&self) -> u128 { + // Always do u128 math _after_ hitting the timing function. + match *self { + PresentationTimer::Dxgi { frequency } => { + let mut counter = 0; + unsafe { QueryPerformanceCounter(&mut counter) }.unwrap(); + + // counter * (1_000_000_000 / freq) but re-ordered to make more precise + (counter as u128 * 1_000_000_000) / frequency as u128 + } + PresentationTimer::IPresentationManager { + fnQueryInterruptTimePrecise, + } => { + let mut counter = 0; + unsafe { fnQueryInterruptTimePrecise(&mut counter) }; + + // QueryInterruptTimePrecise uses units of 100ns for its tick. + counter as u128 * 100 + } + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/auxil/mod.rs b/third_party/wgpu-hal-29.0.4-patched/src/auxil/mod.rs new file mode 100644 index 00000000..4d82c1df --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/auxil/mod.rs @@ -0,0 +1,231 @@ +#[cfg(dx12)] +pub(super) mod dxgi; + +#[cfg(all(native, feature = "renderdoc"))] +pub(super) mod renderdoc; + +pub mod db { + pub mod amd { + /// cbindgen:ignore + pub const VENDOR: u32 = 0x1002; + } + pub mod apple { + /// cbindgen:ignore + pub const VENDOR: u32 = 0x106B; + } + pub mod arm { + /// cbindgen:ignore + pub const VENDOR: u32 = 0x13B5; + } + pub mod broadcom { + /// cbindgen:ignore + pub const VENDOR: u32 = 0x14E4; + } + pub mod imgtec { + /// cbindgen:ignore + pub const VENDOR: u32 = 0x1010; + } + pub mod intel { + /// cbindgen:ignore + pub const VENDOR: u32 = 0x8086; + pub const DEVICE_KABY_LAKE_MASK: u32 = 0x5900; + pub const DEVICE_SKY_LAKE_MASK: u32 = 0x1900; + } + pub mod mesa { + // Mesa does not actually have a PCI vendor id. + // + // To match Vulkan, we use the VkVendorId for Mesa in the gles backend so that lavapipe (Vulkan) and + // llvmpipe (OpenGL) have the same vendor id. + /// cbindgen:ignore + pub const VENDOR: u32 = 0x10005; + } + pub mod nvidia { + /// cbindgen:ignore + pub const VENDOR: u32 = 0x10DE; + } + pub mod qualcomm { + /// cbindgen:ignore + pub const VENDOR: u32 = 0x5143; + } +} + +/// Maximum binding size for the shaders that only support `i32` indexing. +/// Interestingly, the index itself can't reach that high, because the minimum +/// element size is 4 bytes, but the compiler toolchain still computes the +/// offset at some intermediate point, internally, as i32. +pub const MAX_I32_BINDING_SIZE: u32 = (1 << 31) - 1; + +pub use wgpu_naga_bridge::map_naga_stage; + +impl crate::CopyExtent { + pub fn map_extent_to_copy_size(extent: &wgt::Extent3d, dim: wgt::TextureDimension) -> Self { + Self { + width: extent.width, + height: extent.height, + depth: match dim { + wgt::TextureDimension::D1 | wgt::TextureDimension::D2 => 1, + wgt::TextureDimension::D3 => extent.depth_or_array_layers, + }, + } + } + + pub fn min(&self, other: &Self) -> Self { + Self { + width: self.width.min(other.width), + height: self.height.min(other.height), + depth: self.depth.min(other.depth), + } + } + + // Get the copy size at a specific mipmap level. This doesn't make most sense, + // since the copy extents are provided *for* a mipmap level to start with. + // But backends use `CopyExtent` more sparingly, and this piece is shared. + pub fn at_mip_level(&self, level: u32) -> Self { + Self { + width: (self.width >> level).max(1), + height: (self.height >> level).max(1), + depth: (self.depth >> level).max(1), + } + } +} + +impl crate::TextureCopyBase { + pub fn max_copy_size(&self, full_size: &crate::CopyExtent) -> crate::CopyExtent { + let mip = full_size.at_mip_level(self.mip_level); + crate::CopyExtent { + width: mip.width - self.origin.x, + height: mip.height - self.origin.y, + depth: mip.depth - self.origin.z, + } + } +} + +impl crate::BufferTextureCopy { + pub fn clamp_size_to_virtual(&mut self, full_size: &crate::CopyExtent) { + let max_size = self.texture_base.max_copy_size(full_size); + self.size = self.size.min(&max_size); + } +} + +impl crate::TextureCopy { + pub fn clamp_size_to_virtual( + &mut self, + full_src_size: &crate::CopyExtent, + full_dst_size: &crate::CopyExtent, + ) { + let max_src_size = self.src_base.max_copy_size(full_src_size); + let max_dst_size = self.dst_base.max_copy_size(full_dst_size); + self.size = self.size.min(&max_src_size).min(&max_dst_size); + } +} + +/// Adjust `limits` to honor HAL-imposed maximums and comply with WebGPU's +/// adapter capability guarantees. +#[cfg_attr(any(not(any_backend), metal), allow(dead_code))] +pub(crate) fn adjust_raw_limits(mut limits: wgt::Limits) -> wgt::Limits { + // Apply hal limits. + limits.max_bind_groups = limits.max_bind_groups.min(crate::MAX_BIND_GROUPS as u32); + limits.max_vertex_buffers = limits + .max_vertex_buffers + .min(crate::MAX_VERTEX_BUFFERS as u32); + limits.max_color_attachments = limits + .max_color_attachments + .min(crate::MAX_COLOR_ATTACHMENTS as u32); + + // Adjust limits according to WebGPU adapter capability guarantees. + // See . + + // WebGPU requires maxBindingsPerBindGroup to be at least the sum of all + // per-stage limits multiplied with the maximum shader stages per pipeline. + // + // Since backends already report their maximum maxBindingsPerBindGroup, + // we need to lower all per-stage limits to satisfy this guarantee. + const MAX_SHADER_STAGES_PER_PIPELINE: u32 = 2; + let max_per_stage_resources = + limits.max_bindings_per_bind_group / MAX_SHADER_STAGES_PER_PIPELINE; + + cap_limits_to_be_under_the_sum_limit( + [ + &mut limits.max_sampled_textures_per_shader_stage, + &mut limits.max_uniform_buffers_per_shader_stage, + &mut limits.max_storage_textures_per_shader_stage, + &mut limits.max_storage_buffers_per_shader_stage, + &mut limits.max_samplers_per_shader_stage, + &mut limits.max_acceleration_structures_per_shader_stage, + ], + max_per_stage_resources, + ); + + // Not required by the spec but dynamic buffers count + // towards non-dynamic buffer limits as well. + limits.max_dynamic_uniform_buffers_per_pipeline_layout = limits + .max_dynamic_uniform_buffers_per_pipeline_layout + .min(limits.max_uniform_buffers_per_shader_stage); + limits.max_dynamic_storage_buffers_per_pipeline_layout = limits + .max_dynamic_storage_buffers_per_pipeline_layout + .min(limits.max_storage_buffers_per_shader_stage); + + limits.min_uniform_buffer_offset_alignment = limits.min_uniform_buffer_offset_alignment.max(32); + limits.min_storage_buffer_offset_alignment = limits.min_storage_buffer_offset_alignment.max(32); + + limits.max_uniform_buffer_binding_size = limits + .max_uniform_buffer_binding_size + .min(limits.max_buffer_size); + limits.max_storage_buffer_binding_size = limits + .max_storage_buffer_binding_size + .min(limits.max_buffer_size); + + limits.max_storage_buffer_binding_size &= !(u64::from(wgt::STORAGE_BINDING_SIZE_ALIGNMENT) - 1); + limits.max_vertex_buffer_array_stride &= !(wgt::VERTEX_ALIGNMENT as u32 - 1); + + let x = limits.max_compute_workgroup_size_x; + let y = limits.max_compute_workgroup_size_y; + let z = limits.max_compute_workgroup_size_z; + let m = limits.max_compute_invocations_per_workgroup; + limits.max_compute_workgroup_size_x = x.min(m); + limits.max_compute_workgroup_size_y = y.min(m); + limits.max_compute_workgroup_size_z = z.min(m); + limits.max_compute_invocations_per_workgroup = m.min(x.saturating_mul(y).saturating_mul(z)); + + limits +} + +/// Evenly allocates space to each limit, +/// capping them only if strictly necessary. +pub fn cap_limits_to_be_under_the_sum_limit( + mut limits: [&mut u32; N], + sum_limit: u32, +) { + limits.sort(); + + let mut rem_limit = sum_limit; + let mut divisor = limits.len() as u32; + for limit_to_adjust in limits { + let limit = rem_limit / divisor; + *limit_to_adjust = (*limit_to_adjust).min(limit); + rem_limit -= *limit_to_adjust; + divisor -= 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cap_limits_to_be_under_the_sum_limit() { + test([3, 3, 3], 3, [1, 1, 1]); + test([3, 2, 1], 3, [1, 1, 1]); + test([1, 2, 3], 6, [1, 2, 3]); + test([1, 2, 3], 3, [1, 1, 1]); + test([1, 8, 100], 6, [1, 2, 3]); + test([2, 80, 80], 6, [2, 2, 2]); + test([2, 80, 80], 12, [2, 5, 5]); + + #[track_caller] + fn test(mut input: [u32; N], limit: u32, output: [u32; N]) { + cap_limits_to_be_under_the_sum_limit(input.each_mut(), limit); + assert_eq!(input, output); + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/auxil/renderdoc.rs b/third_party/wgpu-hal-29.0.4-patched/src/auxil/renderdoc.rs new file mode 100644 index 00000000..52fa2b0b --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/auxil/renderdoc.rs @@ -0,0 +1,141 @@ +//! RenderDoc integration - +#![cfg_attr(not(any(feature = "gles", feature = "vulkan")), allow(dead_code))] + +use alloc::format; +use alloc::string::String; +use core::{ffi, ptr}; + +/// The dynamically loaded RenderDoc API function table +#[repr(C)] +#[derive(Debug)] +pub struct RenderDocApi { + api: renderdoc_sys::RENDERDOC_API_1_4_1, + lib: libloading::Library, +} + +unsafe impl Send for RenderDocApi {} +unsafe impl Sync for RenderDocApi {} + +/// RenderDoc API type +#[derive(Debug)] +pub enum RenderDoc { + /// RenderDoc functionality is available + Available { + /// RenderDoc API with function pointers + api: RenderDocApi, + }, + /// RenderDoc functionality is _not_ available + NotAvailable { + /// A description why renderdoc functionality is not available + reason: String, + }, +} + +// TODO: replace with libloading API once supported +#[cfg(unix)] +const RTLD_NOLOAD: i32 = 0x4; + +impl RenderDoc { + pub unsafe fn new() -> Self { + type GetApiFn = unsafe extern "C" fn(version: u32, out: *mut *mut ffi::c_void) -> i32; + + #[cfg(windows)] + let renderdoc_filename = "renderdoc.dll"; + #[cfg(all(unix, not(target_os = "android")))] + let renderdoc_filename = "librenderdoc.so"; + #[cfg(target_os = "android")] + let renderdoc_filename = "libVkLayer_GLES_RenderDoc.so"; + + #[cfg(unix)] + let renderdoc_result: Result = unsafe { + libloading::os::unix::Library::open( + Some(renderdoc_filename), + libloading::os::unix::RTLD_NOW | RTLD_NOLOAD, + ) + } + .map(|lib| lib.into()); + + #[cfg(windows)] + let renderdoc_result: Result = + libloading::os::windows::Library::open_already_loaded(renderdoc_filename) + .map(|lib| lib.into()); + + let renderdoc_lib = match renderdoc_result { + Ok(lib) => lib, + Err(e) => { + return RenderDoc::NotAvailable { + reason: format!( + "Unable to load renderdoc library '{renderdoc_filename}': {e:?}" + ), + } + } + }; + + let get_api: libloading::Symbol = + match unsafe { renderdoc_lib.get(c"RENDERDOC_GetAPI".to_bytes()) } { + Ok(api) => api, + Err(e) => { + return RenderDoc::NotAvailable { + reason: format!( + "Unable to get RENDERDOC_GetAPI from renderdoc library '{renderdoc_filename}': {e:?}" + ), + } + } + }; + let mut obj = ptr::null_mut(); + match unsafe { get_api(10401, &mut obj) } { + 1 => RenderDoc::Available { + api: RenderDocApi { + api: unsafe { *obj.cast::() }, + lib: renderdoc_lib, + }, + }, + return_value => RenderDoc::NotAvailable { + reason: format!( + "Unable to get API from renderdoc library '{renderdoc_filename}': {return_value}" + ), + }, + } + } +} + +impl Default for RenderDoc { + fn default() -> Self { + if !cfg!(debug_assertions) { + return RenderDoc::NotAvailable { + reason: "RenderDoc support is only enabled with 'debug_assertions'".into(), + }; + } + unsafe { Self::new() } + } +} +/// An implementation specific handle +pub type Handle = *mut ffi::c_void; + +impl RenderDoc { + /// Start a RenderDoc frame capture + pub unsafe fn start_frame_capture(&self, device_handle: Handle, window_handle: Handle) -> bool { + match *self { + Self::Available { api: ref entry } => { + unsafe { entry.api.StartFrameCapture.unwrap()(device_handle, window_handle) }; + true + } + Self::NotAvailable { ref reason } => { + log::warn!("Could not start RenderDoc frame capture: {reason}"); + false + } + } + } + + /// End a RenderDoc frame capture + pub unsafe fn end_frame_capture(&self, device_handle: Handle, window_handle: Handle) { + match *self { + Self::Available { api: ref entry } => { + unsafe { entry.api.EndFrameCapture.unwrap()(device_handle, window_handle) }; + } + Self::NotAvailable { ref reason } => { + log::warn!("Could not end RenderDoc frame capture: {reason}") + } + }; + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/adapter.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/adapter.rs new file mode 100644 index 00000000..a86c337e --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/adapter.rs @@ -0,0 +1,1472 @@ +use alloc::{string::String, sync::Arc, vec::Vec}; +use core::ptr; +use std::thread; + +use parking_lot::Mutex; +use windows::{ + core::Interface as _, + Win32::{ + Devices::DeviceAndDriverInstallation::{ + SetupDiDestroyDeviceInfoList, SetupDiEnumDeviceInfo, SetupDiGetClassDevsW, + SetupDiGetDeviceRegistryPropertyW, DIGCF_PRESENT, GUID_DEVCLASS_DISPLAY, HDEVINFO, + SPDRP_ADDRESS, SPDRP_BUSNUMBER, SPDRP_HARDWAREID, SP_DEVINFO_DATA, + }, + Foundation::{GetLastError, ERROR_NO_MORE_ITEMS}, + Graphics::{Direct3D, Direct3D12, Dxgi}, + UI::WindowsAndMessaging, + }, +}; + +use super::D3D12Lib; +use crate::{ + auxil::{ + self, + dxgi::{factory::DxgiAdapter, result::HResult}, + }, + dx12::{ + dcomp::DCompLib, device_creation::DeviceFactory, shader_compilation, FeatureLevel, + ShaderModel, SurfaceTarget, + }, +}; + +impl Drop for super::Adapter { + fn drop(&mut self) { + // Debug tracking alive objects + if !thread::panicking() + && self + .private_caps + .instance_flags + .contains(wgt::InstanceFlags::VALIDATION) + { + unsafe { + self.report_live_objects(); + } + } + } +} + +impl super::Adapter { + pub unsafe fn report_live_objects(&self) { + if let Ok(debug_device) = self.raw.cast::() { + unsafe { + debug_device.ReportLiveDeviceObjects( + Direct3D12::D3D12_RLDO_SUMMARY | Direct3D12::D3D12_RLDO_IGNORE_INTERNAL, + ) + } + .unwrap() + } + } + + pub fn raw_adapter(&self) -> &DxgiAdapter { + &self.raw + } + + #[allow(clippy::too_many_arguments)] + pub(super) fn expose( + adapter: DxgiAdapter, + library: &Arc, + device_factory: &Arc, + dcomp_lib: &Arc, + instance_flags: wgt::InstanceFlags, + memory_budget_thresholds: wgt::MemoryBudgetThresholds, + compiler_container: Arc, + backend_options: wgt::Dx12BackendOptions, + telemetry: Option, + ) -> Option> { + let desc = unsafe { adapter.GetDesc2() }.unwrap(); + let driver_version = unsafe { adapter.CheckInterfaceSupport(&Dxgi::IDXGIDevice::IID) }; + let driver_version = driver_version + .map(|driver_version| { + let driver_version = driver_version as u64; + [ + (driver_version >> 48) as u16, + (driver_version >> 32) as u16, + (driver_version >> 16) as u16, + driver_version as u16, + ] + }) + .map_err(|e| e.code()); + + // Create the device so that we can get the capabilities. + let res = { + profiling::scope!("ID3D12Device::create_device"); + device_factory.create_device(library, &adapter, Direct3D::D3D_FEATURE_LEVEL_11_0) + }; + if let Some(telemetry) = telemetry { + if let Err(err) = res { + (telemetry.d3d12_expose_adapter)( + &desc, + driver_version, + crate::D3D12ExposeAdapterResult::CreateDeviceError(err), + ); + } + } + let device = res.ok()?; + + profiling::scope!("feature queries"); + + // Detect the highest supported feature level. + let d3d_feature_level = [ + Direct3D::D3D_FEATURE_LEVEL_12_2, + Direct3D::D3D_FEATURE_LEVEL_12_1, + Direct3D::D3D_FEATURE_LEVEL_12_0, + Direct3D::D3D_FEATURE_LEVEL_11_1, + Direct3D::D3D_FEATURE_LEVEL_11_0, + ]; + let mut device_levels = Direct3D12::D3D12_FEATURE_DATA_FEATURE_LEVELS { + NumFeatureLevels: d3d_feature_level.len() as u32, + pFeatureLevelsRequested: d3d_feature_level.as_ptr().cast(), + MaxSupportedFeatureLevel: Default::default(), + }; + unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_FEATURE_LEVELS, + <*mut _>::cast(&mut device_levels), + size_of_val(&device_levels) as u32, + ) + } + .unwrap(); + let max_feature_level = match device_levels.MaxSupportedFeatureLevel { + Direct3D::D3D_FEATURE_LEVEL_11_0 => FeatureLevel::_11_0, + Direct3D::D3D_FEATURE_LEVEL_11_1 => FeatureLevel::_11_1, + Direct3D::D3D_FEATURE_LEVEL_12_0 => FeatureLevel::_12_0, + Direct3D::D3D_FEATURE_LEVEL_12_1 => FeatureLevel::_12_1, + Direct3D::D3D_FEATURE_LEVEL_12_2 => FeatureLevel::_12_2, + fl => { + if let Some(telemetry) = telemetry { + (telemetry.d3d12_expose_adapter)( + &desc, + driver_version, + crate::D3D12ExposeAdapterResult::UnknownFeatureLevel(fl.0), + ); + } + return None; + } + }; + + let device_name = auxil::dxgi::conv::map_adapter_name(desc.Description); + + let mut features_architecture = Direct3D12::D3D12_FEATURE_DATA_ARCHITECTURE::default(); + + unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_ARCHITECTURE, + <*mut _>::cast(&mut features_architecture), + size_of_val(&features_architecture) as u32, + ) + } + .unwrap(); + + let mut features1 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS1::default(); + let hr = unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS1, + <*mut _>::cast(&mut features1), + size_of_val(&features1) as u32, + ) + }; + + let mut workarounds = super::Workarounds::default(); + + let is_warp = device_name.contains("Microsoft Basic Render Driver"); + + // WARP uses two different versioning schemes. Versions that ship with windows + // use a version that starts with 10.x.x.x. Versions that ship from Nuget use 1.0.x.x. + // + // As far as we know, this is only an issue on the Nuget versions. + if let Ok(driver_version) = driver_version { + if is_warp && driver_version >= [1, 0, 13, 0] && driver_version[0] < 10 { + workarounds.avoid_shader_debug_info = true; + } + } + + let driver_version_string = { + let driver_version = driver_version.unwrap_or([0, 0, 0, 0]); + format!( + "{}.{}.{}.{}", + driver_version[0], driver_version[1], driver_version[2], driver_version[3] + ) + }; + + let info = wgt::AdapterInfo { + backend: wgt::Backend::Dx12, + name: device_name, + vendor: desc.VendorId, + device: desc.DeviceId, + device_type: if Dxgi::DXGI_ADAPTER_FLAG(desc.Flags as i32) + .contains(Dxgi::DXGI_ADAPTER_FLAG_SOFTWARE) + { + wgt::DeviceType::Cpu + } else if features_architecture.UMA.as_bool() { + wgt::DeviceType::IntegratedGpu + } else { + wgt::DeviceType::DiscreteGpu + }, + device_pci_bus_id: get_adapter_pci_info(desc.VendorId, desc.DeviceId), + driver: driver_version_string, + driver_info: String::new(), + subgroup_min_size: features1.WaveLaneCountMin, + subgroup_max_size: features1.WaveLaneCountMax, + transient_saves_memory: false, + }; + + let mut options = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS::default(); + unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS, + <*mut _>::cast(&mut options), + size_of_val(&options) as u32, + ) + } + .unwrap(); + + /// Resource Binding Tiers: https://learn.microsoft.com/en-us/windows/win32/direct3d12/hardware-support#limits-dependant-on-hardware + #[derive(PartialEq, Eq, PartialOrd, Ord)] + enum ResourceBindingTier { + T1, + T2, + T3, + } + let rbt = match options.ResourceBindingTier { + Direct3D12::D3D12_RESOURCE_BINDING_TIER_1 => ResourceBindingTier::T1, + Direct3D12::D3D12_RESOURCE_BINDING_TIER_2 => ResourceBindingTier::T2, + tier if tier.0 >= Direct3D12::D3D12_RESOURCE_BINDING_TIER_3.0 => { + ResourceBindingTier::T3 + } + other => { + log::debug!("Got zero or negative value for resource binding tier {other:?}"); + ResourceBindingTier::T1 + } + }; + + if rbt == ResourceBindingTier::T1 { + if let Some(telemetry) = telemetry { + (telemetry.d3d12_expose_adapter)( + &desc, + driver_version, + crate::D3D12ExposeAdapterResult::ResourceBindingTier2Requirement, + ); + } + // We require Tier 2 or higher for the ability to make samplers bindless in all cases. + return None; + } + + let _depth_bounds_test_supported = { + let mut features2 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS2::default(); + unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS2, + <*mut _>::cast(&mut features2), + size_of_val(&features2) as u32, + ) + } + .is_ok() + && features2.DepthBoundsTestSupported.as_bool() + }; + + let (casting_fully_typed_format_supported, view_instancing) = { + let mut features3 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS3::default(); + if unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS3, + <*mut _>::cast(&mut features3), + size_of_val(&features3) as u32, + ) + } + .is_ok() + { + ( + features3.CastingFullyTypedFormatSupported.as_bool(), + features3.ViewInstancingTier.0 >= Direct3D12::D3D12_VIEW_INSTANCING_TIER_1.0, + ) + } else { + (false, false) + } + }; + + let heap_create_not_zeroed = { + // For D3D12_HEAP_FLAG_CREATE_NOT_ZEROED we just need to + // make sure that options7 can be queried. See also: + // https://devblogs.microsoft.com/directx/coming-to-directx-12-more-control-over-memory-allocation/ + let mut features7 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS7::default(); + unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS7, + <*mut _>::cast(&mut features7), + size_of_val(&features7) as u32, + ) + } + .is_ok() + }; + + let unrestricted_buffer_texture_copy_pitch_supported = { + let mut features13 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS13::default(); + unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS13, + <*mut _>::cast(&mut features13), + size_of_val(&features13) as u32, + ) + } + .is_ok() + && features13 + .UnrestrictedBufferTextureCopyPitchSupported + .as_bool() + }; + + let mut max_sampler_descriptor_heap_size = + Direct3D12::D3D12_MAX_SHADER_VISIBLE_SAMPLER_HEAP_SIZE; + { + let mut features19 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS19::default(); + let res = unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS19, + <*mut _>::cast(&mut features19), + size_of_val(&features19) as u32, + ) + }; + + // Sometimes on Windows 11 23H2, the function returns success, even though the runtime + // does not know about `Options19`. This can cause this number to be 0 as the structure isn't written to. + // This value is nonsense and creating zero-sized sampler heaps can cause drivers to explode. + // As as we're guaranteed 2048 anyway, we make sure this value is not under 2048. + // + // https://github.com/gfx-rs/wgpu/issues/7053 + let is_ok = res.is_ok(); + let is_above_minimum = features19.MaxSamplerDescriptorHeapSize + > Direct3D12::D3D12_MAX_SHADER_VISIBLE_SAMPLER_HEAP_SIZE; + if is_ok && is_above_minimum { + max_sampler_descriptor_heap_size = features19.MaxSamplerDescriptorHeapSize; + } + }; + + let mut shader_models_after_5_1 = [ + Direct3D12::D3D_SHADER_MODEL_6_9, + Direct3D12::D3D_SHADER_MODEL_6_8, + Direct3D12::D3D_SHADER_MODEL_6_7, + Direct3D12::D3D_SHADER_MODEL_6_6, + Direct3D12::D3D_SHADER_MODEL_6_5, + Direct3D12::D3D_SHADER_MODEL_6_4, + Direct3D12::D3D_SHADER_MODEL_6_3, + Direct3D12::D3D_SHADER_MODEL_6_2, + Direct3D12::D3D_SHADER_MODEL_6_1, + Direct3D12::D3D_SHADER_MODEL_6_0, + ] + .iter(); + let max_device_shader_model = loop { + if let Some(&sm) = shader_models_after_5_1.next() { + let mut sm = Direct3D12::D3D12_FEATURE_DATA_SHADER_MODEL { + HighestShaderModel: sm, + }; + if unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_SHADER_MODEL, + <*mut _>::cast(&mut sm), + size_of_val(&sm) as u32, + ) + } + .is_ok() + { + break match sm.HighestShaderModel { + Direct3D12::D3D_SHADER_MODEL_5_1 => ShaderModel::_5_1, + Direct3D12::D3D_SHADER_MODEL_6_0 => ShaderModel::_6_0, + Direct3D12::D3D_SHADER_MODEL_6_1 => ShaderModel::_6_1, + Direct3D12::D3D_SHADER_MODEL_6_2 => ShaderModel::_6_2, + Direct3D12::D3D_SHADER_MODEL_6_3 => ShaderModel::_6_3, + Direct3D12::D3D_SHADER_MODEL_6_4 => ShaderModel::_6_4, + Direct3D12::D3D_SHADER_MODEL_6_5 => ShaderModel::_6_5, + Direct3D12::D3D_SHADER_MODEL_6_6 => ShaderModel::_6_6, + Direct3D12::D3D_SHADER_MODEL_6_7 => ShaderModel::_6_7, + Direct3D12::D3D_SHADER_MODEL_6_8 => ShaderModel::_6_8, + Direct3D12::D3D_SHADER_MODEL_6_9 => ShaderModel::_6_9, + _ => unreachable!(), + }; + } + } else { + break ShaderModel::_5_1; + } + }; + + let wgt_shader_model = backend_options + .force_shader_model + .get() + .or(compiler_container.max_shader_model()); + + let shader_model = if let Some(max_shader_model) = wgt_shader_model { + let max_dxc_shader_model = match max_shader_model { + wgt::DxcShaderModel::V6_0 => ShaderModel::_6_0, + wgt::DxcShaderModel::V6_1 => ShaderModel::_6_1, + wgt::DxcShaderModel::V6_2 => ShaderModel::_6_2, + wgt::DxcShaderModel::V6_3 => ShaderModel::_6_3, + wgt::DxcShaderModel::V6_4 => ShaderModel::_6_4, + wgt::DxcShaderModel::V6_5 => ShaderModel::_6_5, + wgt::DxcShaderModel::V6_6 => ShaderModel::_6_6, + wgt::DxcShaderModel::V6_7 => ShaderModel::_6_7, + wgt::DxcShaderModel::V6_8 => ShaderModel::_6_8, + wgt::DxcShaderModel::V6_9 => ShaderModel::_6_9, + }; + + let shader_model = max_device_shader_model.min(max_dxc_shader_model); + + match shader_model { + ShaderModel::_5_1 => { + if let Some(telemetry) = telemetry { + (telemetry.d3d12_expose_adapter)( + &desc, + driver_version, + crate::D3D12ExposeAdapterResult::ShaderModel6Requirement, + ); + } + // don't expose this adapter if it doesn't support DXIL + return None; + } + ShaderModel::_6_0 => naga::back::hlsl::ShaderModel::V6_0, + ShaderModel::_6_1 => naga::back::hlsl::ShaderModel::V6_1, + ShaderModel::_6_2 => naga::back::hlsl::ShaderModel::V6_2, + ShaderModel::_6_3 => naga::back::hlsl::ShaderModel::V6_3, + ShaderModel::_6_4 => naga::back::hlsl::ShaderModel::V6_4, + ShaderModel::_6_5 => naga::back::hlsl::ShaderModel::V6_5, + ShaderModel::_6_6 => naga::back::hlsl::ShaderModel::V6_6, + ShaderModel::_6_7 => naga::back::hlsl::ShaderModel::V6_7, + ShaderModel::_6_8 => naga::back::hlsl::ShaderModel::V6_8, + ShaderModel::_6_9 => naga::back::hlsl::ShaderModel::V6_9, + } + } else { + naga::back::hlsl::ShaderModel::V5_1 + }; + let private_caps = super::PrivateCapabilities { + instance_flags, + workarounds, + heterogeneous_resource_heaps: options.ResourceHeapTier + != Direct3D12::D3D12_RESOURCE_HEAP_TIER_1, + memory_architecture: if features_architecture.UMA.as_bool() { + super::MemoryArchitecture::Unified { + cache_coherent: features_architecture.CacheCoherentUMA.as_bool(), + } + } else { + super::MemoryArchitecture::NonUnified + }, + heap_create_not_zeroed, + casting_fully_typed_format_supported, + // See https://github.com/gfx-rs/wgpu/issues/3552 + suballocation_supported: !info.name.contains("Iris(R) Xe"), + shader_model, + max_sampler_descriptor_heap_size, + unrestricted_buffer_texture_copy_pitch_supported, + }; + + // these should always be available on d3d12 + let mut features = wgt::Features::empty() + | wgt::Features::DEPTH_CLIP_CONTROL + | wgt::Features::DEPTH32FLOAT_STENCIL8 + | wgt::Features::INDIRECT_FIRST_INSTANCE + | wgt::Features::MAPPABLE_PRIMARY_BUFFERS + | wgt::Features::MULTI_DRAW_INDIRECT_COUNT + | wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER + | wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO + | wgt::Features::POLYGON_MODE_LINE + | wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES + | wgt::Features::TIMESTAMP_QUERY + | wgt::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS + | wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES + | wgt::Features::TEXTURE_COMPRESSION_BC + | wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D + | wgt::Features::CLEAR_TEXTURE + | wgt::Features::TEXTURE_FORMAT_16BIT_NORM + | wgt::Features::IMMEDIATES + | wgt::Features::PRIMITIVE_INDEX + | wgt::Features::RG11B10UFLOAT_RENDERABLE + | wgt::Features::DUAL_SOURCE_BLENDING + | wgt::Features::TEXTURE_FORMAT_NV12 + | wgt::Features::FLOAT32_FILTERABLE + | wgt::Features::TEXTURE_ATOMIC + | wgt::Features::PASSTHROUGH_SHADERS + | wgt::Features::EXTERNAL_TEXTURE + | wgt::Features::MEMORY_DECORATION_COHERENT; + + //TODO: in order to expose this, we need to run a compute shader + // that extract the necessary statistics out of the D3D12 result. + // Alternatively, we could allocate a buffer for the query set, + // write the results there, and issue a bunch of copy commands. + //| wgt::Features::PIPELINE_STATISTICS_QUERY + + if max_feature_level >= FeatureLevel::_11_1 { + features |= wgt::Features::VERTEX_WRITABLE_STORAGE; + } + + features.set( + wgt::Features::CONSERVATIVE_RASTERIZATION, + options.ConservativeRasterizationTier + != Direct3D12::D3D12_CONSERVATIVE_RASTERIZATION_TIER_NOT_SUPPORTED, + ); + + features.set( + wgt::Features::TEXTURE_BINDING_ARRAY + | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY + | wgt::Features::STORAGE_TEXTURE_ARRAY_NON_UNIFORM_INDEXING + | wgt::Features::SAMPLED_TEXTURE_AND_STORAGE_BUFFER_ARRAY_NON_UNIFORM_INDEXING + // See note below the table https://learn.microsoft.com/en-us/windows/win32/direct3d12/hardware-support + | wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY, + shader_model >= naga::back::hlsl::ShaderModel::V5_1 && rbt >= ResourceBindingTier::T3, + ); + + let bgra8unorm_storage_supported = { + let mut bgra8unorm_info = Direct3D12::D3D12_FEATURE_DATA_FORMAT_SUPPORT { + Format: Dxgi::Common::DXGI_FORMAT_B8G8R8A8_UNORM, + ..Default::default() + }; + let hr = unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_FORMAT_SUPPORT, + <*mut _>::cast(&mut bgra8unorm_info), + size_of_val(&bgra8unorm_info) as u32, + ) + }; + hr.is_ok() + && bgra8unorm_info + .Support2 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT2_UAV_TYPED_STORE) + }; + features.set( + wgt::Features::BGRA8UNORM_STORAGE, + bgra8unorm_storage_supported, + ); + + let p010_format_supported = { + let mut p010_info = Direct3D12::D3D12_FEATURE_DATA_FORMAT_SUPPORT { + Format: Dxgi::Common::DXGI_FORMAT_P010, + ..Default::default() + }; + let hr = unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_FORMAT_SUPPORT, + <*mut _>::cast(&mut p010_info), + size_of_val(&p010_info) as u32, + ) + }; + if hr.is_ok() { + let supports_texture2d = p010_info + .Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_TEXTURE2D); + let supports_shader_load = p010_info + .Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_SHADER_LOAD); + let supports_shader_sample = p010_info + .Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_SHADER_SAMPLE); + supports_texture2d && supports_shader_load && supports_shader_sample + } else { + false + } + }; + features.set(wgt::Features::TEXTURE_FORMAT_P010, p010_format_supported); + + features.set( + wgt::Features::SHADER_INT64, + shader_model >= naga::back::hlsl::ShaderModel::V6_0 + && hr.is_ok() + && features1.Int64ShaderOps.as_bool(), + ); + + let float16_supported = { + let mut features4 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS4::default(); + let hr = unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS4, // https://learn.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_feature#syntax + ptr::from_mut(&mut features4).cast(), + size_of::() as _, + ) + }; + hr.is_ok() && features4.Native16BitShaderOpsSupported.as_bool() + }; + + features.set( + wgt::Features::SHADER_F16, + shader_model >= naga::back::hlsl::ShaderModel::V6_2 && float16_supported, + ); + + features.set( + wgt::Features::SUBGROUP, + shader_model >= naga::back::hlsl::ShaderModel::V6_0 + && hr.is_ok() + && features1.WaveOps.as_bool(), + ); + let mut features5 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS5::default(); + let has_features5 = unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS5, + <*mut _>::cast(&mut features5), + size_of_val(&features5) as u32, + ) + } + .is_ok(); + + // Once ray tracing pipelines are supported they also will go here + let supports_ray_tracing = features5.RaytracingTier.0 + >= Direct3D12::D3D12_RAYTRACING_TIER_1_1.0 + && shader_model >= naga::back::hlsl::ShaderModel::V6_5 + && has_features5; + + features.set( + wgt::Features::EXPERIMENTAL_RAY_QUERY + | wgt::Features::EXTENDED_ACCELERATION_STRUCTURE_VERTEX_FORMATS, + supports_ray_tracing, + ); + + // Binding arrays of TLAS are supported on D3D12 when ray tracing is supported. + // + // This flag is used for shader-side `binding_array` as well as + // allowing `BindGroupLayoutEntry::count = Some(...)` for `BindingType::AccelerationStructure`. + features.set( + wgt::Features::ACCELERATION_STRUCTURE_BINDING_ARRAY, + supports_ray_tracing, + ); + + // Check for Int64 atomic support on buffers. This is very convoluted, but is based on a conservative reading + // of https://microsoft.github.io/DirectX-Specs/d3d/HLSL_SM_6_6_Int64_and_Float_Atomics.html#integer-64-bit-capabilities. + let atomic_int64_buffers; + let atomic_int64_textures; + { + let mut features9 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS9::default(); + let hr9 = unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS9, + <*mut _>::cast(&mut features9), + size_of_val(&features9) as u32, + ) + } + .is_ok(); + + let mut features11 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS11::default(); + let hr11 = unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS11, + <*mut _>::cast(&mut features11), + size_of_val(&features11) as u32, + ) + } + .is_ok(); + + atomic_int64_buffers = hr9 && hr11 && hr.is_ok() + // Int64 atomics show up in SM6.6. + && shader_model >= naga::back::hlsl::ShaderModel::V6_6 + // They require Int64 to be available in the shader at all. + && features1.Int64ShaderOps.as_bool() + // As our RWByteAddressBuffers can exist on both descriptor heaps and + // as root descriptors, we need to ensure that both cases are supported. + // base SM6.6 only guarantees Int64 atomics on resources in root descriptors. + && features11.AtomicInt64OnDescriptorHeapResourceSupported.as_bool() + // Our Int64 atomic caps currently require groupshared. This + // prevents Intel or Qcomm from using Int64 currently. + // https://github.com/gfx-rs/wgpu/issues/8666 + && features9.AtomicInt64OnGroupSharedSupported.as_bool(); + + atomic_int64_textures = hr9 && hr11 && hr.is_ok() + // Int64 atomics show up in SM6.6. + && shader_model >= naga::back::hlsl::ShaderModel::V6_6 + // They require Int64 to be available in the shader at all. + && features1.Int64ShaderOps.as_bool() + // Textures are typed resources, so we need this flag. + && features9.AtomicInt64OnTypedResourceSupported.as_bool() + // As textures can only exist in descriptor heaps, we require this. + // However, all architectures that support atomics on typed resources + // support this as well, so this is somewhat redundant. + && features11.AtomicInt64OnDescriptorHeapResourceSupported.as_bool(); + }; + features.set( + wgt::Features::SHADER_INT64_ATOMIC_ALL_OPS | wgt::Features::SHADER_INT64_ATOMIC_MIN_MAX, + atomic_int64_buffers, + ); + features.set(wgt::Features::TEXTURE_INT64_ATOMIC, atomic_int64_textures); + let mesh_shader_supported = { + let mut features7 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS7::default(); + unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS7, + <*mut _>::cast(&mut features7), + size_of_val(&features7) as u32, + ) + } + .is_ok() + && features7.MeshShaderTier != Direct3D12::D3D12_MESH_SHADER_TIER_NOT_SUPPORTED + }; + features.set( + wgt::Features::EXPERIMENTAL_MESH_SHADER, + mesh_shader_supported, + ); + let shader_barycentrics_supported = { + let mut features3 = Direct3D12::D3D12_FEATURE_DATA_D3D12_OPTIONS3::default(); + unsafe { + device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_D3D12_OPTIONS3, + <*mut _>::cast(&mut features3), + size_of_val(&features3) as u32, + ) + } + .is_ok() + && features3.BarycentricsSupported.as_bool() + && shader_model >= naga::back::hlsl::ShaderModel::V6_1 + }; + features.set( + wgt::Features::SHADER_BARYCENTRICS, + shader_barycentrics_supported, + ); + + features.set( + wgt::Features::MULTIVIEW, + view_instancing && shader_model >= naga::back::hlsl::ShaderModel::V6_1, + ); + features.set( + wgt::Features::SELECTIVE_MULTIVIEW, + view_instancing && shader_model >= naga::back::hlsl::ShaderModel::V6_1, + ); + + features.set( + wgt::Features::EXPERIMENTAL_MESH_SHADER_MULTIVIEW, + mesh_shader_supported + && view_instancing + && shader_model >= naga::back::hlsl::ShaderModel::V6_1, + ); + + // TODO: Determine if IPresentationManager is supported + let presentation_timer = auxil::dxgi::time::PresentationTimer::new_dxgi(); + + let downlevel = wgt::DownlevelCapabilities::default(); + + // Limits that must share D3D12's root signature size of + // D3D12_MAX_ROOT_COST 64 DWORDS (256 bytes). + // + // Root constants and root tables use 1 DWORD. + // Root descriptors use 2 DWORDs. + // Source: https://learn.microsoft.com/en-us/windows/win32/direct3d12/root-signature-limits#memory-limits-and-costs + // + // Per pipeline layout: + // - RootElement::Constant, (immediates) 32 root constants + // (bounded by maxImmediateSize) = 32 x 4 bytes = 128 bytes + // - RootElement::SamplerHeap, a root table = 4 bytes + // - RootElement::SpecialConstantBuffer, 3 root constants = 3 x 4 bytes = 12 bytes + // - RootElement::DynamicOffsetsBuffer, a root constant per dynamic storage buffer + // (bounded by maxDynamicStorageBuffersPerPipelineLayout) = 4 x 4 bytes = 16 bytes + // - RootElement::DynamicUniformBuffer, a root descriptor per dynamic uniform buffer + // (bounded by maxDynamicUniformBuffersPerPipelineLayout) = 8 x 8 bytes = 64 bytes + // Per bind group: + // - RootElement::Table, a root table + // (bounded by maxBindGroups) = 8 x 4 bytes = 32 bytes + // + // Source: logic in `create_pipeline_layout` + // + // Total: 128 + 4 + 12 + 16 + 64 + 32 = 256 bytes + // + let max_immediate_size = 128; + let max_bind_groups = 8; + let max_dynamic_uniform_buffers_per_pipeline_layout = 8; + let max_dynamic_storage_buffers_per_pipeline_layout = 4; + + // "Maximum number of descriptors in a Constant Buffer View (CBV), Shader Resource View (SRV), or Unordered Access View(UAV) heap used for rendering" + let full_heap_count = match rbt { + ResourceBindingTier::T1 | ResourceBindingTier::T2 => 1_000_000, + // 1_000_000+ + ResourceBindingTier::T3 => { + // Theoretically vram limited, but in practice 2^20 is the limit + 1 << 20 + } + }; + + // "Maximum number of Constant Buffer Views in all descriptor tables per shader stage" + let max_uniform_buffers_per_shader_stage = match rbt { + ResourceBindingTier::T1 | ResourceBindingTier::T2 => 14, + _ => full_heap_count, + }; + + // "Maximum number of Shader Resource Views in all descriptor tables per shader stage" + let mut max_srv_per_shader_stage = match rbt { + ResourceBindingTier::T1 => 128, + _ => full_heap_count, + }; + + // We use an extra SRV for all samplers in a bind group. + // See comment in `create_pipeline_layout`. + max_srv_per_shader_stage -= max_bind_groups; + + // If we also support acceleration structures these are shared so we must halve it. + // It's unlikely that this affects anything because most devices that support ray tracing + // probably have a higher binding tier than one. + let mut max_sampled_textures_per_shader_stage = if supports_ray_tracing { + max_srv_per_shader_stage / 2 + } else { + max_srv_per_shader_stage + }; + let mut max_acceleration_structures_per_shader_stage = if supports_ray_tracing { + max_srv_per_shader_stage / 2 + } else { + 0 + }; + + // "Maximum number of Unordered Access Views in all descriptor tables across all stages" + let max_uav_across_all_stages = match rbt { + ResourceBindingTier::T1 => match max_feature_level { + FeatureLevel::_11_0 => 8, + _ => 64, + }, + ResourceBindingTier::T2 => 64, + ResourceBindingTier::T3 => full_heap_count, + }; + const MAX_SHADER_STAGES_PER_PIPELINE: u32 = 2; + // We must share the UAV limit across both storage resource limits. + let max_uav_per_shader_stage = max_uav_across_all_stages / MAX_SHADER_STAGES_PER_PIPELINE; + let max_storage_textures_per_shader_stage = max_uav_per_shader_stage / 2; + let mut max_storage_buffers_per_shader_stage = max_uav_per_shader_stage / 2; + + // WebGPU storage buffers count as 1 SRV if they are read-only + // or as 1 UAV if they are read-write. See comment in + // `create_pipeline_layout`. Make sure we don't exceed + // the maximum number of SRVs for the relevant limits. + auxil::cap_limits_to_be_under_the_sum_limit( + [ + &mut max_sampled_textures_per_shader_stage, + &mut max_acceleration_structures_per_shader_stage, + &mut max_storage_buffers_per_shader_stage, + ], + max_srv_per_shader_stage, + ); + + // "Maximum number of Samplers in all descriptor tables per shader stage" + let max_samplers_per_shader_stage = match rbt { + ResourceBindingTier::T1 => 16, + _ => 2048, + }; + + // See https://microsoft.github.io/DirectX-Specs/d3d/ViewInstancing.html#maximum-viewinstancecount + let max_multiview_view_count = if view_instancing { 4 } else { 0 }; + + if let Some(telemetry) = telemetry { + (telemetry.d3d12_expose_adapter)( + &desc, + driver_version, + crate::D3D12ExposeAdapterResult::Success( + max_feature_level, + max_device_shader_model, + ), + ); + } + + Some(crate::ExposedAdapter { + adapter: super::Adapter { + raw: adapter, + device, + library: Arc::clone(library), + dcomp_lib: Arc::clone(dcomp_lib), + private_caps, + presentation_timer, + memory_budget_thresholds, + compiler_container, + options: backend_options, + }, + info, + features, + capabilities: crate::Capabilities { + limits: auxil::adjust_raw_limits(wgt::Limits { + // + // WebGPU LIMITS: + // Based on https://gpuweb.github.io/gpuweb/correspondence/#limits + // + // 16384 + max_texture_dimension_1d: Direct3D12::D3D12_REQ_TEXTURE1D_U_DIMENSION, + // 16384 + max_texture_dimension_2d: Direct3D12::D3D12_REQ_TEXTURE2D_U_OR_V_DIMENSION + .min(Direct3D12::D3D12_REQ_TEXTURECUBE_DIMENSION), + // 2048 + max_texture_dimension_3d: Direct3D12::D3D12_REQ_TEXTURE3D_U_V_OR_W_DIMENSION, + // 2048 + max_texture_array_layers: Direct3D12::D3D12_REQ_TEXTURE2D_ARRAY_AXIS_DIMENSION, + // No real limit. + max_bindings_per_bind_group: u32::MAX, + max_sampled_textures_per_shader_stage, + max_samplers_per_shader_stage, + max_storage_textures_per_shader_stage, + max_storage_buffers_per_shader_stage, + max_uniform_buffers_per_shader_stage, + // See `InputSlot` param docs: https://learn.microsoft.com/en-ca/windows/win32/api/d3d12/ns-d3d12-d3d12_input_element_desc + max_vertex_buffers: 16, + // Dx12 does not expose a maximum buffer size in the API. + // This limit is chosen to avoid potential issues with drivers should they internally + // store buffer sizes using 32 bit ints (a situation we have already encountered with vulkan). + max_buffer_size: i32::MAX as u64, + max_storage_buffer_binding_size: auxil::MAX_I32_BINDING_SIZE as u64, + // 65536 + max_uniform_buffer_binding_size: + Direct3D12::D3D12_REQ_CONSTANT_BUFFER_ELEMENT_COUNT as u64 * 16, + // 254 + min_uniform_buffer_offset_alignment: + Direct3D12::D3D12_CONSTANT_BUFFER_DATA_PLACEMENT_ALIGNMENT, + // 16 + min_storage_buffer_offset_alignment: + Direct3D12::D3D12_RAW_UAV_SRV_BYTE_ALIGNMENT, + // 32 + max_vertex_attributes: Direct3D12::D3D12_IA_VERTEX_INPUT_RESOURCE_SLOT_COUNT, + // 2048 + max_vertex_buffer_array_stride: Direct3D12::D3D12_SO_BUFFER_MAX_STRIDE_IN_BYTES, + // 31 + max_inter_stage_shader_variables: Direct3D12::D3D12_VS_OUTPUT_REGISTER_COUNT + .min(Direct3D12::D3D12_PS_INPUT_REGISTER_COUNT) + - 1, // - 1 for position + max_immediate_size, + max_bind_groups, + max_dynamic_uniform_buffers_per_pipeline_layout, + max_dynamic_storage_buffers_per_pipeline_layout, + // 8 + max_color_attachments: Direct3D12::D3D12_SIMULTANEOUS_RENDER_TARGET_COUNT, + // 128 (No documented limit) + max_color_attachment_bytes_per_sample: + Direct3D12::D3D12_SIMULTANEOUS_RENDER_TARGET_COUNT + * wgt::TextureFormat::MAX_TARGET_PIXEL_BYTE_COST, + // From: https://microsoft.github.io/DirectX-Specs/d3d/archive/D3D11_3_FunctionalSpec.htm#18.6.6%20Inter-Thread%20Data%20Sharing + max_compute_workgroup_storage_size: 32768, + // 1024 + max_compute_invocations_per_workgroup: + Direct3D12::D3D12_CS_THREAD_GROUP_MAX_THREADS_PER_GROUP, + // 1024 + max_compute_workgroup_size_x: Direct3D12::D3D12_CS_THREAD_GROUP_MAX_X, + // 1024 + max_compute_workgroup_size_y: Direct3D12::D3D12_CS_THREAD_GROUP_MAX_Y, + // 64 + max_compute_workgroup_size_z: Direct3D12::D3D12_CS_THREAD_GROUP_MAX_Z, + // 65535 + max_compute_workgroups_per_dimension: + Direct3D12::D3D12_CS_DISPATCH_MAX_THREAD_GROUPS_PER_DIMENSION, + // + // NATIVE (Non-WebGPU) LIMITS: + // + max_non_sampler_bindings: 1_000_000, + + max_binding_array_elements_per_shader_stage: full_heap_count, + max_binding_array_sampler_elements_per_shader_stage: + Direct3D12::D3D12_MAX_SHADER_VISIBLE_SAMPLER_HEAP_SIZE, + + // Source: https://microsoft.github.io/DirectX-Specs/d3d/MeshShader.html#dispatchmesh-api + max_task_mesh_workgroup_total_count: if mesh_shader_supported { + 2u32.pow(22) + } else { + 0 + }, + // Technically it says "64k" but I highly doubt they want 65536 for compute and exactly 64,000 for task workgroups + max_task_mesh_workgroups_per_dimension: if mesh_shader_supported { + Direct3D12::D3D12_CS_DISPATCH_MAX_THREAD_GROUPS_PER_DIMENSION + } else { + 0 + }, + // Assume this inherits from compute shaders + max_task_invocations_per_workgroup: if mesh_shader_supported { + Direct3D12::D3D12_CS_4_X_THREAD_GROUP_MAX_THREADS_PER_GROUP + } else { + 0 + }, + max_task_invocations_per_dimension: if mesh_shader_supported { + Direct3D12::D3D12_CS_THREAD_GROUP_MAX_Z + } else { + 0 + }, + // Source: https://microsoft.github.io/DirectX-Specs/d3d/MeshShader.html#amplification-shader-and-mesh-shader + max_mesh_invocations_per_workgroup: if mesh_shader_supported { 128 } else { 0 }, + max_mesh_invocations_per_dimension: if mesh_shader_supported { 128 } else { 0 }, + + max_task_payload_size: if mesh_shader_supported { 16384 } else { 0 }, + max_mesh_output_vertices: if mesh_shader_supported { 256 } else { 0 }, + max_mesh_output_primitives: if mesh_shader_supported { 256 } else { 0 }, + // Source: https://microsoft.github.io/DirectX-Specs/d3d/MeshShader.html#sv_rendertargetarrayindex-limitations-based-on-queryable-capability + max_mesh_output_layers: if mesh_shader_supported { 8 } else { 0 }, + max_mesh_multiview_view_count: if mesh_shader_supported { + max_multiview_view_count + } else { + 0 + }, + + max_blas_primitive_count: if supports_ray_tracing { + 1 << 29 // 2^29 + } else { + 0 + }, + max_blas_geometry_count: if supports_ray_tracing { + 1 << 24 // 2^24 + } else { + 0 + }, + max_tlas_instance_count: if supports_ray_tracing { + 1 << 24 // 2^24 + } else { + 0 + }, + max_acceleration_structures_per_shader_stage, + max_binding_array_acceleration_structure_elements_per_shader_stage: + max_acceleration_structures_per_shader_stage, + max_multiview_view_count, + }), + alignments: crate::Alignments { + buffer_copy_offset: wgt::BufferSize::new( + Direct3D12::D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT as u64, + ) + .unwrap(), + buffer_copy_pitch: wgt::BufferSize::new( + Direct3D12::D3D12_TEXTURE_DATA_PITCH_ALIGNMENT as u64, + ) + .unwrap(), + // Direct3D correctly bounds-checks all array accesses: + // https://microsoft.github.io/DirectX-Specs/d3d/archive/D3D11_3_FunctionalSpec.htm#18.6.8.2%20Device%20Memory%20Reads + uniform_bounds_check_alignment: wgt::BufferSize::new(1).unwrap(), + raw_tlas_instance_size: size_of::(), + ray_tracing_scratch_buffer_alignment: + Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_BYTE_ALIGNMENT, + }, + downlevel, + cooperative_matrix_properties: Vec::new(), + }, + }) + } +} + +impl crate::Adapter for super::Adapter { + type A = super::Api; + + unsafe fn open( + &self, + features: wgt::Features, + limits: &wgt::Limits, + memory_hints: &wgt::MemoryHints, + ) -> Result, crate::DeviceError> { + let queue: Direct3D12::ID3D12CommandQueue = { + profiling::scope!("ID3D12Device::CreateCommandQueue"); + unsafe { + self.device + .CreateCommandQueue(&Direct3D12::D3D12_COMMAND_QUEUE_DESC { + Type: Direct3D12::D3D12_COMMAND_LIST_TYPE_DIRECT, + Priority: Direct3D12::D3D12_COMMAND_QUEUE_PRIORITY_NORMAL.0, + Flags: Direct3D12::D3D12_COMMAND_QUEUE_FLAG_NONE, + NodeMask: 0, + }) + } + .into_device_result("Queue creation")? + }; + + let device = super::Device::new( + self.raw.clone(), + self.device.clone(), + queue.clone(), + features, + limits, + memory_hints, + self.private_caps, + &self.library, + &self.dcomp_lib, + self.memory_budget_thresholds, + self.compiler_container.clone(), + self.options.clone(), + )?; + Ok(crate::OpenDevice { + device, + queue: super::Queue { + raw: queue, + temp_lists: Mutex::new(Vec::new()), + }, + }) + } + + unsafe fn texture_format_capabilities( + &self, + format: wgt::TextureFormat, + ) -> crate::TextureFormatCapabilities { + use crate::TextureFormatCapabilities as Tfc; + + let raw_format = match auxil::dxgi::conv::map_texture_format_failable(format) { + Some(f) => f, + None => return Tfc::empty(), + }; + let srv_uav_format = if format.is_combined_depth_stencil_format() { + auxil::dxgi::conv::map_texture_format_for_srv_uav( + format, + // use the depth aspect here as opposed to stencil since it has more capabilities + crate::FormatAspects::DEPTH, + ) + } else { + auxil::dxgi::conv::map_texture_format_for_srv_uav( + format, + crate::FormatAspects::from(format), + ) + } + .unwrap(); + + let mut data = Direct3D12::D3D12_FEATURE_DATA_FORMAT_SUPPORT { + Format: raw_format, + ..Default::default() + }; + unsafe { + self.device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_FORMAT_SUPPORT, + <*mut _>::cast(&mut data), + size_of_val(&data) as u32, + ) + } + .unwrap(); + + // Because we use a different format for SRV and UAV views of depth textures, we need to check + // the features that use SRV/UAVs using the no-depth format. + let mut data_srv_uav = Direct3D12::D3D12_FEATURE_DATA_FORMAT_SUPPORT { + Format: srv_uav_format, + Support1: Direct3D12::D3D12_FORMAT_SUPPORT1_NONE, + Support2: Direct3D12::D3D12_FORMAT_SUPPORT2_NONE, + }; + if raw_format != srv_uav_format { + // Only-recheck if we're using a different format + unsafe { + self.device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_FORMAT_SUPPORT, + ptr::addr_of_mut!(data_srv_uav).cast(), + size_of::() as u32, + ) + } + .unwrap(); + } else { + // Same format, just copy over. + data_srv_uav = data; + } + + let mut caps = Tfc::COPY_SRC | Tfc::COPY_DST; + // Cannot use the contains() helper, and windows-rs doesn't provide a .intersect() helper + let is_texture = (data.Support1 + & (Direct3D12::D3D12_FORMAT_SUPPORT1_TEXTURE1D + | Direct3D12::D3D12_FORMAT_SUPPORT1_TEXTURE2D + | Direct3D12::D3D12_FORMAT_SUPPORT1_TEXTURE3D + | Direct3D12::D3D12_FORMAT_SUPPORT1_TEXTURECUBE)) + .0 + != 0; + // SRVs use srv_uav_format + caps.set( + Tfc::SAMPLED, + is_texture + && data_srv_uav + .Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_SHADER_LOAD), + ); + caps.set( + Tfc::SAMPLED_LINEAR, + data_srv_uav + .Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_SHADER_SAMPLE), + ); + caps.set( + Tfc::COLOR_ATTACHMENT, + data.Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_RENDER_TARGET), + ); + caps.set( + Tfc::COLOR_ATTACHMENT_BLEND, + data.Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_BLENDABLE), + ); + caps.set( + Tfc::DEPTH_STENCIL_ATTACHMENT, + data.Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_DEPTH_STENCIL), + ); + // UAVs use srv_uav_format + caps.set( + Tfc::STORAGE_READ_ONLY, + data_srv_uav + .Support2 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT2_UAV_TYPED_LOAD), + ); + caps.set( + Tfc::STORAGE_ATOMIC, + data_srv_uav + .Support2 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT2_UAV_ATOMIC_UNSIGNED_MIN_OR_MAX), + ); + caps.set( + Tfc::STORAGE_WRITE_ONLY, + data_srv_uav + .Support2 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT2_UAV_TYPED_STORE), + ); + caps.set( + Tfc::STORAGE_READ_WRITE, + caps.contains(Tfc::STORAGE_READ_ONLY | Tfc::STORAGE_WRITE_ONLY), + ); + + // We load via UAV/SRV so use srv_uav_format + let no_msaa_load = caps.contains(Tfc::SAMPLED) + && !data_srv_uav + .Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_MULTISAMPLE_LOAD); + + let no_msaa_target = (data.Support1 + & (Direct3D12::D3D12_FORMAT_SUPPORT1_RENDER_TARGET + | Direct3D12::D3D12_FORMAT_SUPPORT1_DEPTH_STENCIL)) + .0 + != 0 + && !data + .Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_MULTISAMPLE_RENDERTARGET); + + caps.set( + Tfc::MULTISAMPLE_RESOLVE, + data.Support1 + .contains(Direct3D12::D3D12_FORMAT_SUPPORT1_MULTISAMPLE_RESOLVE), + ); + + let mut ms_levels = Direct3D12::D3D12_FEATURE_DATA_MULTISAMPLE_QUALITY_LEVELS { + Format: raw_format, + SampleCount: 0, + Flags: Direct3D12::D3D12_MULTISAMPLE_QUALITY_LEVELS_FLAG_NONE, + NumQualityLevels: 0, + }; + + let mut set_sample_count = |sc: u32, tfc: Tfc| { + ms_levels.SampleCount = sc; + + if unsafe { + self.device.CheckFeatureSupport( + Direct3D12::D3D12_FEATURE_MULTISAMPLE_QUALITY_LEVELS, + <*mut _>::cast(&mut ms_levels), + size_of_val(&ms_levels) as u32, + ) + } + .is_ok() + && ms_levels.NumQualityLevels != 0 + { + caps.set(tfc, !no_msaa_load && !no_msaa_target); + } + }; + + set_sample_count(2, Tfc::MULTISAMPLE_X2); + set_sample_count(4, Tfc::MULTISAMPLE_X4); + set_sample_count(8, Tfc::MULTISAMPLE_X8); + set_sample_count(16, Tfc::MULTISAMPLE_X16); + + caps + } + + unsafe fn surface_capabilities( + &self, + surface: &super::Surface, + ) -> Option { + let current_extent = { + match surface.target { + SurfaceTarget::WndHandle(wnd_handle) + | SurfaceTarget::VisualFromWndHandle { + handle: wnd_handle, .. + } => { + let mut rect = Default::default(); + if unsafe { WindowsAndMessaging::GetClientRect(wnd_handle, &mut rect) }.is_ok() + { + Some(wgt::Extent3d { + width: (rect.right - rect.left) as u32, + height: (rect.bottom - rect.top) as u32, + depth_or_array_layers: 1, + }) + } else { + log::warn!("Unable to get the window client rect"); + None + } + } + SurfaceTarget::Visual(_) + | SurfaceTarget::SurfaceHandle(_) + | SurfaceTarget::SwapChainPanel(_) => None, + } + }; + + let mut present_modes = vec![wgt::PresentMode::Mailbox, wgt::PresentMode::Fifo]; + if surface.supports_allow_tearing { + present_modes.push(wgt::PresentMode::Immediate); + } + + Some(crate::SurfaceCapabilities { + formats: vec![ + wgt::TextureFormat::Bgra8UnormSrgb, + wgt::TextureFormat::Bgra8Unorm, + wgt::TextureFormat::Rgba8UnormSrgb, + wgt::TextureFormat::Rgba8Unorm, + wgt::TextureFormat::Rgb10a2Unorm, + wgt::TextureFormat::Rgba16Float, + ], + // See https://learn.microsoft.com/en-us/windows/win32/api/dxgi/nf-dxgi-idxgidevice1-setmaximumframelatency + maximum_frame_latency: 1..=16, + current_extent, + usage: wgt::TextureUses::COLOR_TARGET + | wgt::TextureUses::COPY_SRC + | wgt::TextureUses::COPY_DST, + present_modes, + composite_alpha_modes: match surface.target { + SurfaceTarget::WndHandle(_) => vec![wgt::CompositeAlphaMode::Opaque], + SurfaceTarget::Visual(_) + | SurfaceTarget::VisualFromWndHandle { .. } + | SurfaceTarget::SurfaceHandle(_) + | SurfaceTarget::SwapChainPanel(_) => vec![ + wgt::CompositeAlphaMode::Auto, + wgt::CompositeAlphaMode::Inherit, + wgt::CompositeAlphaMode::Opaque, + wgt::CompositeAlphaMode::PostMultiplied, + wgt::CompositeAlphaMode::PreMultiplied, + ], + }, + }) + } + + unsafe fn get_presentation_timestamp(&self) -> wgt::PresentationTimestamp { + wgt::PresentationTimestamp(self.presentation_timer.get_timestamp_ns()) + } + + fn get_ordered_buffer_usages(&self) -> wgt::BufferUses { + wgt::BufferUses::INCLUSIVE | wgt::BufferUses::MAP_WRITE + } + + // Don't put barriers between inclusive uses + // DX12 implicitly orders renderpasses on the same resources. + fn get_ordered_texture_usages(&self) -> wgt::TextureUses { + wgt::TextureUses::INCLUSIVE + | wgt::TextureUses::COLOR_TARGET + | wgt::TextureUses::DEPTH_STENCIL_WRITE + } +} + +fn get_adapter_pci_info(vendor_id: u32, device_id: u32) -> String { + // SAFETY: SetupDiGetClassDevsW is called with valid parameters + let device_info_set = unsafe { + match SetupDiGetClassDevsW(Some(&GUID_DEVCLASS_DISPLAY), None, None, DIGCF_PRESENT) { + Ok(set) => set, + Err(_) => return String::new(), + } + }; + + struct DeviceInfoSetGuard(HDEVINFO); + impl Drop for DeviceInfoSetGuard { + fn drop(&mut self) { + // SAFETY: device_info_set is a valid HDEVINFO and is only dropped once via this guard + unsafe { + let _ = SetupDiDestroyDeviceInfoList(self.0); + } + } + } + let _guard = DeviceInfoSetGuard(device_info_set); + + let mut device_index = 0u32; + loop { + let mut device_info_data = SP_DEVINFO_DATA { + cbSize: size_of::() as u32, + ..Default::default() + }; + + // SAFETY: device_info_set is a valid HDEVINFO, device_index starts at 0 and + // device_info_data is properly initialized above + unsafe { + if SetupDiEnumDeviceInfo(device_info_set, device_index, &mut device_info_data).is_err() + { + if GetLastError() == ERROR_NO_MORE_ITEMS { + break; + } + device_index += 1; + continue; + } + } + + let mut hardware_id_size = 0u32; + // SAFETY: device_info_set and device_info_data are valid + unsafe { + let _ = SetupDiGetDeviceRegistryPropertyW( + device_info_set, + &device_info_data, + SPDRP_HARDWAREID, + None, + None, + Some(&mut hardware_id_size), + ); + } + + if hardware_id_size == 0 { + device_index += 1; + continue; + } + + let mut hardware_id_buffer = vec![0u8; hardware_id_size as usize]; + // SAFETY: device_info_set and device_info_data are valid + unsafe { + if SetupDiGetDeviceRegistryPropertyW( + device_info_set, + &device_info_data, + SPDRP_HARDWAREID, + None, + Some(&mut hardware_id_buffer), + Some(&mut hardware_id_size), + ) + .is_err() + { + device_index += 1; + continue; + } + } + + let hardware_id_u16: Vec = hardware_id_buffer + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + let hardware_ids: Vec = hardware_id_u16 + .split(|&c| c == 0) + .filter(|s| !s.is_empty()) + .map(|s| String::from_utf16_lossy(s).to_uppercase()) + .collect(); + + // https://learn.microsoft.com/en-us/windows-hardware/drivers/install/identifiers-for-pci-devices + let expected_id = format!("PCI\\VEN_{vendor_id:04X}&DEV_{device_id:04X}"); + if !hardware_ids.iter().any(|id| id.contains(&expected_id)) { + device_index += 1; + continue; + } + + let mut bus_buffer = [0u8; 4]; + let mut data_size = bus_buffer.len() as u32; + // SAFETY: device_info_set and device_info_data are valid + let bus_number = unsafe { + if SetupDiGetDeviceRegistryPropertyW( + device_info_set, + &device_info_data, + SPDRP_BUSNUMBER, + None, + Some(&mut bus_buffer), + Some(&mut data_size), + ) + .is_err() + { + device_index += 1; + continue; + } + u32::from_le_bytes(bus_buffer) + }; + + let mut addr_buffer = [0u8; 4]; + let mut addr_size = addr_buffer.len() as u32; + // SAFETY: device_info_set and device_info_data are valid + unsafe { + if SetupDiGetDeviceRegistryPropertyW( + device_info_set, + &device_info_data, + SPDRP_ADDRESS, + None, + Some(&mut addr_buffer), + Some(&mut addr_size), + ) + .is_err() + { + device_index += 1; + continue; + } + } + let address = u32::from_le_bytes(addr_buffer); + + // https://learn.microsoft.com/en-us/windows-hardware/drivers/kernel/obtaining-device-configuration-information-at-irql---dispatch-level + let device = (address >> 16) & 0x0000FFFF; + let function = address & 0x0000FFFF; + + // domain:bus:device.function + return format!("{:04x}:{:02x}:{:02x}.{:x}", 0, bus_number, device, function); + } + + String::new() +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/command.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/command.rs new file mode 100644 index 00000000..5f7d5ba3 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/command.rs @@ -0,0 +1,1859 @@ +use alloc::vec::Vec; +use core::{mem, ops::Range}; + +use windows::{ + core::Interface as _, + Win32::{ + Foundation, + Graphics::{Direct3D12, Dxgi}, + }, +}; + +use super::conv; +use crate::{ + auxil::{ + self, + dxgi::{name::ObjectExt as _, result::HResult as _}, + }, + dx12::borrow_interface_temporarily, + AccelerationStructureEntries, CommandEncoder as _, +}; + +fn make_box(origin: &wgt::Origin3d, size: &crate::CopyExtent) -> Direct3D12::D3D12_BOX { + Direct3D12::D3D12_BOX { + left: origin.x, + top: origin.y, + right: origin.x + size.width, + bottom: origin.y + size.height, + front: origin.z, + back: origin.z + size.depth, + } +} + +impl crate::BufferTextureCopy { + fn to_subresource_footprint( + &self, + format: wgt::TextureFormat, + ) -> Direct3D12::D3D12_PLACED_SUBRESOURCE_FOOTPRINT { + let (block_width, _) = format.block_dimensions(); + Direct3D12::D3D12_PLACED_SUBRESOURCE_FOOTPRINT { + Offset: self.buffer_layout.offset, + Footprint: Direct3D12::D3D12_SUBRESOURCE_FOOTPRINT { + Format: auxil::dxgi::conv::map_texture_format_for_copy( + format, + self.texture_base.aspect, + ) + .unwrap(), + Width: self.size.width, + Height: self.size.height, + Depth: self.size.depth, + RowPitch: { + let actual = self.buffer_layout.bytes_per_row.unwrap_or_else(|| { + // this may happen for single-line updates + let block_size = format + .block_copy_size(Some(self.texture_base.aspect.map())) + .unwrap(); + (self.size.width / block_width) * block_size + }); + wgt::math::align_to(actual, Direct3D12::D3D12_TEXTURE_DATA_PITCH_ALIGNMENT) + }, + }, + } + } +} + +impl super::Temp { + fn prepare_marker(&mut self, marker: &str) -> (&[u16], u32) { + self.marker.clear(); + self.marker.extend(marker.encode_utf16()); + self.marker.push(0); + (&self.marker, self.marker.len() as u32 * 2) + } +} + +impl Drop for super::CommandEncoder { + fn drop(&mut self) { + use crate::CommandEncoder; + unsafe { self.discard_encoding() } + + let mut rtv_pool = self.rtv_pool.lock(); + for handle in self.temp_rtv_handles.drain(..) { + rtv_pool.free_handle(handle); + } + drop(rtv_pool); + + self.counters.command_encoders.sub(1); + } +} + +impl super::CommandEncoder { + unsafe fn begin_pass(&mut self, kind: super::PassKind, label: crate::Label) { + let list = self.list.as_ref().unwrap(); + self.pass.kind = kind; + if let Some(label) = label { + let (wide_label, size) = self.temp.prepare_marker(label); + unsafe { list.BeginEvent(0, Some(wide_label.as_ptr().cast()), size) }; + self.pass.has_label = true; + } + self.pass.dirty_root_elements = 0; + self.pass.dirty_vertex_buffers = 0; + unsafe { + list.SetDescriptorHeaps(&[ + Some(self.shared.heap_views.raw.clone()), + Some(self.shared.sampler_heap.heap().clone()), + ]) + }; + } + + unsafe fn end_pass(&mut self) { + let list = self.list.as_ref().unwrap(); + unsafe { list.SetDescriptorHeaps(&[]) }; + if self.pass.has_label { + unsafe { list.EndEvent() }; + } + self.pass.clear(); + } + + unsafe fn prepare_vertex_buffers(&mut self) { + while self.pass.dirty_vertex_buffers != 0 { + let list = self.list.as_ref().unwrap(); + let index = self.pass.dirty_vertex_buffers.trailing_zeros(); + self.pass.dirty_vertex_buffers ^= 1 << index; + unsafe { + list.IASetVertexBuffers( + index, + Some(&self.pass.vertex_buffers[index as usize..][..1]), + ); + } + } + } + + unsafe fn prepare_draw(&mut self, first_vertex: i32, first_instance: u32) { + unsafe { + self.prepare_vertex_buffers(); + } + if let Some(root_index) = self + .pass + .layout + .special_constants + .as_ref() + .map(|sc| sc.root_index) + { + let needs_update = match self.pass.root_elements[root_index as usize] { + super::RootElement::SpecialConstantBuffer { + first_vertex: other_vertex, + first_instance: other_instance, + other: _, + } => first_vertex != other_vertex || first_instance != other_instance, + _ => true, + }; + if needs_update { + self.pass.dirty_root_elements |= 1 << root_index; + self.pass.root_elements[root_index as usize] = + super::RootElement::SpecialConstantBuffer { + first_vertex, + first_instance, + other: 0, + }; + } + } + self.update_root_elements(); + } + + fn prepare_dispatch(&mut self, count: [u32; 3]) { + if let Some(root_index) = self + .pass + .layout + .special_constants + .as_ref() + .map(|sc| sc.root_index) + { + let needs_update = match self.pass.root_elements[root_index as usize] { + super::RootElement::SpecialConstantBuffer { + first_vertex, + first_instance, + other, + } => [first_vertex as u32, first_instance, other] != count, + _ => true, + }; + if needs_update { + self.pass.dirty_root_elements |= 1 << root_index; + self.pass.root_elements[root_index as usize] = + super::RootElement::SpecialConstantBuffer { + first_vertex: count[0] as i32, + first_instance: count[1], + other: count[2], + }; + } + } + self.update_root_elements(); + } + + // Note: we have to call this lazily before draw calls. Otherwise, D3D complains + // about the root parameters being incompatible with root signature. + fn update_root_elements(&mut self) { + use super::PassKind as Pk; + + while self.pass.dirty_root_elements != 0 { + let list = self.list.as_ref().unwrap(); + let index = self.pass.dirty_root_elements.trailing_zeros(); + self.pass.dirty_root_elements ^= 1 << index; + + match self.pass.root_elements[index as usize] { + super::RootElement::Empty => log::error!("Root index {index} is not bound"), + super::RootElement::Constant => { + let info = self.pass.layout.root_constant_info.as_ref().unwrap(); + + for offset in info.range.clone() { + let val = self.pass.constant_data[offset as usize]; + match self.pass.kind { + Pk::Render => unsafe { + list.SetGraphicsRoot32BitConstant(index, val, offset) + }, + Pk::Compute => unsafe { + list.SetComputeRoot32BitConstant(index, val, offset) + }, + Pk::Transfer => (), + } + } + } + super::RootElement::SpecialConstantBuffer { + first_vertex, + first_instance, + other, + } => match self.pass.kind { + Pk::Render => { + unsafe { list.SetGraphicsRoot32BitConstant(index, first_vertex as u32, 0) }; + unsafe { list.SetGraphicsRoot32BitConstant(index, first_instance, 1) }; + } + Pk::Compute => { + unsafe { list.SetComputeRoot32BitConstant(index, first_vertex as u32, 0) }; + unsafe { list.SetComputeRoot32BitConstant(index, first_instance, 1) }; + unsafe { list.SetComputeRoot32BitConstant(index, other, 2) }; + } + Pk::Transfer => (), + }, + super::RootElement::Table(descriptor) => match self.pass.kind { + Pk::Render => unsafe { list.SetGraphicsRootDescriptorTable(index, descriptor) }, + Pk::Compute => unsafe { list.SetComputeRootDescriptorTable(index, descriptor) }, + Pk::Transfer => (), + }, + super::RootElement::DynamicUniformBuffer { address } => { + let address = address.ptr; + match self.pass.kind { + Pk::Render => unsafe { + list.SetGraphicsRootConstantBufferView(index, address) + }, + Pk::Compute => unsafe { + list.SetComputeRootConstantBufferView(index, address) + }, + Pk::Transfer => (), + } + } + super::RootElement::DynamicOffsetsBuffer { start, end } => { + let values = &self.pass.dynamic_storage_buffer_offsets[start..end]; + + for (offset, &value) in values.iter().enumerate() { + match self.pass.kind { + Pk::Render => unsafe { + list.SetGraphicsRoot32BitConstant(index, value, offset as u32) + }, + Pk::Compute => unsafe { + list.SetComputeRoot32BitConstant(index, value, offset as u32) + }, + Pk::Transfer => (), + } + } + } + super::RootElement::SamplerHeap => match self.pass.kind { + Pk::Render => unsafe { + list.SetGraphicsRootDescriptorTable( + index, + self.shared.sampler_heap.gpu_descriptor_table(), + ) + }, + Pk::Compute => unsafe { + list.SetComputeRootDescriptorTable( + index, + self.shared.sampler_heap.gpu_descriptor_table(), + ) + }, + Pk::Transfer => (), + }, + } + } + } + + fn reset_signature(&mut self, layout: &super::PipelineLayoutShared) { + if let Some(root_index) = layout.special_constants.as_ref().map(|sc| sc.root_index) { + self.pass.root_elements[root_index as usize] = + super::RootElement::SpecialConstantBuffer { + first_vertex: 0, + first_instance: 0, + other: 0, + }; + } + if let Some(root_index) = layout.sampler_heap_root_index { + self.pass.root_elements[root_index as usize] = super::RootElement::SamplerHeap; + } + self.pass.layout = layout.clone(); + self.pass.dirty_root_elements = (1 << layout.total_root_elements) - 1; + } + + fn write_pass_end_timestamp_if_requested(&mut self) { + if let Some((query_set_raw, index)) = self.end_of_pass_timer_query.take() { + use crate::CommandEncoder as _; + unsafe { + self.write_timestamp( + &crate::dx12::QuerySet { + raw: query_set_raw, + raw_ty: Direct3D12::D3D12_QUERY_TYPE_TIMESTAMP, + }, + index, + ); + } + } + } + + unsafe fn buf_tex_intermediate( + &mut self, + region: crate::BufferTextureCopy, + tex_fmt: wgt::TextureFormat, + copy_op: impl FnOnce(&mut Self, &super::Buffer, wgt::BufferSize, crate::BufferTextureCopy) -> T, + ) -> (T, super::Buffer) { + let size = { + let copy_info = region.buffer_layout.get_buffer_texture_copy_info( + tex_fmt, + region.texture_base.aspect.map(), + ®ion.size.into(), + ); + copy_info.unwrap().bytes_in_copy + }; + + let size = wgt::BufferSize::new(size).unwrap(); + + let buffer = { + let (resource, allocation) = + super::suballocation::DeviceAllocationContext::from(&*self) + .create_buffer(&crate::BufferDescriptor { + label: None, + size: size.get(), + usage: wgt::BufferUses::COPY_SRC | wgt::BufferUses::COPY_DST, + memory_flags: crate::MemoryFlags::empty(), + }) + .expect(concat!( + "internal error: ", + "failed to allocate intermediate buffer ", + "for offset alignment" + )); + super::Buffer { + resource, + size: size.get(), + allocation, + } + }; + + let mut region = region; + region.buffer_layout.offset = 0; + + unsafe { + self.transition_buffers( + [crate::BufferBarrier { + buffer: &buffer, + usage: crate::StateTransition { + from: wgt::BufferUses::empty(), + to: wgt::BufferUses::COPY_DST, + }, + }] + .into_iter(), + ) + }; + + let t = copy_op(self, &buffer, size, region); + + unsafe { + self.transition_buffers( + [crate::BufferBarrier { + buffer: &buffer, + usage: crate::StateTransition { + from: wgt::BufferUses::COPY_DST, + to: wgt::BufferUses::COPY_SRC, + }, + }] + .into_iter(), + ) + }; + + (t, buffer) + } +} + +impl crate::CommandEncoder for super::CommandEncoder { + type A = super::Api; + + unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> { + let list = loop { + if let Some(list) = self.free_lists.pop() { + // TODO: Is an error expected here and should we print it? + let reset_result = unsafe { list.Reset(&self.allocator, None) }; + if reset_result.is_ok() { + break Some(list); + } + } else { + break None; + } + }; + + let list = if let Some(list) = list { + list + } else { + unsafe { + self.device.CreateCommandList( + 0, + Direct3D12::D3D12_COMMAND_LIST_TYPE_DIRECT, + &self.allocator, + None, + ) + } + .into_device_result("Create command list")? + }; + + if let Some(label) = label { + list.set_name(label)?; + } + + self.list = Some(list); + self.temp.clear(); + self.pass.clear(); + Ok(()) + } + unsafe fn discard_encoding(&mut self) { + if let Some(list) = self.list.take() { + if unsafe { list.Close() }.is_ok() { + self.free_lists.push(list); + } + } + } + unsafe fn end_encoding(&mut self) -> Result { + let raw = self.list.take().unwrap(); + unsafe { raw.Close() }.into_device_result("GraphicsCommandList::close")?; + Ok(super::CommandBuffer { raw }) + } + unsafe fn reset_all>(&mut self, command_buffers: I) { + self.intermediate_copy_bufs.clear(); + for cmd_buf in command_buffers { + self.free_lists.push(cmd_buf.raw); + } + if let Err(e) = unsafe { self.allocator.Reset() } { + log::error!("ID3D12CommandAllocator::Reset() failed with {e}"); + } + } + + unsafe fn transition_buffers<'a, T>(&mut self, barriers: T) + where + T: Iterator>, + { + self.temp.barriers.clear(); + + for barrier in barriers { + let s0 = conv::map_buffer_usage_to_state(barrier.usage.from); + let s1 = conv::map_buffer_usage_to_state(barrier.usage.to); + if s0 != s1 { + let raw = Direct3D12::D3D12_RESOURCE_BARRIER { + Type: Direct3D12::D3D12_RESOURCE_BARRIER_TYPE_TRANSITION, + Flags: Direct3D12::D3D12_RESOURCE_BARRIER_FLAG_NONE, + Anonymous: Direct3D12::D3D12_RESOURCE_BARRIER_0 { + Transition: mem::ManuallyDrop::new( + Direct3D12::D3D12_RESOURCE_TRANSITION_BARRIER { + pResource: unsafe { + borrow_interface_temporarily(&barrier.buffer.resource) + }, + Subresource: Direct3D12::D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES, + StateBefore: s0, + StateAfter: s1, + }, + ), + }, + }; + self.temp.barriers.push(raw); + } else if barrier.usage.from == wgt::BufferUses::STORAGE_READ_WRITE + || barrier.usage.from == wgt::BufferUses::ACCELERATION_STRUCTURE_QUERY + { + let raw = Direct3D12::D3D12_RESOURCE_BARRIER { + Type: Direct3D12::D3D12_RESOURCE_BARRIER_TYPE_UAV, + Flags: Direct3D12::D3D12_RESOURCE_BARRIER_FLAG_NONE, + Anonymous: Direct3D12::D3D12_RESOURCE_BARRIER_0 { + UAV: mem::ManuallyDrop::new(Direct3D12::D3D12_RESOURCE_UAV_BARRIER { + pResource: unsafe { + borrow_interface_temporarily(&barrier.buffer.resource) + }, + }), + }, + }; + self.temp.barriers.push(raw); + } + } + + if !self.temp.barriers.is_empty() { + unsafe { + self.list + .as_ref() + .unwrap() + .ResourceBarrier(&self.temp.barriers) + }; + } + } + + unsafe fn transition_textures<'a, T>(&mut self, barriers: T) + where + T: Iterator>, + { + self.temp.barriers.clear(); + + for barrier in barriers { + let s0 = conv::map_texture_usage_to_state(barrier.usage.from); + let s1 = conv::map_texture_usage_to_state(barrier.usage.to); + if s0 != s1 { + let mut raw = Direct3D12::D3D12_RESOURCE_BARRIER { + Type: Direct3D12::D3D12_RESOURCE_BARRIER_TYPE_TRANSITION, + Flags: Direct3D12::D3D12_RESOURCE_BARRIER_FLAG_NONE, + Anonymous: Direct3D12::D3D12_RESOURCE_BARRIER_0 { + Transition: mem::ManuallyDrop::new( + Direct3D12::D3D12_RESOURCE_TRANSITION_BARRIER { + pResource: unsafe { + borrow_interface_temporarily(&barrier.texture.resource) + }, + Subresource: Direct3D12::D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES, + StateBefore: s0, + StateAfter: s1, + }, + ), + }, + }; + + let tex_mip_level_count = barrier.texture.mip_level_count; + let tex_array_layer_count = barrier.texture.array_layer_count(); + + if barrier.range.is_full_resource( + barrier.texture.format, + tex_mip_level_count, + tex_array_layer_count, + ) { + // Only one barrier if it affects the whole image. + self.temp.barriers.push(raw); + } else { + // Selected texture aspect is relevant if the texture format has both depth _and_ stencil aspects. + let planes = if barrier.texture.format.is_combined_depth_stencil_format() { + match barrier.range.aspect { + wgt::TextureAspect::All => 0..2, + wgt::TextureAspect::DepthOnly => 0..1, + wgt::TextureAspect::StencilOnly => 1..2, + _ => unreachable!(), + } + } else if let Some(planes) = barrier.texture.format.planes() { + match barrier.range.aspect { + wgt::TextureAspect::All => 0..planes, + wgt::TextureAspect::Plane0 => 0..1, + wgt::TextureAspect::Plane1 => 1..2, + wgt::TextureAspect::Plane2 => 2..3, + _ => unreachable!(), + } + } else { + match barrier.texture.format { + wgt::TextureFormat::Stencil8 => 1..2, + wgt::TextureFormat::Depth24Plus => 0..2, // TODO: investigate why tests fail if we set this to 0..1 + _ => 0..1, + } + }; + + for mip_level in barrier.range.mip_range(tex_mip_level_count) { + for array_layer in barrier.range.layer_range(tex_array_layer_count) { + for plane in planes.clone() { + unsafe { &mut *raw.Anonymous.Transition }.Subresource = barrier + .texture + .calc_subresource(mip_level, array_layer, plane); + self.temp.barriers.push(raw.clone()); + } + } + } + } + } else if barrier.usage.from == wgt::TextureUses::STORAGE_READ_WRITE { + let raw = Direct3D12::D3D12_RESOURCE_BARRIER { + Type: Direct3D12::D3D12_RESOURCE_BARRIER_TYPE_UAV, + Flags: Direct3D12::D3D12_RESOURCE_BARRIER_FLAG_NONE, + Anonymous: Direct3D12::D3D12_RESOURCE_BARRIER_0 { + UAV: mem::ManuallyDrop::new(Direct3D12::D3D12_RESOURCE_UAV_BARRIER { + pResource: unsafe { + borrow_interface_temporarily(&barrier.texture.resource) + }, + }), + }, + }; + self.temp.barriers.push(raw); + } + } + + if !self.temp.barriers.is_empty() { + unsafe { + self.list + .as_ref() + .unwrap() + .ResourceBarrier(&self.temp.barriers) + }; + } + } + + unsafe fn clear_buffer(&mut self, buffer: &super::Buffer, range: crate::MemoryRange) { + let list = self.list.as_ref().unwrap(); + let mut offset = range.start; + while offset < range.end { + let size = super::ZERO_BUFFER_SIZE.min(range.end - offset); + unsafe { + list.CopyBufferRegion(&buffer.resource, offset, &self.shared.zero_buffer, 0, size) + }; + offset += size; + } + } + + unsafe fn copy_buffer_to_buffer( + &mut self, + src: &super::Buffer, + dst: &super::Buffer, + regions: T, + ) where + T: Iterator, + { + let list = self.list.as_ref().unwrap(); + for r in regions { + unsafe { + list.CopyBufferRegion( + &dst.resource, + r.dst_offset, + &src.resource, + r.src_offset, + r.size.get(), + ) + }; + } + } + + unsafe fn copy_texture_to_texture( + &mut self, + src: &super::Texture, + _src_usage: wgt::TextureUses, + dst: &super::Texture, + regions: T, + ) where + T: Iterator, + { + let list = self.list.as_ref().unwrap(); + + for r in regions { + let src_location = Direct3D12::D3D12_TEXTURE_COPY_LOCATION { + pResource: unsafe { borrow_interface_temporarily(&src.resource) }, + Type: Direct3D12::D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX, + Anonymous: Direct3D12::D3D12_TEXTURE_COPY_LOCATION_0 { + SubresourceIndex: src.calc_subresource_for_copy(&r.src_base), + }, + }; + let dst_location = Direct3D12::D3D12_TEXTURE_COPY_LOCATION { + pResource: unsafe { borrow_interface_temporarily(&dst.resource) }, + Type: Direct3D12::D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX, + Anonymous: Direct3D12::D3D12_TEXTURE_COPY_LOCATION_0 { + SubresourceIndex: dst.calc_subresource_for_copy(&r.dst_base), + }, + }; + + let src_box = make_box(&r.src_base.origin, &r.size); + + unsafe { + list.CopyTextureRegion( + &dst_location, + r.dst_base.origin.x, + r.dst_base.origin.y, + r.dst_base.origin.z, + &src_location, + Some(&src_box), + ) + }; + } + } + + unsafe fn copy_buffer_to_texture( + &mut self, + src: &super::Buffer, + dst: &super::Texture, + regions: T, + ) where + T: Iterator, + { + let offset_alignment = self.shared.private_caps.texture_data_placement_alignment(); + + for naive_copy_region in regions { + let is_offset_aligned = naive_copy_region.buffer_layout.offset % offset_alignment == 0; + let (final_copy_region, src) = if is_offset_aligned { + (naive_copy_region, src) + } else { + let (intermediate_to_dst_region, intermediate_buf) = unsafe { + let src_offset = naive_copy_region.buffer_layout.offset; + self.buf_tex_intermediate( + naive_copy_region, + dst.format, + |this, buf, size, intermediate_to_dst_region| { + let layout = crate::BufferCopy { + src_offset, + dst_offset: 0, + size, + }; + this.copy_buffer_to_buffer(src, buf, [layout].into_iter()); + intermediate_to_dst_region + }, + ) + }; + self.intermediate_copy_bufs.push(intermediate_buf); + let intermediate_buf = self.intermediate_copy_bufs.last().unwrap(); + (intermediate_to_dst_region, intermediate_buf) + }; + + let list = self.list.as_ref().unwrap(); + + let src_location = Direct3D12::D3D12_TEXTURE_COPY_LOCATION { + pResource: unsafe { borrow_interface_temporarily(&src.resource) }, + Type: Direct3D12::D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT, + Anonymous: Direct3D12::D3D12_TEXTURE_COPY_LOCATION_0 { + PlacedFootprint: final_copy_region.to_subresource_footprint(dst.format), + }, + }; + let dst_location = Direct3D12::D3D12_TEXTURE_COPY_LOCATION { + pResource: unsafe { borrow_interface_temporarily(&dst.resource) }, + Type: Direct3D12::D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX, + Anonymous: Direct3D12::D3D12_TEXTURE_COPY_LOCATION_0 { + SubresourceIndex: dst + .calc_subresource_for_copy(&final_copy_region.texture_base), + }, + }; + + let src_box = make_box(&wgt::Origin3d::ZERO, &final_copy_region.size); + unsafe { + list.CopyTextureRegion( + &dst_location, + final_copy_region.texture_base.origin.x, + final_copy_region.texture_base.origin.y, + final_copy_region.texture_base.origin.z, + &src_location, + Some(&src_box), + ) + }; + } + } + + unsafe fn copy_texture_to_buffer( + &mut self, + src: &super::Texture, + _src_usage: wgt::TextureUses, + dst: &super::Buffer, + regions: T, + ) where + T: Iterator, + { + let copy_aligned = |this: &mut Self, + src: &super::Texture, + dst: &super::Buffer, + r: crate::BufferTextureCopy| { + let list = this.list.as_ref().unwrap(); + + let src_location = Direct3D12::D3D12_TEXTURE_COPY_LOCATION { + pResource: unsafe { borrow_interface_temporarily(&src.resource) }, + Type: Direct3D12::D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX, + Anonymous: Direct3D12::D3D12_TEXTURE_COPY_LOCATION_0 { + SubresourceIndex: src.calc_subresource_for_copy(&r.texture_base), + }, + }; + let dst_location = Direct3D12::D3D12_TEXTURE_COPY_LOCATION { + pResource: unsafe { borrow_interface_temporarily(&dst.resource) }, + Type: Direct3D12::D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT, + Anonymous: Direct3D12::D3D12_TEXTURE_COPY_LOCATION_0 { + PlacedFootprint: r.to_subresource_footprint(src.format), + }, + }; + + let src_box = make_box(&r.texture_base.origin, &r.size); + unsafe { + list.CopyTextureRegion(&dst_location, 0, 0, 0, &src_location, Some(&src_box)) + }; + }; + + let offset_alignment = self.shared.private_caps.texture_data_placement_alignment(); + + for r in regions { + let is_offset_aligned = r.buffer_layout.offset % offset_alignment == 0; + if is_offset_aligned { + copy_aligned(self, src, dst, r) + } else { + let orig_offset = r.buffer_layout.offset; + let (intermediate_to_dst_region, src) = unsafe { + self.buf_tex_intermediate( + r, + src.format, + |this, buf, size, intermediate_region| { + copy_aligned(this, src, buf, intermediate_region); + crate::BufferCopy { + src_offset: 0, + dst_offset: orig_offset, + size, + } + }, + ) + }; + + unsafe { + self.copy_buffer_to_buffer(&src, dst, [intermediate_to_dst_region].into_iter()); + } + + self.intermediate_copy_bufs.push(src); + }; + } + } + + unsafe fn begin_query(&mut self, set: &super::QuerySet, index: u32) { + unsafe { + self.list + .as_ref() + .unwrap() + .BeginQuery(&set.raw, set.raw_ty, index) + }; + } + unsafe fn end_query(&mut self, set: &super::QuerySet, index: u32) { + unsafe { + self.list + .as_ref() + .unwrap() + .EndQuery(&set.raw, set.raw_ty, index) + }; + } + unsafe fn write_timestamp(&mut self, set: &super::QuerySet, index: u32) { + unsafe { + self.list.as_ref().unwrap().EndQuery( + &set.raw, + Direct3D12::D3D12_QUERY_TYPE_TIMESTAMP, + index, + ) + }; + } + unsafe fn read_acceleration_structure_compact_size( + &mut self, + acceleration_structure: &super::AccelerationStructure, + buf: &super::Buffer, + ) { + let list = self + .list + .as_ref() + .unwrap() + .cast::() + .unwrap(); + unsafe { + list.EmitRaytracingAccelerationStructurePostbuildInfo( + &Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_POSTBUILD_INFO_DESC { + DestBuffer: buf.resource.GetGPUVirtualAddress(), + InfoType: Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_POSTBUILD_INFO_COMPACTED_SIZE, + }, + &[ + acceleration_structure.resource.GetGPUVirtualAddress() + ], + ) + } + } + unsafe fn reset_queries(&mut self, _set: &super::QuerySet, _range: Range) { + // nothing to do here + } + unsafe fn copy_query_results( + &mut self, + set: &super::QuerySet, + range: Range, + buffer: &super::Buffer, + offset: wgt::BufferAddress, + _stride: wgt::BufferSize, + ) { + unsafe { + self.list.as_ref().unwrap().ResolveQueryData( + &set.raw, + set.raw_ty, + range.start, + range.end - range.start, + &buffer.resource, + offset, + ) + }; + } + + // render + + unsafe fn begin_render_pass( + &mut self, + desc: &crate::RenderPassDescriptor, + ) -> Result<(), crate::DeviceError> { + unsafe { self.begin_pass(super::PassKind::Render, desc.label) }; + + // Start timestamp if any (before all other commands but after debug marker) + if let Some(timestamp_writes) = desc.timestamp_writes.as_ref() { + if let Some(index) = timestamp_writes.beginning_of_pass_write_index { + unsafe { + self.write_timestamp(timestamp_writes.query_set, index); + } + } + self.end_of_pass_timer_query = timestamp_writes + .end_of_pass_write_index + .map(|index| (timestamp_writes.query_set.raw.clone(), index)); + } + + let mut color_views = + [Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE { ptr: 0 }; crate::MAX_COLOR_ATTACHMENTS]; + let mut rtv_pool = self.rtv_pool.lock(); + for (rtv, cat) in color_views.iter_mut().zip(desc.color_attachments.iter()) { + if let Some(cat) = cat.as_ref() { + if cat.target.view.dimension == wgt::TextureViewDimension::D3 { + let desc = Direct3D12::D3D12_RENDER_TARGET_VIEW_DESC { + Format: cat.target.view.raw_format, + ViewDimension: Direct3D12::D3D12_RTV_DIMENSION_TEXTURE3D, + Anonymous: Direct3D12::D3D12_RENDER_TARGET_VIEW_DESC_0 { + Texture3D: Direct3D12::D3D12_TEX3D_RTV { + MipSlice: cat.target.view.mip_slice, + FirstWSlice: cat.depth_slice.unwrap(), + WSize: 1, + }, + }, + }; + let handle = rtv_pool.alloc_handle()?; + unsafe { + self.device.CreateRenderTargetView( + &cat.target.view.texture, + Some(&desc), + handle.raw, + ) + }; + *rtv = handle.raw; + self.temp_rtv_handles.push(handle); + } else { + *rtv = cat.target.view.handle_rtv.unwrap().raw; + } + } else { + *rtv = self.null_rtv_handle.raw; + } + } + drop(rtv_pool); + + let ds_view = desc.depth_stencil_attachment.as_ref().map(|ds| { + if ds.target.usage == wgt::TextureUses::DEPTH_STENCIL_WRITE { + ds.target.view.handle_dsv_rw.as_ref().unwrap().raw + } else { + ds.target.view.handle_dsv_ro.as_ref().unwrap().raw + } + }); + + let list = self.list.as_ref().unwrap(); + unsafe { + list.OMSetRenderTargets( + desc.color_attachments.len() as u32, + Some(color_views.as_ptr()), + false, + ds_view.as_ref().map(core::ptr::from_ref), + ) + }; + + self.pass.resolves.clear(); + for (rtv, cat) in color_views.iter().zip(desc.color_attachments.iter()) { + if let Some(cat) = cat.as_ref() { + if cat.ops.contains(crate::AttachmentOps::LOAD_CLEAR) { + let value = [ + cat.clear_value.r as f32, + cat.clear_value.g as f32, + cat.clear_value.b as f32, + cat.clear_value.a as f32, + ]; + unsafe { list.ClearRenderTargetView(*rtv, &value, None) }; + } + if let Some(ref target) = cat.resolve_target { + self.pass.resolves.push(super::PassResolve { + src: ( + cat.target.view.texture.clone(), + cat.target.view.subresource_index, + ), + dst: (target.view.texture.clone(), target.view.subresource_index), + format: target.view.raw_format, + }); + } + } + } + + if let Some(ref ds) = desc.depth_stencil_attachment { + let mut flags = Direct3D12::D3D12_CLEAR_FLAGS::default(); + let aspects = ds.target.view.aspects; + if ds.depth_ops.contains(crate::AttachmentOps::LOAD_CLEAR) + && aspects.contains(crate::FormatAspects::DEPTH) + { + flags |= Direct3D12::D3D12_CLEAR_FLAG_DEPTH; + } + if ds.stencil_ops.contains(crate::AttachmentOps::LOAD_CLEAR) + && aspects.contains(crate::FormatAspects::STENCIL) + { + flags |= Direct3D12::D3D12_CLEAR_FLAG_STENCIL; + } + + if let Some(ds_view) = ds_view { + if flags != Direct3D12::D3D12_CLEAR_FLAGS::default() { + unsafe { + list.ClearDepthStencilView( + ds_view, + flags, + ds.clear_value.0, + ds.clear_value.1 as u8, + None, + ) + } + } + } + } + + if let Some(multiview_mask) = desc.multiview_mask { + unsafe { + list.cast::() + .unwrap() + .SetViewInstanceMask(multiview_mask.get()); + } + } + + let raw_vp = Direct3D12::D3D12_VIEWPORT { + TopLeftX: 0.0, + TopLeftY: 0.0, + Width: desc.extent.width as f32, + Height: desc.extent.height as f32, + MinDepth: 0.0, + MaxDepth: 1.0, + }; + let raw_rect = Foundation::RECT { + left: 0, + top: 0, + right: desc.extent.width as i32, + bottom: desc.extent.height as i32, + }; + unsafe { list.RSSetViewports(core::slice::from_ref(&raw_vp)) }; + unsafe { list.RSSetScissorRects(core::slice::from_ref(&raw_rect)) }; + + Ok(()) + } + + unsafe fn end_render_pass(&mut self) { + if !self.pass.resolves.is_empty() { + let list = self.list.as_ref().unwrap(); + self.temp.barriers.clear(); + + // All the targets are expected to be in `COLOR_TARGET` state, + // but D3D12 has special source/destination states for the resolves. + for resolve in self.pass.resolves.iter() { + let barrier = Direct3D12::D3D12_RESOURCE_BARRIER { + Type: Direct3D12::D3D12_RESOURCE_BARRIER_TYPE_TRANSITION, + Flags: Direct3D12::D3D12_RESOURCE_BARRIER_FLAG_NONE, + Anonymous: Direct3D12::D3D12_RESOURCE_BARRIER_0 { + // Note: this assumes `D3D12_RESOURCE_STATE_RENDER_TARGET`. + // If it's not the case, we can include the `TextureUses` in `PassResolve`. + Transition: mem::ManuallyDrop::new( + Direct3D12::D3D12_RESOURCE_TRANSITION_BARRIER { + pResource: unsafe { borrow_interface_temporarily(&resolve.src.0) }, + Subresource: resolve.src.1, + StateBefore: Direct3D12::D3D12_RESOURCE_STATE_RENDER_TARGET, + StateAfter: Direct3D12::D3D12_RESOURCE_STATE_RESOLVE_SOURCE, + }, + ), + }, + }; + self.temp.barriers.push(barrier); + let barrier = Direct3D12::D3D12_RESOURCE_BARRIER { + Type: Direct3D12::D3D12_RESOURCE_BARRIER_TYPE_TRANSITION, + Flags: Direct3D12::D3D12_RESOURCE_BARRIER_FLAG_NONE, + Anonymous: Direct3D12::D3D12_RESOURCE_BARRIER_0 { + // Note: this assumes `D3D12_RESOURCE_STATE_RENDER_TARGET`. + // If it's not the case, we can include the `TextureUses` in `PassResolve`. + Transition: mem::ManuallyDrop::new( + Direct3D12::D3D12_RESOURCE_TRANSITION_BARRIER { + pResource: unsafe { borrow_interface_temporarily(&resolve.dst.0) }, + Subresource: resolve.dst.1, + StateBefore: Direct3D12::D3D12_RESOURCE_STATE_RENDER_TARGET, + StateAfter: Direct3D12::D3D12_RESOURCE_STATE_RESOLVE_DEST, + }, + ), + }, + }; + self.temp.barriers.push(barrier); + } + + if !self.temp.barriers.is_empty() { + profiling::scope!("ID3D12GraphicsCommandList::ResourceBarrier"); + unsafe { list.ResourceBarrier(&self.temp.barriers) }; + } + + for resolve in self.pass.resolves.iter() { + profiling::scope!("ID3D12GraphicsCommandList::ResolveSubresource"); + unsafe { + list.ResolveSubresource( + &resolve.dst.0, + resolve.dst.1, + &resolve.src.0, + resolve.src.1, + resolve.format, + ) + }; + } + + // Flip all the barriers to reverse, back into `COLOR_TARGET`. + for barrier in self.temp.barriers.iter_mut() { + let transition = unsafe { &mut *barrier.Anonymous.Transition }; + mem::swap(&mut transition.StateBefore, &mut transition.StateAfter); + } + if !self.temp.barriers.is_empty() { + profiling::scope!("ID3D12GraphicsCommandList::ResourceBarrier"); + unsafe { list.ResourceBarrier(&self.temp.barriers) }; + } + } + + self.write_pass_end_timestamp_if_requested(); + + unsafe { self.end_pass() }; + } + + unsafe fn set_bind_group( + &mut self, + layout: &super::PipelineLayout, + index: u32, + group: &super::BindGroup, + dynamic_offsets: &[wgt::DynamicOffset], + ) { + let info = layout.bind_group_infos[index as usize].as_ref().unwrap(); + let mut root_index = info.base_root_index as usize; + + // Bind CBV/SRC/UAV descriptor tables + if info.tables.contains(super::TableTypes::SRV_CBV_UAV) { + self.pass.root_elements[root_index] = + super::RootElement::Table(group.handle_views.unwrap().gpu); + root_index += 1; + } + + let mut offsets_index = 0; + if let Some(dynamic_storage_buffer_offsets) = info.dynamic_storage_buffer_offsets.as_ref() { + let root_index = dynamic_storage_buffer_offsets.root_index; + let range = &dynamic_storage_buffer_offsets.range; + + if range.end > self.pass.dynamic_storage_buffer_offsets.len() { + self.pass + .dynamic_storage_buffer_offsets + .resize(range.end, 0); + } + + offsets_index += range.start; + + self.pass.root_elements[root_index as usize] = + super::RootElement::DynamicOffsetsBuffer { + start: range.start, + end: range.end, + }; + + if self.pass.layout.signature == layout.shared.signature { + self.pass.dirty_root_elements |= 1 << root_index; + } else { + // D3D12 requires full reset on signature change + // but we don't reset it here since it will be reset below + }; + } + + // Bind root descriptors for dynamic uniform buffers + // or set root constants for offsets of dynamic storage buffers + for (&dynamic_buffer, &offset) in group.dynamic_buffers.iter().zip(dynamic_offsets) { + match dynamic_buffer { + super::DynamicBuffer::Uniform(gpu_base) => { + self.pass.root_elements[root_index] = + super::RootElement::DynamicUniformBuffer { + address: Direct3D12::D3D12_GPU_DESCRIPTOR_HANDLE { + ptr: gpu_base.ptr + offset as u64, + }, + }; + root_index += 1; + } + super::DynamicBuffer::Storage => { + self.pass.dynamic_storage_buffer_offsets[offsets_index] = offset; + offsets_index += 1; + } + } + } + + if self.pass.layout.signature == layout.shared.signature { + self.pass.dirty_root_elements |= (1 << root_index) - (1 << info.base_root_index); + } else { + // D3D12 requires full reset on signature change + self.reset_signature(&layout.shared); + }; + } + unsafe fn set_immediates( + &mut self, + layout: &super::PipelineLayout, + offset_bytes: u32, + data: &[u32], + ) { + let offset_words = offset_bytes as usize / 4; + + let info = layout.shared.root_constant_info.as_ref().unwrap(); + + self.pass.root_elements[info.root_index as usize] = super::RootElement::Constant; + + self.pass.constant_data[offset_words..(offset_words + data.len())].copy_from_slice(data); + + if self.pass.layout.signature == layout.shared.signature { + self.pass.dirty_root_elements |= 1 << info.root_index; + } else { + // D3D12 requires full reset on signature change + self.reset_signature(&layout.shared); + }; + } + + unsafe fn insert_debug_marker(&mut self, label: &str) { + let (wide_label, size) = self.temp.prepare_marker(label); + unsafe { + self.list + .as_ref() + .unwrap() + .SetMarker(0, Some(wide_label.as_ptr().cast()), size) + }; + } + unsafe fn begin_debug_marker(&mut self, group_label: &str) { + let (wide_label, size) = self.temp.prepare_marker(group_label); + unsafe { + self.list + .as_ref() + .unwrap() + .BeginEvent(0, Some(wide_label.as_ptr().cast()), size) + }; + } + unsafe fn end_debug_marker(&mut self) { + unsafe { self.list.as_ref().unwrap().EndEvent() } + } + + unsafe fn set_render_pipeline(&mut self, pipeline: &super::RenderPipeline) { + let list = self.list.clone().unwrap(); + + if self.pass.layout.signature != pipeline.layout.signature { + // D3D12 requires full reset on signature change + unsafe { list.SetGraphicsRootSignature(pipeline.layout.signature.as_ref()) }; + self.reset_signature(&pipeline.layout); + }; + + unsafe { list.SetPipelineState(&pipeline.raw) }; + unsafe { list.IASetPrimitiveTopology(pipeline.topology) }; + + for (index, (vb, &stride)) in self + .pass + .vertex_buffers + .iter_mut() + .zip(pipeline.vertex_strides.iter()) + .enumerate() + { + if let Some(stride) = stride { + if vb.StrideInBytes != stride { + vb.StrideInBytes = stride; + self.pass.dirty_vertex_buffers |= 1 << index; + } + } + } + } + + unsafe fn set_index_buffer<'a>( + &mut self, + binding: crate::BufferBinding<'a, super::Buffer>, + format: wgt::IndexFormat, + ) { + let ibv = Direct3D12::D3D12_INDEX_BUFFER_VIEW { + BufferLocation: binding.resolve_address(), + SizeInBytes: binding.resolve_size().try_into().unwrap(), + Format: auxil::dxgi::conv::map_index_format(format), + }; + + unsafe { self.list.as_ref().unwrap().IASetIndexBuffer(Some(&ibv)) } + } + unsafe fn set_vertex_buffer<'a>( + &mut self, + index: u32, + binding: crate::BufferBinding<'a, super::Buffer>, + ) { + let vb = &mut self.pass.vertex_buffers[index as usize]; + vb.BufferLocation = binding.resolve_address(); + vb.SizeInBytes = binding.resolve_size().try_into().unwrap(); + self.pass.dirty_vertex_buffers |= 1 << index; + } + + unsafe fn set_viewport(&mut self, rect: &crate::Rect, depth_range: Range) { + let raw_vp = Direct3D12::D3D12_VIEWPORT { + TopLeftX: rect.x, + TopLeftY: rect.y, + Width: rect.w, + Height: rect.h, + MinDepth: depth_range.start, + MaxDepth: depth_range.end, + }; + unsafe { + self.list + .as_ref() + .unwrap() + .RSSetViewports(core::slice::from_ref(&raw_vp)) + } + } + unsafe fn set_scissor_rect(&mut self, rect: &crate::Rect) { + let raw_rect = Foundation::RECT { + left: rect.x as i32, + top: rect.y as i32, + right: (rect.x + rect.w) as i32, + bottom: (rect.y + rect.h) as i32, + }; + unsafe { + self.list + .as_ref() + .unwrap() + .RSSetScissorRects(core::slice::from_ref(&raw_rect)) + } + } + unsafe fn set_stencil_reference(&mut self, value: u32) { + unsafe { self.list.as_ref().unwrap().OMSetStencilRef(value) } + } + unsafe fn set_blend_constants(&mut self, color: &[f32; 4]) { + unsafe { self.list.as_ref().unwrap().OMSetBlendFactor(Some(color)) } + } + + unsafe fn draw( + &mut self, + first_vertex: u32, + vertex_count: u32, + first_instance: u32, + instance_count: u32, + ) { + unsafe { self.prepare_draw(first_vertex as i32, first_instance) }; + unsafe { + self.list.as_ref().unwrap().DrawInstanced( + vertex_count, + instance_count, + first_vertex, + first_instance, + ) + } + } + unsafe fn draw_indexed( + &mut self, + first_index: u32, + index_count: u32, + base_vertex: i32, + first_instance: u32, + instance_count: u32, + ) { + unsafe { self.prepare_draw(base_vertex, first_instance) }; + unsafe { + self.list.as_ref().unwrap().DrawIndexedInstanced( + index_count, + instance_count, + first_index, + base_vertex, + first_instance, + ) + } + } + unsafe fn draw_mesh_tasks( + &mut self, + group_count_x: u32, + group_count_y: u32, + group_count_z: u32, + ) { + self.prepare_dispatch([group_count_x, group_count_y, group_count_z]); + let cmd_list6: Direct3D12::ID3D12GraphicsCommandList6 = + self.list.as_ref().unwrap().cast().unwrap(); + unsafe { + cmd_list6.DispatchMesh(group_count_x, group_count_y, group_count_z); + } + } + unsafe fn draw_indirect( + &mut self, + buffer: &super::Buffer, + offset: wgt::BufferAddress, + draw_count: u32, + ) { + if self + .pass + .layout + .special_constants + .as_ref() + .and_then(|sc| sc.indirect_cmd_signatures.as_ref()) + .is_some() + { + unsafe { self.prepare_vertex_buffers() }; + self.update_root_elements(); + } else { + unsafe { self.prepare_draw(0, 0) }; + } + + let cmd_signature = &self + .pass + .layout + .special_constants + .as_ref() + .and_then(|sc| sc.indirect_cmd_signatures.as_ref()) + .unwrap_or_else(|| &self.shared.cmd_signatures) + .draw; + unsafe { + self.list.as_ref().unwrap().ExecuteIndirect( + cmd_signature, + draw_count, + &buffer.resource, + offset, + None, + 0, + ) + } + } + unsafe fn draw_indexed_indirect( + &mut self, + buffer: &super::Buffer, + offset: wgt::BufferAddress, + draw_count: u32, + ) { + if self + .pass + .layout + .special_constants + .as_ref() + .and_then(|sc| sc.indirect_cmd_signatures.as_ref()) + .is_some() + { + unsafe { self.prepare_vertex_buffers() }; + self.update_root_elements(); + } else { + unsafe { self.prepare_draw(0, 0) }; + } + + let cmd_signature = &self + .pass + .layout + .special_constants + .as_ref() + .and_then(|sc| sc.indirect_cmd_signatures.as_ref()) + .unwrap_or_else(|| &self.shared.cmd_signatures) + .draw_indexed; + unsafe { + self.list.as_ref().unwrap().ExecuteIndirect( + cmd_signature, + draw_count, + &buffer.resource, + offset, + None, + 0, + ) + } + } + unsafe fn draw_mesh_tasks_indirect( + &mut self, + buffer: &::Buffer, + offset: wgt::BufferAddress, + draw_count: u32, + ) { + if self + .pass + .layout + .special_constants + .as_ref() + .and_then(|sc| sc.indirect_cmd_signatures.as_ref()) + .is_some() + { + self.update_root_elements(); + } else { + self.prepare_dispatch([0; 3]); + } + + let cmd_list6: Direct3D12::ID3D12GraphicsCommandList6 = + self.list.as_ref().unwrap().cast().unwrap(); + let Some(cmd_signature) = &self + .pass + .layout + .special_constants + .as_ref() + .and_then(|sc| sc.indirect_cmd_signatures.as_ref()) + .unwrap_or_else(|| &self.shared.cmd_signatures) + .draw_mesh + else { + panic!("Feature `MESH_SHADING` not enabled"); + }; + unsafe { + cmd_list6.ExecuteIndirect(cmd_signature, draw_count, &buffer.resource, offset, None, 0); + } + } + unsafe fn draw_indirect_count( + &mut self, + buffer: &super::Buffer, + offset: wgt::BufferAddress, + count_buffer: &super::Buffer, + count_offset: wgt::BufferAddress, + max_count: u32, + ) { + unsafe { self.prepare_draw(0, 0) }; + unsafe { + self.list.as_ref().unwrap().ExecuteIndirect( + &self.shared.cmd_signatures.draw, + max_count, + &buffer.resource, + offset, + &count_buffer.resource, + count_offset, + ) + } + } + unsafe fn draw_indexed_indirect_count( + &mut self, + buffer: &super::Buffer, + offset: wgt::BufferAddress, + count_buffer: &super::Buffer, + count_offset: wgt::BufferAddress, + max_count: u32, + ) { + unsafe { self.prepare_draw(0, 0) }; + unsafe { + self.list.as_ref().unwrap().ExecuteIndirect( + &self.shared.cmd_signatures.draw_indexed, + max_count, + &buffer.resource, + offset, + &count_buffer.resource, + count_offset, + ) + } + } + unsafe fn draw_mesh_tasks_indirect_count( + &mut self, + buffer: &::Buffer, + offset: wgt::BufferAddress, + count_buffer: &::Buffer, + count_offset: wgt::BufferAddress, + max_count: u32, + ) { + self.prepare_dispatch([0; 3]); + let cmd_list6: Direct3D12::ID3D12GraphicsCommandList6 = + self.list.as_ref().unwrap().cast().unwrap(); + let Some(ref command_signature) = self.shared.cmd_signatures.draw_mesh else { + panic!("Feature `MESH_SHADING` not enabled"); + }; + unsafe { + cmd_list6.ExecuteIndirect( + command_signature, + max_count, + &buffer.resource, + offset, + &count_buffer.resource, + count_offset, + ); + } + } + + // compute + + unsafe fn begin_compute_pass<'a>( + &mut self, + desc: &crate::ComputePassDescriptor<'a, super::QuerySet>, + ) { + unsafe { self.begin_pass(super::PassKind::Compute, desc.label) }; + + if let Some(timestamp_writes) = desc.timestamp_writes.as_ref() { + if let Some(index) = timestamp_writes.beginning_of_pass_write_index { + unsafe { + self.write_timestamp(timestamp_writes.query_set, index); + } + } + self.end_of_pass_timer_query = timestamp_writes + .end_of_pass_write_index + .map(|index| (timestamp_writes.query_set.raw.clone(), index)); + } + } + unsafe fn end_compute_pass(&mut self) { + self.write_pass_end_timestamp_if_requested(); + unsafe { self.end_pass() }; + } + + unsafe fn set_compute_pipeline(&mut self, pipeline: &super::ComputePipeline) { + let list = self.list.clone().unwrap(); + + if self.pass.layout.signature != pipeline.layout.signature { + // D3D12 requires full reset on signature change + unsafe { list.SetComputeRootSignature(pipeline.layout.signature.as_ref()) }; + self.reset_signature(&pipeline.layout); + }; + + unsafe { list.SetPipelineState(&pipeline.raw) } + } + + unsafe fn dispatch(&mut self, count @ [x, y, z]: [u32; 3]) { + self.prepare_dispatch(count); + unsafe { self.list.as_ref().unwrap().Dispatch(x, y, z) } + } + + unsafe fn dispatch_indirect(&mut self, buffer: &super::Buffer, offset: wgt::BufferAddress) { + if self + .pass + .layout + .special_constants + .as_ref() + .and_then(|sc| sc.indirect_cmd_signatures.as_ref()) + .is_some() + { + self.update_root_elements(); + } else { + self.prepare_dispatch([0; 3]); + } + + let cmd_signature = &self + .pass + .layout + .special_constants + .as_ref() + .and_then(|sc| sc.indirect_cmd_signatures.as_ref()) + .unwrap_or_else(|| &self.shared.cmd_signatures) + .dispatch; + unsafe { + self.list.as_ref().unwrap().ExecuteIndirect( + cmd_signature, + 1, + &buffer.resource, + offset, + None, + 0, + ) + } + } + + unsafe fn build_acceleration_structures<'a, T>( + &mut self, + _descriptor_count: u32, + descriptors: T, + ) where + super::Api: 'a, + T: IntoIterator< + Item = crate::BuildAccelerationStructureDescriptor< + 'a, + super::Buffer, + super::AccelerationStructure, + >, + >, + { + // Implement using `BuildRaytracingAccelerationStructure`: + // https://microsoft.github.io/DirectX-Specs/d3d/Raytracing.html#buildraytracingaccelerationstructure + let list = self + .list + .as_ref() + .unwrap() + .cast::() + .unwrap(); + for descriptor in descriptors { + // TODO: This is the same as getting build sizes apart from requiring buffers, should this be de-duped? + let mut geometry_desc; + let ty; + let inputs0; + let num_desc; + match descriptor.entries { + AccelerationStructureEntries::Instances(instances) => { + let desc_address = unsafe { + instances + .buffer + .expect("needs buffer to build") + .resource + .GetGPUVirtualAddress() + } + instances.offset as u64; + ty = Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL; + inputs0 = Direct3D12::D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS_0 { + InstanceDescs: desc_address, + }; + num_desc = instances.count; + } + AccelerationStructureEntries::Triangles(triangles) => { + geometry_desc = Vec::with_capacity(triangles.len()); + for triangle in triangles { + let transform_address = + triangle.transform.as_ref().map_or(0, |transform| unsafe { + transform.buffer.resource.GetGPUVirtualAddress() + + transform.offset as u64 + }); + let index_format = triangle + .indices + .as_ref() + .map_or(Dxgi::Common::DXGI_FORMAT_UNKNOWN, |indices| { + auxil::dxgi::conv::map_index_format(indices.format) + }); + let vertex_format = + auxil::dxgi::conv::map_vertex_format(triangle.vertex_format); + let index_count = + triangle.indices.as_ref().map_or(0, |indices| indices.count); + let index_address = triangle.indices.as_ref().map_or(0, |indices| unsafe { + indices + .buffer + .expect("needs buffer to build") + .resource + .GetGPUVirtualAddress() + + indices.offset as u64 + }); + let vertex_address = unsafe { + triangle + .vertex_buffer + .expect("needs buffer to build") + .resource + .GetGPUVirtualAddress() + + (triangle.first_vertex as u64 * triangle.vertex_stride) + }; + + let triangle_desc = Direct3D12::D3D12_RAYTRACING_GEOMETRY_TRIANGLES_DESC { + Transform3x4: transform_address, + IndexFormat: index_format, + VertexFormat: vertex_format, + IndexCount: index_count, + VertexCount: triangle.vertex_count, + IndexBuffer: index_address, + VertexBuffer: Direct3D12::D3D12_GPU_VIRTUAL_ADDRESS_AND_STRIDE { + StartAddress: vertex_address, + StrideInBytes: triangle.vertex_stride, + }, + }; + + geometry_desc.push(Direct3D12::D3D12_RAYTRACING_GEOMETRY_DESC { + Type: Direct3D12::D3D12_RAYTRACING_GEOMETRY_TYPE_TRIANGLES, + Flags: conv::map_acceleration_structure_geometry_flags(triangle.flags), + Anonymous: Direct3D12::D3D12_RAYTRACING_GEOMETRY_DESC_0 { + Triangles: triangle_desc, + }, + }) + } + ty = Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL; + inputs0 = Direct3D12::D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS_0 { + pGeometryDescs: geometry_desc.as_ptr(), + }; + num_desc = geometry_desc.len() as u32; + } + AccelerationStructureEntries::AABBs(aabbs) => { + geometry_desc = Vec::with_capacity(aabbs.len()); + for aabb in aabbs { + let aabb_address = unsafe { + aabb.buffer + .expect("needs buffer to build") + .resource + .GetGPUVirtualAddress() + + (aabb.offset as u64 * aabb.stride) + }; + + let aabb_desc = Direct3D12::D3D12_RAYTRACING_GEOMETRY_AABBS_DESC { + AABBCount: aabb.count as u64, + AABBs: Direct3D12::D3D12_GPU_VIRTUAL_ADDRESS_AND_STRIDE { + StartAddress: aabb_address, + StrideInBytes: aabb.stride, + }, + }; + + geometry_desc.push(Direct3D12::D3D12_RAYTRACING_GEOMETRY_DESC { + Type: Direct3D12::D3D12_RAYTRACING_GEOMETRY_TYPE_PROCEDURAL_PRIMITIVE_AABBS, + Flags: conv::map_acceleration_structure_geometry_flags(aabb.flags), + Anonymous: Direct3D12::D3D12_RAYTRACING_GEOMETRY_DESC_0 { + AABBs: aabb_desc, + }, + }) + } + ty = Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL; + inputs0 = Direct3D12::D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS_0 { + pGeometryDescs: geometry_desc.as_ptr(), + }; + num_desc = geometry_desc.len() as u32; + } + }; + let acceleration_structure_inputs = + Direct3D12::D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS { + Type: ty, + Flags: conv::map_acceleration_structure_build_flags( + descriptor.flags, + Some(descriptor.mode), + ), + NumDescs: num_desc, + DescsLayout: Direct3D12::D3D12_ELEMENTS_LAYOUT_ARRAY, + Anonymous: inputs0, + }; + + let dst_acceleration_structure_address = unsafe { + descriptor + .destination_acceleration_structure + .resource + .GetGPUVirtualAddress() + }; + let src_acceleration_structure_address = descriptor + .source_acceleration_structure + .as_ref() + .map_or(0, |source| unsafe { + source.resource.GetGPUVirtualAddress() + }); + let scratch_address = unsafe { + descriptor.scratch_buffer.resource.GetGPUVirtualAddress() + + descriptor.scratch_buffer_offset + }; + + let desc = Direct3D12::D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_DESC { + DestAccelerationStructureData: dst_acceleration_structure_address, + Inputs: acceleration_structure_inputs, + SourceAccelerationStructureData: src_acceleration_structure_address, + ScratchAccelerationStructureData: scratch_address, + }; + unsafe { list.BuildRaytracingAccelerationStructure(&desc, None) }; + } + } + + unsafe fn place_acceleration_structure_barrier( + &mut self, + _barriers: crate::AccelerationStructureBarrier, + ) { + // TODO: This is not very optimal, we should be using [enhanced barriers](https://microsoft.github.io/DirectX-Specs/d3d/D3D12EnhancedBarriers.html) if possible + let list = self + .list + .as_ref() + .unwrap() + .cast::() + .unwrap(); + unsafe { + list.ResourceBarrier(&[Direct3D12::D3D12_RESOURCE_BARRIER { + Type: Direct3D12::D3D12_RESOURCE_BARRIER_TYPE_UAV, + Flags: Direct3D12::D3D12_RESOURCE_BARRIER_FLAG_NONE, + Anonymous: Direct3D12::D3D12_RESOURCE_BARRIER_0 { + UAV: mem::ManuallyDrop::new(Direct3D12::D3D12_RESOURCE_UAV_BARRIER { + pResource: Default::default(), + }), + }, + }]) + } + } + + unsafe fn copy_acceleration_structure_to_acceleration_structure( + &mut self, + src: &super::AccelerationStructure, + dst: &super::AccelerationStructure, + copy: wgt::AccelerationStructureCopy, + ) { + let list = self + .list + .as_ref() + .unwrap() + .cast::() + .unwrap(); + unsafe { + list.CopyRaytracingAccelerationStructure( + dst.resource.GetGPUVirtualAddress(), + src.resource.GetGPUVirtualAddress(), + conv::map_acceleration_structure_copy_mode(copy), + ) + } + } + + unsafe fn set_acceleration_structure_dependencies( + _command_buffers: &[&super::CommandBuffer], + _dependencies: &[&super::AccelerationStructure], + ) { + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/conv.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/conv.rs new file mode 100644 index 00000000..064700b5 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/conv.rs @@ -0,0 +1,449 @@ +use windows::Win32::Graphics::{Direct3D, Direct3D12, Dxgi}; + +pub fn map_buffer_usage_to_resource_flags( + usage: wgt::BufferUses, +) -> Direct3D12::D3D12_RESOURCE_FLAGS { + let mut flags = Direct3D12::D3D12_RESOURCE_FLAG_NONE; + if usage.contains(wgt::BufferUses::STORAGE_READ_WRITE) + || usage.contains(wgt::BufferUses::ACCELERATION_STRUCTURE_QUERY) + { + flags |= Direct3D12::D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; + } + flags +} + +pub fn map_buffer_descriptor( + desc: &crate::BufferDescriptor<'_>, +) -> Direct3D12::D3D12_RESOURCE_DESC { + Direct3D12::D3D12_RESOURCE_DESC { + Dimension: Direct3D12::D3D12_RESOURCE_DIMENSION_BUFFER, + Alignment: 0, + Width: desc.size, + Height: 1, + DepthOrArraySize: 1, + MipLevels: 1, + Format: Dxgi::Common::DXGI_FORMAT_UNKNOWN, + SampleDesc: Dxgi::Common::DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Layout: Direct3D12::D3D12_TEXTURE_LAYOUT_ROW_MAJOR, + Flags: map_buffer_usage_to_resource_flags(desc.usage), + } +} + +pub fn map_texture_dimension(dim: wgt::TextureDimension) -> Direct3D12::D3D12_RESOURCE_DIMENSION { + match dim { + wgt::TextureDimension::D1 => Direct3D12::D3D12_RESOURCE_DIMENSION_TEXTURE1D, + wgt::TextureDimension::D2 => Direct3D12::D3D12_RESOURCE_DIMENSION_TEXTURE2D, + wgt::TextureDimension::D3 => Direct3D12::D3D12_RESOURCE_DIMENSION_TEXTURE3D, + } +} + +pub fn map_texture_usage_to_resource_flags( + usage: wgt::TextureUses, +) -> Direct3D12::D3D12_RESOURCE_FLAGS { + let mut flags = Direct3D12::D3D12_RESOURCE_FLAG_NONE; + + if usage.contains(wgt::TextureUses::COLOR_TARGET) { + flags |= Direct3D12::D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET; + } + if usage + .intersects(wgt::TextureUses::DEPTH_STENCIL_READ | wgt::TextureUses::DEPTH_STENCIL_WRITE) + { + flags |= Direct3D12::D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL; + if !usage.contains(wgt::TextureUses::RESOURCE) { + flags |= Direct3D12::D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE; + } + } + if usage.intersects( + wgt::TextureUses::STORAGE_READ_ONLY + | wgt::TextureUses::STORAGE_WRITE_ONLY + | wgt::TextureUses::STORAGE_READ_WRITE, + ) { + flags |= Direct3D12::D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; + } + + flags +} + +pub fn map_address_mode(mode: wgt::AddressMode) -> Direct3D12::D3D12_TEXTURE_ADDRESS_MODE { + use wgt::AddressMode as Am; + match mode { + Am::Repeat => Direct3D12::D3D12_TEXTURE_ADDRESS_MODE_WRAP, + Am::MirrorRepeat => Direct3D12::D3D12_TEXTURE_ADDRESS_MODE_MIRROR, + Am::ClampToEdge => Direct3D12::D3D12_TEXTURE_ADDRESS_MODE_CLAMP, + Am::ClampToBorder => Direct3D12::D3D12_TEXTURE_ADDRESS_MODE_BORDER, + //Am::MirrorClamp => Direct3D12::D3D12_TEXTURE_ADDRESS_MODE_MIRROR_ONCE, + } +} + +pub fn map_filter_mode(mode: wgt::FilterMode) -> Direct3D12::D3D12_FILTER_TYPE { + match mode { + wgt::FilterMode::Nearest => Direct3D12::D3D12_FILTER_TYPE_POINT, + wgt::FilterMode::Linear => Direct3D12::D3D12_FILTER_TYPE_LINEAR, + } +} + +pub fn map_mipmap_filter_mode(mode: wgt::MipmapFilterMode) -> Direct3D12::D3D12_FILTER_TYPE { + match mode { + wgt::MipmapFilterMode::Nearest => Direct3D12::D3D12_FILTER_TYPE_POINT, + wgt::MipmapFilterMode::Linear => Direct3D12::D3D12_FILTER_TYPE_LINEAR, + } +} + +pub fn map_comparison(func: wgt::CompareFunction) -> Direct3D12::D3D12_COMPARISON_FUNC { + use wgt::CompareFunction as Cf; + match func { + Cf::Never => Direct3D12::D3D12_COMPARISON_FUNC_NEVER, + Cf::Less => Direct3D12::D3D12_COMPARISON_FUNC_LESS, + Cf::LessEqual => Direct3D12::D3D12_COMPARISON_FUNC_LESS_EQUAL, + Cf::Equal => Direct3D12::D3D12_COMPARISON_FUNC_EQUAL, + Cf::GreaterEqual => Direct3D12::D3D12_COMPARISON_FUNC_GREATER_EQUAL, + Cf::Greater => Direct3D12::D3D12_COMPARISON_FUNC_GREATER, + Cf::NotEqual => Direct3D12::D3D12_COMPARISON_FUNC_NOT_EQUAL, + Cf::Always => Direct3D12::D3D12_COMPARISON_FUNC_ALWAYS, + } +} + +pub fn map_border_color(border_color: Option) -> [f32; 4] { + use wgt::SamplerBorderColor as Sbc; + match border_color { + Some(Sbc::TransparentBlack) | Some(Sbc::Zero) | None => [0.0; 4], + Some(Sbc::OpaqueBlack) => [0.0, 0.0, 0.0, 1.0], + Some(Sbc::OpaqueWhite) => [1.0; 4], + } +} + +pub fn map_visibility(visibility: wgt::ShaderStages) -> Direct3D12::D3D12_SHADER_VISIBILITY { + match visibility { + wgt::ShaderStages::VERTEX => Direct3D12::D3D12_SHADER_VISIBILITY_VERTEX, + wgt::ShaderStages::FRAGMENT => Direct3D12::D3D12_SHADER_VISIBILITY_PIXEL, + _ => Direct3D12::D3D12_SHADER_VISIBILITY_ALL, + } +} + +pub fn map_binding_type(ty: &wgt::BindingType) -> Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE { + use wgt::BindingType as Bt; + match *ty { + Bt::Sampler { .. } => Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_SAMPLER, + Bt::Buffer { + ty: wgt::BufferBindingType::Uniform, + .. + } => Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_CBV, + Bt::Buffer { + ty: wgt::BufferBindingType::Storage { read_only: true }, + .. + } + | Bt::Texture { .. } => Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_SRV, + Bt::Buffer { + ty: wgt::BufferBindingType::Storage { read_only: false }, + .. + } + | Bt::StorageTexture { .. } => Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_UAV, + Bt::AccelerationStructure { .. } => Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_SRV, + // External textures require multiple bindings and therefore cannot + // be mapped to a single descriptor range type. They must be handled + // separately by the caller. + Bt::ExternalTexture => unreachable!("External textures must be handled separately"), + } +} + +pub fn map_buffer_usage_to_state(usage: wgt::BufferUses) -> Direct3D12::D3D12_RESOURCE_STATES { + use wgt::BufferUses as Bu; + let mut state = Direct3D12::D3D12_RESOURCE_STATE_COMMON; + + if usage.intersects(Bu::COPY_SRC) { + state |= Direct3D12::D3D12_RESOURCE_STATE_COPY_SOURCE; + } + if usage.intersects(Bu::COPY_DST) { + state |= Direct3D12::D3D12_RESOURCE_STATE_COPY_DEST; + } + if usage.intersects(Bu::INDEX) { + state |= Direct3D12::D3D12_RESOURCE_STATE_INDEX_BUFFER; + } + if usage.intersects(Bu::VERTEX | Bu::UNIFORM) { + state |= Direct3D12::D3D12_RESOURCE_STATE_VERTEX_AND_CONSTANT_BUFFER; + } + if usage.intersects(Bu::STORAGE_READ_WRITE) { + state |= Direct3D12::D3D12_RESOURCE_STATE_UNORDERED_ACCESS; + } else if usage.intersects(Bu::STORAGE_READ_ONLY) { + state |= Direct3D12::D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE + | Direct3D12::D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE; + } + if usage.intersects(Bu::INDIRECT) { + state |= Direct3D12::D3D12_RESOURCE_STATE_INDIRECT_ARGUMENT; + } + if usage.intersects(Bu::ACCELERATION_STRUCTURE_QUERY) { + state |= Direct3D12::D3D12_RESOURCE_STATE_UNORDERED_ACCESS; + } + state +} + +pub fn map_texture_usage_to_state(usage: wgt::TextureUses) -> Direct3D12::D3D12_RESOURCE_STATES { + use wgt::TextureUses as Tu; + let mut state = Direct3D12::D3D12_RESOURCE_STATE_COMMON; + //Note: `RESOLVE_SOURCE` and `RESOLVE_DEST` are not used here + //Note: `PRESENT` is the same as `COMMON` + if usage == wgt::TextureUses::UNINITIALIZED { + return state; + } + + if usage.intersects(Tu::COPY_SRC) { + state |= Direct3D12::D3D12_RESOURCE_STATE_COPY_SOURCE; + } + if usage.intersects(Tu::COPY_DST) { + state |= Direct3D12::D3D12_RESOURCE_STATE_COPY_DEST; + } + if usage.intersects(Tu::RESOURCE) { + state |= Direct3D12::D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE + | Direct3D12::D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE; + } + if usage.intersects(Tu::COLOR_TARGET) { + state |= Direct3D12::D3D12_RESOURCE_STATE_RENDER_TARGET; + } + if usage.intersects(Tu::DEPTH_STENCIL_READ) { + state |= Direct3D12::D3D12_RESOURCE_STATE_DEPTH_READ; + } + if usage.intersects(Tu::DEPTH_STENCIL_WRITE) { + state |= Direct3D12::D3D12_RESOURCE_STATE_DEPTH_WRITE; + } + if usage.intersects(Tu::STORAGE_READ_ONLY | Tu::STORAGE_WRITE_ONLY | Tu::STORAGE_READ_WRITE) { + state |= Direct3D12::D3D12_RESOURCE_STATE_UNORDERED_ACCESS; + } + state +} + +pub fn map_topology( + topology: wgt::PrimitiveTopology, +) -> ( + Direct3D12::D3D12_PRIMITIVE_TOPOLOGY_TYPE, + Direct3D::D3D_PRIMITIVE_TOPOLOGY, +) { + match topology { + wgt::PrimitiveTopology::PointList => ( + Direct3D12::D3D12_PRIMITIVE_TOPOLOGY_TYPE_POINT, + Direct3D::D3D_PRIMITIVE_TOPOLOGY_POINTLIST, + ), + wgt::PrimitiveTopology::LineList => ( + Direct3D12::D3D12_PRIMITIVE_TOPOLOGY_TYPE_LINE, + Direct3D::D3D_PRIMITIVE_TOPOLOGY_LINELIST, + ), + wgt::PrimitiveTopology::LineStrip => ( + Direct3D12::D3D12_PRIMITIVE_TOPOLOGY_TYPE_LINE, + Direct3D::D3D_PRIMITIVE_TOPOLOGY_LINESTRIP, + ), + wgt::PrimitiveTopology::TriangleList => ( + Direct3D12::D3D12_PRIMITIVE_TOPOLOGY_TYPE_TRIANGLE, + Direct3D::D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST, + ), + wgt::PrimitiveTopology::TriangleStrip => ( + Direct3D12::D3D12_PRIMITIVE_TOPOLOGY_TYPE_TRIANGLE, + Direct3D::D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP, + ), + } +} + +pub fn map_polygon_mode(mode: wgt::PolygonMode) -> Direct3D12::D3D12_FILL_MODE { + match mode { + wgt::PolygonMode::Fill => Direct3D12::D3D12_FILL_MODE_SOLID, + wgt::PolygonMode::Line => Direct3D12::D3D12_FILL_MODE_WIREFRAME, + wgt::PolygonMode::Point => panic!( + "{:?} is not enabled for this backend", + wgt::Features::POLYGON_MODE_POINT + ), + } +} + +/// D3D12 doesn't support passing factors ending in `_COLOR` for alpha blending +/// (see ). +/// Therefore this function takes an additional `is_alpha` argument +/// which if set will return an equivalent `_ALPHA` factor. +fn map_blend_factor(factor: wgt::BlendFactor, is_alpha: bool) -> Direct3D12::D3D12_BLEND { + use wgt::BlendFactor as Bf; + match factor { + Bf::Zero => Direct3D12::D3D12_BLEND_ZERO, + Bf::One => Direct3D12::D3D12_BLEND_ONE, + Bf::Src if is_alpha => Direct3D12::D3D12_BLEND_SRC_ALPHA, + Bf::Src => Direct3D12::D3D12_BLEND_SRC_COLOR, + Bf::OneMinusSrc if is_alpha => Direct3D12::D3D12_BLEND_INV_SRC_ALPHA, + Bf::OneMinusSrc => Direct3D12::D3D12_BLEND_INV_SRC_COLOR, + Bf::Dst if is_alpha => Direct3D12::D3D12_BLEND_DEST_ALPHA, + Bf::Dst => Direct3D12::D3D12_BLEND_DEST_COLOR, + Bf::OneMinusDst if is_alpha => Direct3D12::D3D12_BLEND_INV_DEST_ALPHA, + Bf::OneMinusDst => Direct3D12::D3D12_BLEND_INV_DEST_COLOR, + Bf::SrcAlpha => Direct3D12::D3D12_BLEND_SRC_ALPHA, + Bf::OneMinusSrcAlpha => Direct3D12::D3D12_BLEND_INV_SRC_ALPHA, + Bf::DstAlpha => Direct3D12::D3D12_BLEND_DEST_ALPHA, + Bf::OneMinusDstAlpha => Direct3D12::D3D12_BLEND_INV_DEST_ALPHA, + Bf::Constant => Direct3D12::D3D12_BLEND_BLEND_FACTOR, + Bf::OneMinusConstant => Direct3D12::D3D12_BLEND_INV_BLEND_FACTOR, + Bf::SrcAlphaSaturated => Direct3D12::D3D12_BLEND_SRC_ALPHA_SAT, + Bf::Src1 if is_alpha => Direct3D12::D3D12_BLEND_SRC1_ALPHA, + Bf::Src1 => Direct3D12::D3D12_BLEND_SRC1_COLOR, + Bf::OneMinusSrc1 if is_alpha => Direct3D12::D3D12_BLEND_INV_SRC1_ALPHA, + Bf::OneMinusSrc1 => Direct3D12::D3D12_BLEND_INV_SRC1_COLOR, + Bf::Src1Alpha => Direct3D12::D3D12_BLEND_SRC1_ALPHA, + Bf::OneMinusSrc1Alpha => Direct3D12::D3D12_BLEND_INV_SRC1_ALPHA, + } +} + +fn map_blend_component( + component: &wgt::BlendComponent, + is_alpha: bool, +) -> ( + Direct3D12::D3D12_BLEND_OP, + Direct3D12::D3D12_BLEND, + Direct3D12::D3D12_BLEND, +) { + let raw_op = match component.operation { + wgt::BlendOperation::Add => Direct3D12::D3D12_BLEND_OP_ADD, + wgt::BlendOperation::Subtract => Direct3D12::D3D12_BLEND_OP_SUBTRACT, + wgt::BlendOperation::ReverseSubtract => Direct3D12::D3D12_BLEND_OP_REV_SUBTRACT, + wgt::BlendOperation::Min => Direct3D12::D3D12_BLEND_OP_MIN, + wgt::BlendOperation::Max => Direct3D12::D3D12_BLEND_OP_MAX, + }; + let raw_src = map_blend_factor(component.src_factor, is_alpha); + let raw_dst = map_blend_factor(component.dst_factor, is_alpha); + (raw_op, raw_src, raw_dst) +} + +pub fn map_render_targets( + color_targets: &[Option], +) -> [Direct3D12::D3D12_RENDER_TARGET_BLEND_DESC; + Direct3D12::D3D12_SIMULTANEOUS_RENDER_TARGET_COUNT as usize] { + let dummy_target = Direct3D12::D3D12_RENDER_TARGET_BLEND_DESC { + BlendEnable: false.into(), + LogicOpEnable: false.into(), + SrcBlend: Direct3D12::D3D12_BLEND_ZERO, + DestBlend: Direct3D12::D3D12_BLEND_ZERO, + BlendOp: Direct3D12::D3D12_BLEND_OP_ADD, + SrcBlendAlpha: Direct3D12::D3D12_BLEND_ZERO, + DestBlendAlpha: Direct3D12::D3D12_BLEND_ZERO, + BlendOpAlpha: Direct3D12::D3D12_BLEND_OP_ADD, + LogicOp: Direct3D12::D3D12_LOGIC_OP_CLEAR, + RenderTargetWriteMask: 0, + }; + let mut raw_targets = + [dummy_target; Direct3D12::D3D12_SIMULTANEOUS_RENDER_TARGET_COUNT as usize]; + + for (raw, ct) in raw_targets.iter_mut().zip(color_targets.iter()) { + if let Some(ct) = ct.as_ref() { + raw.RenderTargetWriteMask = ct.write_mask.bits() as u8; + if let Some(ref blend) = ct.blend { + let (color_op, color_src, color_dst) = map_blend_component(&blend.color, false); + let (alpha_op, alpha_src, alpha_dst) = map_blend_component(&blend.alpha, true); + raw.BlendEnable = true.into(); + raw.BlendOp = color_op; + raw.SrcBlend = color_src; + raw.DestBlend = color_dst; + raw.BlendOpAlpha = alpha_op; + raw.SrcBlendAlpha = alpha_src; + raw.DestBlendAlpha = alpha_dst; + } + } + } + + raw_targets +} + +fn map_stencil_op(op: wgt::StencilOperation) -> Direct3D12::D3D12_STENCIL_OP { + use wgt::StencilOperation as So; + match op { + So::Keep => Direct3D12::D3D12_STENCIL_OP_KEEP, + So::Zero => Direct3D12::D3D12_STENCIL_OP_ZERO, + So::Replace => Direct3D12::D3D12_STENCIL_OP_REPLACE, + So::IncrementClamp => Direct3D12::D3D12_STENCIL_OP_INCR_SAT, + So::IncrementWrap => Direct3D12::D3D12_STENCIL_OP_INCR, + So::DecrementClamp => Direct3D12::D3D12_STENCIL_OP_DECR_SAT, + So::DecrementWrap => Direct3D12::D3D12_STENCIL_OP_DECR, + So::Invert => Direct3D12::D3D12_STENCIL_OP_INVERT, + } +} + +fn map_stencil_face(face: &wgt::StencilFaceState) -> Direct3D12::D3D12_DEPTH_STENCILOP_DESC { + Direct3D12::D3D12_DEPTH_STENCILOP_DESC { + StencilFailOp: map_stencil_op(face.fail_op), + StencilDepthFailOp: map_stencil_op(face.depth_fail_op), + StencilPassOp: map_stencil_op(face.pass_op), + StencilFunc: map_comparison(face.compare), + } +} + +pub fn map_depth_stencil(ds: &wgt::DepthStencilState) -> Direct3D12::D3D12_DEPTH_STENCIL_DESC { + Direct3D12::D3D12_DEPTH_STENCIL_DESC { + DepthEnable: ds.is_depth_enabled().into(), + DepthWriteMask: if ds.depth_write_enabled.unwrap_or_default() { + Direct3D12::D3D12_DEPTH_WRITE_MASK_ALL + } else { + Direct3D12::D3D12_DEPTH_WRITE_MASK_ZERO + }, + DepthFunc: map_comparison(ds.depth_compare.unwrap_or_default()), + StencilEnable: ds.stencil.is_enabled().into(), + StencilReadMask: ds.stencil.read_mask as u8, + StencilWriteMask: ds.stencil.write_mask as u8, + FrontFace: map_stencil_face(&ds.stencil.front), + BackFace: map_stencil_face(&ds.stencil.back), + } +} + +pub(crate) fn map_acceleration_structure_build_flags( + flags: wgt::AccelerationStructureFlags, + mode: Option, +) -> Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_BUILD_FLAGS { + let mut d3d_flags = Default::default(); + if flags.contains(wgt::AccelerationStructureFlags::ALLOW_COMPACTION) { + d3d_flags |= + Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_BUILD_FLAG_ALLOW_COMPACTION; + } + + if flags.contains(wgt::AccelerationStructureFlags::ALLOW_UPDATE) { + d3d_flags |= Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_BUILD_FLAG_ALLOW_UPDATE; + } + + if flags.contains(wgt::AccelerationStructureFlags::LOW_MEMORY) { + d3d_flags |= Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_BUILD_FLAG_MINIMIZE_MEMORY; + } + + if flags.contains(wgt::AccelerationStructureFlags::PREFER_FAST_BUILD) { + d3d_flags |= + Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_BUILD_FLAG_PREFER_FAST_BUILD; + } + + if flags.contains(wgt::AccelerationStructureFlags::PREFER_FAST_TRACE) { + d3d_flags |= + Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_BUILD_FLAG_PREFER_FAST_TRACE; + } + + if let Some(crate::AccelerationStructureBuildMode::Update) = mode { + d3d_flags |= Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_BUILD_FLAG_PERFORM_UPDATE + } + + d3d_flags +} + +pub(crate) fn map_acceleration_structure_geometry_flags( + flags: wgt::AccelerationStructureGeometryFlags, +) -> Direct3D12::D3D12_RAYTRACING_GEOMETRY_FLAGS { + let mut d3d_flags = Default::default(); + if flags.contains(wgt::AccelerationStructureGeometryFlags::OPAQUE) { + d3d_flags |= Direct3D12::D3D12_RAYTRACING_GEOMETRY_FLAG_OPAQUE; + } + if flags.contains(wgt::AccelerationStructureGeometryFlags::NO_DUPLICATE_ANY_HIT_INVOCATION) { + d3d_flags |= Direct3D12::D3D12_RAYTRACING_GEOMETRY_FLAG_NO_DUPLICATE_ANYHIT_INVOCATION; + } + d3d_flags +} + +pub(crate) fn map_acceleration_structure_copy_mode( + mode: wgt::AccelerationStructureCopy, +) -> Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE { + match mode { + wgt::AccelerationStructureCopy::Clone => { + Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE_CLONE + } + wgt::AccelerationStructureCopy::Compact => { + Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE_COMPACT + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/dcomp.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/dcomp.rs new file mode 100644 index 00000000..46ef3da4 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/dcomp.rs @@ -0,0 +1,120 @@ +use alloc::sync::Arc; +use core::{ffi, ptr}; + +use once_cell::sync::Lazy; +use windows::{ + core::Interface as _, + Win32::{Foundation::HWND, Graphics::DirectComposition}, +}; + +use super::DynLib; + +// Lazy-loaded DirectComposition library +#[derive(Debug)] +pub(crate) struct DCompLib { + lib: Lazy>, +} + +impl DCompLib { + pub(crate) fn new() -> Self { + Self { + lib: Lazy::new(|| unsafe { + DynLib::new("dcomp.dll").map_err(|err| { + log::error!("Error loading dcomp.dll: {err}"); + crate::SurfaceError::Other("Error loading dcomp.dll") + }) + }), + } + } + + fn get_lib(&self) -> Result<&DynLib, crate::SurfaceError> { + match self.lib.as_ref() { + Ok(lib) => Ok(lib), + Err(err) => Err(err.clone()), + } + } + + pub(crate) fn create_device( + &self, + ) -> Result { + let lib = self.get_lib()?; + + // Calls windows::Win32::Graphics::DirectComposition::DCompositionCreateDevice2 on dcomp.dll + type Fun = extern "system" fn( + pdxdevice: *mut ffi::c_void, + riid: *const windows_core::GUID, + ppdcompdevice: *mut *mut ffi::c_void, + ) -> windows_core::HRESULT; + let func: libloading::Symbol = + unsafe { lib.get(c"DCompositionCreateDevice2".to_bytes()) }?; + + let mut res: Option = None; + + (func)( + ptr::null_mut(), + &DirectComposition::IDCompositionDevice::IID, + <*mut _>::cast(&mut res), + ) + .map(|| res.unwrap()) + .map_err(|err| { + log::error!("DirectComposition::DCompositionCreateDevice2 failed: {err}"); + crate::SurfaceError::Other("DirectComposition::DCompositionCreateDevice2") + }) + } +} + +#[derive(Default)] +pub struct DCompState { + inner: Option, +} + +impl DCompState { + /// This will create a DirectComposition device and a target for the window handle if not already initialized. + /// If the device is already initialized, it will return the existing state. + pub unsafe fn get_or_init( + &mut self, + lib: &Arc, + hwnd: &HWND, + ) -> Result<&mut InnerState, crate::SurfaceError> { + if self.inner.is_none() { + self.inner = Some(unsafe { InnerState::init(lib, hwnd) }?); + } + Ok(self.inner.as_mut().unwrap()) + } +} + +pub struct InnerState { + pub visual: DirectComposition::IDCompositionVisual, + pub device: DirectComposition::IDCompositionDevice, + // Must be kept alive but is otherwise unused after initialization. + pub _target: DirectComposition::IDCompositionTarget, +} + +impl InnerState { + /// Creates a DirectComposition device and a target for the given window handle. + pub unsafe fn init(lib: &Arc, hwnd: &HWND) -> Result { + profiling::scope!("DCompState::init"); + let dcomp_device = lib.create_device()?; + + let target = unsafe { dcomp_device.CreateTargetForHwnd(*hwnd, false) }.map_err(|err| { + log::error!("IDCompositionDevice::CreateTargetForHwnd failed: {err}"); + crate::SurfaceError::Other("IDCompositionDevice::CreateTargetForHwnd") + })?; + + let visual = unsafe { dcomp_device.CreateVisual() }.map_err(|err| { + log::error!("IDCompositionDevice::CreateVisual failed: {err}"); + crate::SurfaceError::Other("IDCompositionDevice::CreateVisual") + })?; + + unsafe { target.SetRoot(&visual) }.map_err(|err| { + log::error!("IDCompositionTarget::SetRoot failed: {err}"); + crate::SurfaceError::Other("IDCompositionTarget::SetRoot") + })?; + + Ok(InnerState { + visual, + device: dcomp_device, + _target: target, + }) + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/descriptor.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/descriptor.rs new file mode 100644 index 00000000..f57ba46e --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/descriptor.rs @@ -0,0 +1,329 @@ +use alloc::vec::Vec; +use core::fmt; + +use bit_set::BitSet; +use parking_lot::Mutex; +use range_alloc::RangeAllocator; +use windows::Win32::Graphics::Direct3D12; + +use crate::auxil::dxgi::result::HResult as _; + +const HEAP_SIZE_FIXED: usize = 64; + +#[derive(Copy, Clone)] +pub(super) struct DualHandle { + cpu: Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE, + pub gpu: Direct3D12::D3D12_GPU_DESCRIPTOR_HANDLE, + /// How large the block allocated to this handle is. + count: u64, +} + +impl fmt::Debug for DualHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DualHandle") + .field("cpu", &self.cpu.ptr) + .field("gpu", &self.gpu.ptr) + .field("count", &self.count) + .finish() + } +} + +type DescriptorIndex = u64; + +pub(super) struct GeneralHeap { + pub raw: Direct3D12::ID3D12DescriptorHeap, + ty: Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE, + handle_size: u64, + total_handles: u64, + start: DualHandle, + ranges: Mutex>, +} + +impl GeneralHeap { + pub(super) fn new( + device: &Direct3D12::ID3D12Device, + ty: Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE, + total_handles: u64, + ) -> Result { + let raw = { + profiling::scope!("ID3D12Device::CreateDescriptorHeap"); + let desc = Direct3D12::D3D12_DESCRIPTOR_HEAP_DESC { + Type: ty, + NumDescriptors: total_handles as u32, + Flags: Direct3D12::D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE, + NodeMask: 0, + }; + unsafe { device.CreateDescriptorHeap::(&desc) } + .into_device_result("Descriptor heap creation")? + }; + + let start = DualHandle { + cpu: unsafe { raw.GetCPUDescriptorHandleForHeapStart() }, + gpu: unsafe { raw.GetGPUDescriptorHandleForHeapStart() }, + count: 0, + }; + + Ok(Self { + raw, + ty, + handle_size: unsafe { device.GetDescriptorHandleIncrementSize(ty) } as u64, + total_handles, + start, + ranges: Mutex::new(RangeAllocator::new(0..total_handles)), + }) + } + + pub(super) fn at(&self, index: DescriptorIndex, count: u64) -> DualHandle { + assert!(index < self.total_handles); + DualHandle { + cpu: self.cpu_descriptor_at(index), + gpu: self.gpu_descriptor_at(index), + count, + } + } + + fn cpu_descriptor_at(&self, index: u64) -> Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE { + Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE { + ptr: self.start.cpu.ptr + (self.handle_size * index) as usize, + } + } + + fn gpu_descriptor_at(&self, index: u64) -> Direct3D12::D3D12_GPU_DESCRIPTOR_HANDLE { + Direct3D12::D3D12_GPU_DESCRIPTOR_HANDLE { + ptr: self.start.gpu.ptr + self.handle_size * index, + } + } + + pub(super) fn allocate_slice(&self, count: u64) -> Result { + let range = self.ranges.lock().allocate_range(count).map_err(|err| { + log::error!("Unable to allocate descriptors: {err:?}"); + crate::DeviceError::OutOfMemory + })?; + Ok(range.start) + } + + /// Free handles previously given out by this `DescriptorHeapSlice`. + /// Do not use this with handles not given out by this `DescriptorHeapSlice`. + pub(crate) fn free_slice(&self, handle: DualHandle) { + let start = (handle.gpu.ptr - self.start.gpu.ptr) / self.handle_size; + self.ranges.lock().free_range(start..start + handle.count); + } +} + +/// Fixed-size free-list allocator for CPU descriptors. +struct FixedSizeHeap { + _raw: Direct3D12::ID3D12DescriptorHeap, + /// Bit flag representation of available handles in the heap. + /// + /// 0 - Occupied + /// 1 - free + availability: u64, + handle_size: usize, + start: Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE, +} + +impl FixedSizeHeap { + fn new( + device: &Direct3D12::ID3D12Device, + ty: Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE, + ) -> Result { + let desc = Direct3D12::D3D12_DESCRIPTOR_HEAP_DESC { + Type: ty, + NumDescriptors: HEAP_SIZE_FIXED as u32, + Flags: Direct3D12::D3D12_DESCRIPTOR_HEAP_FLAG_NONE, + NodeMask: 0, + }; + let heap = + unsafe { device.CreateDescriptorHeap::(&desc) } + .into_device_result("Descriptor heap creation")?; + + Ok(Self { + handle_size: unsafe { device.GetDescriptorHandleIncrementSize(ty) } as usize, + availability: !0, // all free! + start: unsafe { heap.GetCPUDescriptorHandleForHeapStart() }, + _raw: heap, + }) + } + + fn alloc_handle( + &mut self, + ) -> Result { + // Find first free slot. + let slot = self.availability.trailing_zeros() as usize; + if slot >= HEAP_SIZE_FIXED { + log::error!("Failed to allocate a handle form a fixed size heap"); + return Err(crate::DeviceError::OutOfMemory); + } + // Set the slot as occupied. + self.availability ^= 1 << slot; + + Ok(Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE { + ptr: self.start.ptr + self.handle_size * slot, + }) + } + + fn free_handle(&mut self, handle: Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE) { + let slot = (handle.ptr - self.start.ptr) / self.handle_size; + assert!(slot < HEAP_SIZE_FIXED); + assert_eq!(self.availability & (1 << slot), 0); + self.availability ^= 1 << slot; + } + + fn is_full(&self) -> bool { + self.availability == 0 + } +} + +#[derive(Clone, Copy)] +pub(super) struct Handle { + pub raw: Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE, + heap_index: usize, +} + +impl fmt::Debug for Handle { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.debug_struct("Handle") + .field("ptr", &self.raw.ptr) + .field("heap_index", &self.heap_index) + .finish() + } +} + +pub(super) struct CpuPool { + device: Direct3D12::ID3D12Device, + ty: Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE, + heaps: Vec, + available_heap_indices: BitSet, +} + +impl CpuPool { + pub(super) fn new( + device: Direct3D12::ID3D12Device, + ty: Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE, + ) -> Self { + Self { + device, + ty, + heaps: Vec::new(), + available_heap_indices: BitSet::new(), + } + } + + pub(super) fn alloc_handle(&mut self) -> Result { + let heap_index = self + .available_heap_indices + .iter() + .next() + .unwrap_or(self.heaps.len()); + + // Allocate a new heap + if heap_index == self.heaps.len() { + self.heaps.push(FixedSizeHeap::new(&self.device, self.ty)?); + self.available_heap_indices.insert(heap_index); + } + + let heap = &mut self.heaps[heap_index]; + let handle = Handle { + raw: heap.alloc_handle()?, + heap_index, + }; + if heap.is_full() { + self.available_heap_indices.remove(heap_index); + } + + Ok(handle) + } + + pub(super) fn free_handle(&mut self, handle: Handle) { + self.heaps[handle.heap_index].free_handle(handle.raw); + self.available_heap_indices.insert(handle.heap_index); + } +} + +pub(super) struct CpuHeapInner { + pub _raw: Direct3D12::ID3D12DescriptorHeap, + pub stage: Vec, +} + +pub(super) struct CpuHeap { + pub inner: Mutex, + start: Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE, + handle_size: u32, + total: u32, +} + +unsafe impl Send for CpuHeap {} +unsafe impl Sync for CpuHeap {} + +impl CpuHeap { + pub(super) fn new( + device: &Direct3D12::ID3D12Device, + ty: Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE, + total: u32, + ) -> Result { + let handle_size = unsafe { device.GetDescriptorHandleIncrementSize(ty) }; + let desc = Direct3D12::D3D12_DESCRIPTOR_HEAP_DESC { + Type: ty, + NumDescriptors: total, + Flags: Direct3D12::D3D12_DESCRIPTOR_HEAP_FLAG_NONE, + NodeMask: 0, + }; + let raw = unsafe { device.CreateDescriptorHeap::(&desc) } + .into_device_result("CPU descriptor heap creation")?; + + let start = unsafe { raw.GetCPUDescriptorHandleForHeapStart() }; + + Ok(Self { + inner: Mutex::new(CpuHeapInner { + _raw: raw, + stage: Vec::new(), + }), + start, + handle_size, + total, + }) + } + + pub(super) fn at(&self, index: u32) -> Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE { + debug_assert!( + index < self.total, + "Index ({index}) out of bounds {total}", + total = self.total + ); + Direct3D12::D3D12_CPU_DESCRIPTOR_HANDLE { + ptr: self.start.ptr + (self.handle_size * index) as usize, + } + } +} + +impl fmt::Debug for CpuHeap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CpuHeap") + .field("start", &self.start.ptr) + .field("handle_size", &self.handle_size) + .field("total", &self.total) + .finish() + } +} + +pub(super) unsafe fn upload( + device: &Direct3D12::ID3D12Device, + src: &CpuHeapInner, + dst: &GeneralHeap, + dummy_copy_counts: &[u32], +) -> Result { + let count = src.stage.len() as u32; + let index = dst.allocate_slice(count as u64)?; + unsafe { + device.CopyDescriptors( + 1, + &dst.cpu_descriptor_at(index), + Some(&count), + count, + src.stage.as_ptr(), + Some(dummy_copy_counts.as_ptr()), + dst.ty, + ) + }; + Ok(dst.at(index, count as u64)) +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/device.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/device.rs new file mode 100644 index 00000000..5571d014 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/device.rs @@ -0,0 +1,2636 @@ +use alloc::borrow::ToOwned; +use alloc::{ + borrow::Cow, + string::{String, ToString as _}, + sync::Arc, + vec::Vec, +}; +use arrayvec::ArrayVec; +use core::{ffi, num::NonZeroU32, ptr, time::Duration}; +use std::time::Instant; + +use bytemuck::TransparentWrapper; +use parking_lot::Mutex; +use windows::{ + core::Interface as _, + Win32::{ + Foundation, + Graphics::{Direct3D12, Dxgi}, + System::Threading, + }, +}; + +use super::{conv, descriptor, D3D12Lib}; +use crate::{ + auxil::{ + self, + dxgi::{name::ObjectExt as _, result::HResult as _}, + }, + dx12::{ + borrow_optional_interface_temporarily, pipeline_desc::RenderPipelineStateStreamDesc, + shader_compilation, suballocation, DCompLib, DynamicStorageBufferOffsets, Event, + ShaderCacheKey, ShaderCacheValue, + }, + AccelerationStructureEntries, TlasInstance, +}; + +// this has to match Naga's HLSL backend, and also needs to be null-terminated +const NAGA_LOCATION_SEMANTIC: &[u8] = c"LOC".to_bytes(); + +impl super::Device { + #[allow(clippy::too_many_arguments)] + pub(super) fn new( + adapter: auxil::dxgi::factory::DxgiAdapter, + raw: Direct3D12::ID3D12Device, + present_queue: Direct3D12::ID3D12CommandQueue, + features: wgt::Features, + limits: &wgt::Limits, + memory_hints: &wgt::MemoryHints, + private_caps: super::PrivateCapabilities, + library: &Arc, + dcomp_lib: &Arc, + memory_budget_thresholds: wgt::MemoryBudgetThresholds, + compiler_container: Arc, + backend_options: wgt::Dx12BackendOptions, + ) -> Result { + if private_caps + .instance_flags + .contains(wgt::InstanceFlags::VALIDATION) + { + auxil::dxgi::exception::register_exception_handler(); + } + + let mem_allocator = + suballocation::Allocator::new(&raw, memory_hints, memory_budget_thresholds)?; + + let idle_fence: Direct3D12::ID3D12Fence = unsafe { + profiling::scope!("ID3D12Device::CreateFence"); + raw.CreateFence(0, Direct3D12::D3D12_FENCE_FLAG_NONE) + } + .into_device_result("Idle fence creation")?; + + let raw_desc = Direct3D12::D3D12_RESOURCE_DESC { + Dimension: Direct3D12::D3D12_RESOURCE_DIMENSION_BUFFER, + Alignment: 0, + Width: super::ZERO_BUFFER_SIZE, + Height: 1, + DepthOrArraySize: 1, + MipLevels: 1, + Format: Dxgi::Common::DXGI_FORMAT_UNKNOWN, + SampleDesc: Dxgi::Common::DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Layout: Direct3D12::D3D12_TEXTURE_LAYOUT_ROW_MAJOR, + Flags: Direct3D12::D3D12_RESOURCE_FLAG_NONE, + }; + + let heap_properties = Direct3D12::D3D12_HEAP_PROPERTIES { + Type: Direct3D12::D3D12_HEAP_TYPE_CUSTOM, + CPUPageProperty: Direct3D12::D3D12_CPU_PAGE_PROPERTY_NOT_AVAILABLE, + MemoryPoolPreference: match private_caps.memory_architecture { + super::MemoryArchitecture::Unified { .. } => Direct3D12::D3D12_MEMORY_POOL_L0, + super::MemoryArchitecture::NonUnified => Direct3D12::D3D12_MEMORY_POOL_L1, + }, + CreationNodeMask: 0, + VisibleNodeMask: 0, + }; + + profiling::scope!("Zero Buffer Allocation"); + let mut zero_buffer = None::; + unsafe { + raw.CreateCommittedResource( + &heap_properties, + Direct3D12::D3D12_HEAP_FLAG_NONE, + &raw_desc, + Direct3D12::D3D12_RESOURCE_STATE_COMMON, + None, + &mut zero_buffer, + ) + } + .into_device_result("Zero buffer creation")?; + + let zero_buffer = zero_buffer.ok_or(crate::DeviceError::Unexpected)?; + + // Note: without `D3D12_HEAP_FLAG_CREATE_NOT_ZEROED` + // this resource is zeroed by default. + + // maximum number of CBV/SRV/UAV descriptors in heap for Tier 1 + let capacity_views = limits.max_non_sampler_bindings as u64; + + let draw_mesh = if features + .features_wgpu + .contains(wgt::FeaturesWGPU::EXPERIMENTAL_MESH_SHADER) + { + Some(Self::create_command_signature( + &raw, + None, + size_of::(), + &[Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC { + Type: Direct3D12::D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH_MESH, + ..Default::default() + }], + 0, + )?) + } else { + None + }; + + let shared = super::DeviceShared { + adapter, + zero_buffer, + cmd_signatures: super::CommandSignatures { + draw: Self::create_command_signature( + &raw, + None, + size_of::(), + &[Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC { + Type: Direct3D12::D3D12_INDIRECT_ARGUMENT_TYPE_DRAW, + ..Default::default() + }], + 0, + )?, + draw_indexed: Self::create_command_signature( + &raw, + None, + size_of::(), + &[Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC { + Type: Direct3D12::D3D12_INDIRECT_ARGUMENT_TYPE_DRAW_INDEXED, + ..Default::default() + }], + 0, + )?, + draw_mesh, + dispatch: Self::create_command_signature( + &raw, + None, + size_of::(), + &[Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC { + Type: Direct3D12::D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH, + ..Default::default() + }], + 0, + )?, + }, + heap_views: descriptor::GeneralHeap::new( + &raw, + Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV, + capacity_views, + )?, + sampler_heap: super::sampler::SamplerHeap::new(&raw, &private_caps)?, + private_caps, + }; + + let mut rtv_pool = + descriptor::CpuPool::new(raw.clone(), Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE_RTV); + let null_rtv_handle = rtv_pool.alloc_handle()?; + // A null pResource is used to initialize a null descriptor, + // which guarantees D3D11-like null binding behavior (reading 0s, writes are discarded) + unsafe { + raw.CreateRenderTargetView( + None, + Some(&Direct3D12::D3D12_RENDER_TARGET_VIEW_DESC { + Format: Dxgi::Common::DXGI_FORMAT_R8G8B8A8_UNORM, + ViewDimension: Direct3D12::D3D12_RTV_DIMENSION_TEXTURE2D, + Anonymous: Direct3D12::D3D12_RENDER_TARGET_VIEW_DESC_0 { + Texture2D: Direct3D12::D3D12_TEX2D_RTV { + MipSlice: 0, + PlaneSlice: 0, + }, + }, + }), + null_rtv_handle.raw, + ) + }; + + Ok(super::Device { + raw: raw.clone(), + present_queue, + idler: super::Idler { fence: idle_fence }, + features, + shared: Arc::new(shared), + rtv_pool: Arc::new(Mutex::new(rtv_pool)), + dsv_pool: Mutex::new(descriptor::CpuPool::new( + raw.clone(), + Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE_DSV, + )), + srv_uav_pool: Mutex::new(descriptor::CpuPool::new( + raw.clone(), + Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV, + )), + options: backend_options, + library: Arc::clone(library), + dcomp_lib: Arc::clone(dcomp_lib), + #[cfg(feature = "renderdoc")] + render_doc: Default::default(), + null_rtv_handle, + mem_allocator, + compiler_container, + shader_cache: Default::default(), + counters: Default::default(), + }) + } + + fn create_command_signature( + raw: &Direct3D12::ID3D12Device, + root_signature: Option<&Direct3D12::ID3D12RootSignature>, + byte_stride: usize, + arguments: &[Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC], + node_mask: u32, + ) -> Result { + let mut signature = None; + unsafe { + raw.CreateCommandSignature( + &Direct3D12::D3D12_COMMAND_SIGNATURE_DESC { + ByteStride: byte_stride as u32, + NumArgumentDescs: arguments.len() as u32, + pArgumentDescs: arguments.as_ptr(), + NodeMask: node_mask, + }, + root_signature, + &mut signature, + ) + } + .into_device_result("Command signature creation")?; + signature.ok_or(crate::DeviceError::Unexpected) + } + + // Blocks until the dedicated present queue is finished with all of its work. + // + // Once this method completes, the surface is able to be resized or deleted. + pub(super) unsafe fn wait_for_present_queue_idle(&self) -> Result<(), crate::DeviceError> { + let cur_value = unsafe { self.idler.fence.GetCompletedValue() }; + if cur_value == !0 { + return Err(crate::DeviceError::Lost); + } + + let event = Event::create(false, false)?; + + let value = cur_value + 1; + unsafe { self.present_queue.Signal(&self.idler.fence, value) } + .into_device_result("Signal")?; + let hr = unsafe { self.idler.fence.SetEventOnCompletion(value, event.0) }; + hr.into_device_result("Set event")?; + unsafe { Threading::WaitForSingleObject(event.0, Threading::INFINITE) }; + Ok(()) + } + + /// When generating the vertex shader, the fragment stage must be passed if it exists! + /// Otherwise, the generated HLSL may be incorrect since the fragment shader inputs are + /// allowed to be a subset of the vertex outputs. + fn load_shader( + &self, + stage: &crate::ProgrammableStage, + layout: &super::PipelineLayout, + naga_stage: naga::ShaderStage, + fragment_stage: Option<&crate::ProgrammableStage>, + ) -> Result { + let stage_bit = auxil::map_naga_stage(naga_stage); + + let needs_temp_options = stage.zero_initialize_workgroup_memory + != layout.naga_options.zero_initialize_workgroup_memory + || stage.module.runtime_checks.bounds_checks != layout.naga_options.restrict_indexing + || stage.module.runtime_checks.force_loop_bounding + != layout.naga_options.force_loop_bounding + || stage + .module + .runtime_checks + .ray_query_initialization_tracking + != layout.naga_options.ray_query_initialization_tracking; + let mut temp_options; + let naga_options = if needs_temp_options { + temp_options = layout.naga_options.clone(); + temp_options.zero_initialize_workgroup_memory = stage.zero_initialize_workgroup_memory; + temp_options.restrict_indexing = stage.module.runtime_checks.bounds_checks; + temp_options.force_loop_bounding = stage.module.runtime_checks.force_loop_bounding; + temp_options.ray_query_initialization_tracking = stage + .module + .runtime_checks + .ray_query_initialization_tracking; + &temp_options + } else { + &layout.naga_options + }; + + let key = match &stage.module.source { + super::ShaderModuleSource::Naga(naga_shader) => { + use naga::back::hlsl; + + let frag_ep = match fragment_stage { + Some(crate::ProgrammableStage { + module: + super::ShaderModule { + source: super::ShaderModuleSource::Naga(naga_shader), + .. + }, + entry_point, + .. + }) => Some( + hlsl::FragmentEntryPoint::new(&naga_shader.module, entry_point).ok_or( + crate::PipelineError::EntryPoint(naga::ShaderStage::Fragment), + ), + ), + _ => None, + } + .transpose()?; + let (module, info) = naga::back::pipeline_constants::process_overrides( + &naga_shader.module, + &naga_shader.info, + Some((naga_stage, stage.entry_point)), + stage.constants, + ) + .map_err(|e| { + crate::PipelineError::PipelineConstants(stage_bit, format!("HLSL: {e:?}")) + })?; + + let pipeline_options = hlsl::PipelineOptions { + entry_point: Some((naga_stage, stage.entry_point.to_string())), + }; + + //TODO: reuse the writer + let (source, entry_point) = { + let mut source = String::new(); + let mut writer = + hlsl::Writer::new(&mut source, naga_options, &pipeline_options); + + profiling::scope!("naga::back::hlsl::write"); + let mut reflection_info = writer + .write(&module, &info, frag_ep.as_ref()) + .map_err(|e| { + crate::PipelineError::Linkage(stage_bit, format!("HLSL: {e:?}")) + })?; + + assert_eq!(reflection_info.entry_point_names.len(), 1); + + let entry_point = reflection_info + .entry_point_names + .pop() + .unwrap() + .map_err(|e| crate::PipelineError::Linkage(stage_bit, format!("{e}")))?; + + (source, entry_point) + }; + log::debug!( + "Naga generated shader for {entry_point:?} at {naga_stage:?}:\n{source}" + ); + + ShaderCacheKey { + source, + entry_point, + stage: naga_stage, + shader_model: naga_options.shader_model, + } + } + super::ShaderModuleSource::HlslPassthrough(passthrough) => ShaderCacheKey { + source: passthrough.shader.clone(), + entry_point: stage.entry_point.to_string(), + stage: naga_stage, + shader_model: naga_options.shader_model, + }, + super::ShaderModuleSource::DxilPassthrough(passthrough) => { + return Ok(super::CompiledShader::Precompiled( + passthrough.shader.clone(), + )) + } + }; + + { + let mut shader_cache = self.shader_cache.lock(); + let nr_of_shaders_compiled = shader_cache.nr_of_shaders_compiled; + if let Some(value) = shader_cache.entries.get_mut(&key) { + value.last_used = nr_of_shaders_compiled; + return Ok(value.shader.clone()); + } + } + + let source_name = stage.module.raw_name.as_deref(); + + let full_stage = format!("{}_{}", naga_stage.to_hlsl_str(), key.shader_model.to_str()); + + let compiled_shader = self.compiler_container.compile( + self, + &key.source, + source_name, + &key.entry_point, + stage_bit, + &full_stage, + )?; + + { + let mut shader_cache = self.shader_cache.lock(); + shader_cache.nr_of_shaders_compiled += 1; + let nr_of_shaders_compiled = shader_cache.nr_of_shaders_compiled; + let value = ShaderCacheValue { + last_used: nr_of_shaders_compiled, + shader: compiled_shader.clone(), + }; + shader_cache.entries.insert(key, value); + + // Retain all entries that have been used since we compiled the last 100 shaders. + if shader_cache.entries.len() > 200 { + shader_cache + .entries + .retain(|_, v| v.last_used >= nr_of_shaders_compiled - 100); + } + } + + Ok(compiled_shader) + } + + pub fn raw_device(&self) -> &Direct3D12::ID3D12Device { + &self.raw + } + + pub fn raw_queue(&self) -> &Direct3D12::ID3D12CommandQueue { + &self.present_queue + } + + pub unsafe fn texture_from_raw( + resource: Direct3D12::ID3D12Resource, + format: wgt::TextureFormat, + dimension: wgt::TextureDimension, + size: wgt::Extent3d, + mip_level_count: u32, + sample_count: u32, + ) -> super::Texture { + super::Texture { + resource, + format, + dimension, + size, + mip_level_count, + sample_count, + allocation: suballocation::Allocation::none( + suballocation::AllocationType::Texture, + format.theoretical_memory_footprint(size), + ), + } + } + + pub unsafe fn buffer_from_raw( + resource: Direct3D12::ID3D12Resource, + size: wgt::BufferAddress, + ) -> super::Buffer { + super::Buffer { + resource, + size, + allocation: suballocation::Allocation::none( + suballocation::AllocationType::Buffer, + size, + ), + } + } +} + +impl crate::Device for super::Device { + type A = super::Api; + + unsafe fn create_buffer( + &self, + desc: &crate::BufferDescriptor, + ) -> Result { + let mut desc = desc.clone(); + + if desc.usage.contains(wgt::BufferUses::UNIFORM) { + desc.size = desc + .size + .next_multiple_of(Direct3D12::D3D12_CONSTANT_BUFFER_DATA_PLACEMENT_ALIGNMENT.into()) + } + + let (resource, allocation) = + suballocation::DeviceAllocationContext::from(self).create_buffer(&desc)?; + + self.counters.buffers.add(1); + + Ok(super::Buffer { + resource, + size: desc.size, + allocation, + }) + } + + unsafe fn destroy_buffer(&self, buffer: super::Buffer) { + suballocation::DeviceAllocationContext::from(self) + .free_resource(buffer.resource, buffer.allocation); + + self.counters.buffers.sub(1); + } + + unsafe fn add_raw_buffer(&self, _buffer: &super::Buffer) { + self.counters.buffers.add(1); + } + + unsafe fn map_buffer( + &self, + buffer: &super::Buffer, + range: crate::MemoryRange, + ) -> Result { + let mut ptr = ptr::null_mut(); + // TODO: 0 for subresource should be fine here until map and unmap buffer is subresource aware? + unsafe { buffer.resource.Map(0, None, Some(&mut ptr)) }.into_device_result("Map buffer")?; + + Ok(crate::BufferMapping { + ptr: ptr::NonNull::new(unsafe { ptr.offset(range.start as isize).cast::() }) + .unwrap(), + //TODO: double-check this. Documentation is a bit misleading - + // it implies that Map/Unmap is needed to invalidate/flush memory. + is_coherent: true, + }) + } + + unsafe fn unmap_buffer(&self, buffer: &super::Buffer) { + unsafe { buffer.resource.Unmap(0, None) }; + } + + unsafe fn flush_mapped_ranges(&self, _buffer: &super::Buffer, _ranges: I) {} + unsafe fn invalidate_mapped_ranges(&self, _buffer: &super::Buffer, _ranges: I) {} + + unsafe fn create_texture( + &self, + desc: &crate::TextureDescriptor, + ) -> Result { + let raw_desc = Direct3D12::D3D12_RESOURCE_DESC { + Dimension: conv::map_texture_dimension(desc.dimension), + Alignment: 0, + Width: desc.size.width as u64, + Height: desc.size.height, + DepthOrArraySize: desc.size.depth_or_array_layers as u16, + MipLevels: desc.mip_level_count as u16, + Format: auxil::dxgi::conv::map_texture_format_for_resource( + desc.format, + desc.usage, + !desc.view_formats.is_empty(), + self.shared + .private_caps + .casting_fully_typed_format_supported, + ), + SampleDesc: Dxgi::Common::DXGI_SAMPLE_DESC { + Count: desc.sample_count, + Quality: 0, + }, + Layout: Direct3D12::D3D12_TEXTURE_LAYOUT_UNKNOWN, + Flags: conv::map_texture_usage_to_resource_flags(desc.usage), + }; + + let (resource, allocation) = + suballocation::DeviceAllocationContext::from(self).create_texture(desc, raw_desc)?; + + self.counters.textures.add(1); + + Ok(super::Texture { + resource, + format: desc.format, + dimension: desc.dimension, + size: desc.size, + mip_level_count: desc.mip_level_count, + sample_count: desc.sample_count, + allocation, + }) + } + + unsafe fn destroy_texture(&self, texture: super::Texture) { + suballocation::DeviceAllocationContext::from(self) + .free_resource(texture.resource, texture.allocation); + + self.counters.textures.sub(1); + } + + unsafe fn add_raw_texture(&self, _texture: &super::Texture) { + self.counters.textures.add(1); + } + + unsafe fn create_texture_view( + &self, + texture: &super::Texture, + desc: &crate::TextureViewDescriptor, + ) -> Result { + let view_desc = desc.to_internal(texture); + + self.counters.texture_views.add(1); + + Ok(super::TextureView { + raw_format: view_desc.rtv_dsv_format, + aspects: view_desc.aspects, + dimension: desc.dimension, + texture: texture.resource.clone(), + subresource_index: texture.calc_subresource( + desc.range.base_mip_level, + desc.range.base_array_layer, + 0, + ), + mip_slice: desc.range.base_mip_level, + handle_srv: if desc.usage.intersects(wgt::TextureUses::RESOURCE) { + match unsafe { view_desc.to_srv() } { + Some(raw_desc) => { + let handle = self.srv_uav_pool.lock().alloc_handle()?; + unsafe { + self.raw.CreateShaderResourceView( + &texture.resource, + Some(&raw_desc), + handle.raw, + ) + }; + Some(handle) + } + None => None, + } + } else { + None + }, + handle_uav: if desc.usage.intersects( + wgt::TextureUses::STORAGE_READ_ONLY + | wgt::TextureUses::STORAGE_WRITE_ONLY + | wgt::TextureUses::STORAGE_READ_WRITE, + ) { + match unsafe { view_desc.to_uav() } { + Some(raw_desc) => { + let handle = self.srv_uav_pool.lock().alloc_handle()?; + unsafe { + self.raw.CreateUnorderedAccessView( + &texture.resource, + None, + Some(&raw_desc), + handle.raw, + ); + } + Some(handle) + } + None => None, + } + } else { + None + }, + handle_rtv: if desc.usage.intersects(wgt::TextureUses::COLOR_TARGET) + && desc.dimension != wgt::TextureViewDimension::D3 + // 3D RTVs must be created in the render pass + { + let raw_desc = unsafe { view_desc.to_rtv() }; + let handle = self.rtv_pool.lock().alloc_handle()?; + unsafe { + self.raw + .CreateRenderTargetView(&texture.resource, Some(&raw_desc), handle.raw) + }; + Some(handle) + } else { + None + }, + handle_dsv_ro: if desc.usage.intersects(wgt::TextureUses::DEPTH_STENCIL_READ) { + let raw_desc = unsafe { view_desc.to_dsv(true) }; + let handle = self.dsv_pool.lock().alloc_handle()?; + unsafe { + self.raw + .CreateDepthStencilView(&texture.resource, Some(&raw_desc), handle.raw) + }; + Some(handle) + } else { + None + }, + handle_dsv_rw: if desc.usage.intersects(wgt::TextureUses::DEPTH_STENCIL_WRITE) { + let raw_desc = unsafe { view_desc.to_dsv(false) }; + let handle = self.dsv_pool.lock().alloc_handle()?; + unsafe { + self.raw + .CreateDepthStencilView(&texture.resource, Some(&raw_desc), handle.raw) + }; + Some(handle) + } else { + None + }, + }) + } + + unsafe fn destroy_texture_view(&self, view: super::TextureView) { + if view.handle_srv.is_some() || view.handle_uav.is_some() { + let mut pool = self.srv_uav_pool.lock(); + if let Some(handle) = view.handle_srv { + pool.free_handle(handle); + } + if let Some(handle) = view.handle_uav { + pool.free_handle(handle); + } + } + if let Some(handle) = view.handle_rtv { + self.rtv_pool.lock().free_handle(handle); + } + if view.handle_dsv_ro.is_some() || view.handle_dsv_rw.is_some() { + let mut pool = self.dsv_pool.lock(); + if let Some(handle) = view.handle_dsv_ro { + pool.free_handle(handle); + } + if let Some(handle) = view.handle_dsv_rw { + pool.free_handle(handle); + } + } + + self.counters.texture_views.sub(1); + } + + unsafe fn create_sampler( + &self, + desc: &crate::SamplerDescriptor, + ) -> Result { + let reduction = match desc.compare { + Some(_) => Direct3D12::D3D12_FILTER_REDUCTION_TYPE_COMPARISON, + None => Direct3D12::D3D12_FILTER_REDUCTION_TYPE_STANDARD, + }; + let mut filter = Direct3D12::D3D12_FILTER( + (conv::map_filter_mode(desc.min_filter).0 << Direct3D12::D3D12_MIN_FILTER_SHIFT) + | (conv::map_filter_mode(desc.mag_filter).0 << Direct3D12::D3D12_MAG_FILTER_SHIFT) + | (conv::map_mipmap_filter_mode(desc.mipmap_filter).0 + << Direct3D12::D3D12_MIP_FILTER_SHIFT) + | (reduction.0 << Direct3D12::D3D12_FILTER_REDUCTION_TYPE_SHIFT), + ); + + if desc.anisotropy_clamp != 1 { + filter.0 |= Direct3D12::D3D12_FILTER_ANISOTROPIC.0; + }; + + let border_color = conv::map_border_color(desc.border_color); + + let raw_desc = Direct3D12::D3D12_SAMPLER_DESC { + Filter: filter, + AddressU: conv::map_address_mode(desc.address_modes[0]), + AddressV: conv::map_address_mode(desc.address_modes[1]), + AddressW: conv::map_address_mode(desc.address_modes[2]), + MipLODBias: 0f32, + MaxAnisotropy: desc.anisotropy_clamp as u32, + + ComparisonFunc: conv::map_comparison(desc.compare.unwrap_or_default()), + BorderColor: border_color, + MinLOD: desc.lod_clamp.start, + MaxLOD: desc.lod_clamp.end, + }; + + let index = self + .shared + .sampler_heap + .create_sampler(&self.raw, raw_desc)?; + + self.counters.samplers.add(1); + + Ok(super::Sampler { + index, + desc: raw_desc, + }) + } + + unsafe fn destroy_sampler(&self, sampler: super::Sampler) { + self.shared + .sampler_heap + .destroy_sampler(sampler.desc, sampler.index); + self.counters.samplers.sub(1); + } + + unsafe fn create_command_encoder( + &self, + desc: &crate::CommandEncoderDescriptor, + ) -> Result { + let allocator: Direct3D12::ID3D12CommandAllocator = unsafe { + self.raw + .CreateCommandAllocator(Direct3D12::D3D12_COMMAND_LIST_TYPE_DIRECT) + } + .into_device_result("Command allocator creation")?; + + if let Some(label) = desc.label { + allocator.set_name(label)?; + } + + self.counters.command_encoders.add(1); + + Ok(super::CommandEncoder { + allocator, + device: self.raw.clone(), + shared: Arc::clone(&self.shared), + mem_allocator: self.mem_allocator.clone(), + rtv_pool: Arc::clone(&self.rtv_pool), + temp_rtv_handles: Vec::new(), + intermediate_copy_bufs: Vec::new(), + null_rtv_handle: self.null_rtv_handle, + list: None, + free_lists: Vec::new(), + pass: super::PassState::new(), + temp: super::Temp::default(), + end_of_pass_timer_query: None, + counters: Arc::clone(&self.counters), + }) + } + + unsafe fn create_bind_group_layout( + &self, + desc: &crate::BindGroupLayoutDescriptor, + ) -> Result { + let mut num_views = 0; + let mut has_sampler_in_group = false; + for entry in desc.entries.iter() { + let count = entry.count.map_or(1, NonZeroU32::get); + match entry.ty { + wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Uniform, + has_dynamic_offset: true, + .. + } => {} + wgt::BindingType::Buffer { .. } + | wgt::BindingType::Texture { .. } + | wgt::BindingType::StorageTexture { .. } + | wgt::BindingType::AccelerationStructure { .. } => num_views += count, + wgt::BindingType::Sampler { .. } => has_sampler_in_group = true, + // Three texture planes and one params buffer + wgt::BindingType::ExternalTexture => num_views += 4 * count, + } + } + + if has_sampler_in_group { + num_views += 1; + } + + self.counters.bind_group_layouts.add(1); + + Ok(super::BindGroupLayout { + entries: desc.entries.to_vec(), + cpu_heap_views: if num_views != 0 { + let heap = descriptor::CpuHeap::new( + &self.raw, + Direct3D12::D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV, + num_views, + )?; + Some(heap) + } else { + None + }, + copy_counts: vec![1; num_views as usize], + }) + } + + unsafe fn destroy_bind_group_layout(&self, _bg_layout: super::BindGroupLayout) { + self.counters.bind_group_layouts.sub(1); + } + + unsafe fn create_pipeline_layout( + &self, + desc: &crate::PipelineLayoutDescriptor, + ) -> Result { + use naga::back::hlsl; + // Pipeline layouts are implemented as RootSignature for D3D12. + // + // Immediates are implemented as root constants. + // + // Each bind group layout might use one SRV/CBV/UAV descriptor table. + // With resources in the bind group layout using: + // - 1 CBV per non-dynamic uniform buffer + // - 1 SRV per acceleration structure + // - 1 SRV for all samplers in a bind group + // - 1 SRV per texture + // - 1 SRV per read-only storage buffer + // - 1 UAV per storage texture + // - 1 UAV per read-write storage buffer + // - 3 SRVs & 1 CBV per external texture + // + // Each dynamic uniform buffer takes up a CBV root descriptor. + // This is easier than trying to patch up the offset on the shader side. + // + // Each dynamic storage buffer is an SRV or UAV in the descriptor table + // and its dynamic offsets are passed via root constants. + // + // All samplers go into a single sampler descriptor table. + // + // 3 additional root constants are used to populate built-in (shader) inputs. + // + // Root signature layout: + // Root Constants: Parameter=0, Space=0 + // ... + // (bind group [0]) - Space=0 + // View descriptor table, if any + // Sampler buffer descriptor table, if any + // Root descriptors (for dynamic offset buffers) + // (bind group [1]) - Space=0 + // ... + // (bind group [2]) - Space=0 + // Special constant buffer: Space=0 + // Sampler descriptor tables: Space=0 + // SamplerState Array: Space=0, Register=0-2047 + // SamplerComparisonState Array: Space=0, Register=2048-4095 + + //TODO: put lower bind group indices further down the root signature. See: + // https://microsoft.github.io/DirectX-Specs/d3d/ResourceBinding.html#binding-model + // Currently impossible because wgpu-core only re-binds the descriptor sets based + // on Vulkan-like layout compatibility rules. + + let mut binding_map = hlsl::BindingMap::default(); + let mut sampler_buffer_binding_map = hlsl::SamplerIndexBufferBindingMap::default(); + let mut external_texture_binding_map = hlsl::ExternalTextureBindingMap::default(); + let mut bind_cbv = hlsl::BindTarget::default(); + let mut bind_srv = hlsl::BindTarget::default(); + let mut bind_uav = hlsl::BindTarget::default(); + let mut parameters = Vec::new(); + let mut immediates_target = None; + let mut root_constant_info = None; + + if desc.immediate_size != 0 { + let parameter_index = parameters.len(); + let size = desc.immediate_size / 4; + parameters.push(Direct3D12::D3D12_ROOT_PARAMETER { + ParameterType: Direct3D12::D3D12_ROOT_PARAMETER_TYPE_32BIT_CONSTANTS, + Anonymous: Direct3D12::D3D12_ROOT_PARAMETER_0 { + Constants: Direct3D12::D3D12_ROOT_CONSTANTS { + ShaderRegister: bind_cbv.register, + RegisterSpace: bind_cbv.space as u32, + Num32BitValues: size, + }, + }, + ShaderVisibility: Direct3D12::D3D12_SHADER_VISIBILITY_ALL, + }); + let binding = bind_cbv; + bind_cbv.register += 1; + root_constant_info = Some(super::RootConstantInfo { + root_index: parameter_index as u32, + range: 0..size, + }); + immediates_target = Some(binding); + + bind_cbv.space += 1; + } + + let mut dynamic_storage_buffer_offsets_targets = alloc::collections::BTreeMap::new(); + let mut total_dynamic_storage_buffers = 0; + + // Collect the whole number of bindings we will create upfront. + // It allows us to preallocate enough storage to avoid reallocation, + // which could cause invalid pointers. + let mut total_non_dynamic_entries = 0_usize; + let mut sampler_in_any_bind_group = false; + for bgl in desc.bind_group_layouts { + let Some(bgl) = bgl else { + continue; + }; + + let mut sampler_in_bind_group = false; + + for entry in &bgl.entries { + match entry.ty { + wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Uniform, + has_dynamic_offset: true, + .. + } => {} + wgt::BindingType::Sampler(_) => sampler_in_bind_group = true, + // Three texture planes and one params buffer + wgt::BindingType::ExternalTexture => total_non_dynamic_entries += 4, + _ => total_non_dynamic_entries += 1, + } + } + + if sampler_in_bind_group { + // One for the sampler buffer + total_non_dynamic_entries += 1; + sampler_in_any_bind_group = true; + } + } + + if sampler_in_any_bind_group { + // Two for the sampler arrays themselves + total_non_dynamic_entries += 2; + } + + let mut ranges = Vec::with_capacity(total_non_dynamic_entries); + + let mut bind_group_infos = [const { None }; crate::MAX_BIND_GROUPS]; + for (index, bgl) in desc.bind_group_layouts.iter().enumerate() { + let Some(bgl) = bgl else { + continue; + }; + + let mut info = super::BindGroupInfo { + tables: super::TableTypes::empty(), + base_root_index: parameters.len() as u32, + dynamic_storage_buffer_offsets: None, + }; + + let mut visibility_view_static = wgt::ShaderStages::empty(); + let mut visibility_view_dynamic_uniform = wgt::ShaderStages::empty(); + let mut visibility_view_dynamic_storage = wgt::ShaderStages::empty(); + for entry in bgl.entries.iter() { + match entry.ty { + wgt::BindingType::Sampler { .. } => { + visibility_view_static |= wgt::ShaderStages::all() + } + wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Uniform, + has_dynamic_offset: true, + .. + } => visibility_view_dynamic_uniform |= entry.visibility, + wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Storage { .. }, + has_dynamic_offset: true, + .. + } => visibility_view_dynamic_storage |= entry.visibility, + _ => visibility_view_static |= entry.visibility, + } + } + + let mut dynamic_storage_buffers = 0; + + // SRV/CBV/UAV descriptor tables + let range_base = ranges.len(); + for entry in bgl.entries.iter() { + let count = entry.count.map_or(1, NonZeroU32::get); + if let wgt::BindingType::ExternalTexture = entry.ty { + // External textures need 3 SRVs (a texture for each plane) + // and 1 CBV for the parameters buffer. + let bind_target = hlsl::ExternalTextureBindTarget { + planes: core::array::from_fn(|_| hlsl::BindTarget { + register: { + let register = bind_srv.register; + bind_srv.register += count; + register + }, + ..bind_srv + }), + params: hlsl::BindTarget { + register: { + let register = bind_cbv.register; + bind_cbv.register += count; + register + }, + ..bind_cbv + }, + }; + external_texture_binding_map.insert( + naga::ResourceBinding { + group: index as u32, + binding: entry.binding, + }, + bind_target, + ); + for bt in bind_target.planes { + ranges.push(Direct3D12::D3D12_DESCRIPTOR_RANGE { + RangeType: Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_SRV, + NumDescriptors: count, + BaseShaderRegister: bt.register, + RegisterSpace: bt.space as u32, + OffsetInDescriptorsFromTableStart: + Direct3D12::D3D12_DESCRIPTOR_RANGE_OFFSET_APPEND, + }); + } + ranges.push(Direct3D12::D3D12_DESCRIPTOR_RANGE { + RangeType: Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_CBV, + NumDescriptors: count, + BaseShaderRegister: bind_target.params.register, + RegisterSpace: bind_target.params.space as u32, + OffsetInDescriptorsFromTableStart: + Direct3D12::D3D12_DESCRIPTOR_RANGE_OFFSET_APPEND, + }); + } else { + let (range_ty, has_dynamic_offset) = match entry.ty { + wgt::BindingType::Buffer { + ty, + has_dynamic_offset: true, + .. + } => match ty { + wgt::BufferBindingType::Uniform => continue, + wgt::BufferBindingType::Storage { .. } => { + (conv::map_binding_type(&entry.ty), true) + } + }, + ref other => (conv::map_binding_type(other), false), + }; + let bt = match range_ty { + Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_CBV => &mut bind_cbv, + Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_SRV => &mut bind_srv, + Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_UAV => &mut bind_uav, + Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_SAMPLER => continue, + _ => todo!(), + }; + + let binding_array_size = entry.count.map(NonZeroU32::get); + + let dynamic_storage_buffer_offsets_index = if has_dynamic_offset { + debug_assert!( + binding_array_size.is_none(), + "binding arrays and dynamic buffers are mutually exclusive" + ); + let ret = Some(dynamic_storage_buffers); + dynamic_storage_buffers += 1; + ret + } else { + None + }; + + binding_map.insert( + naga::ResourceBinding { + group: index as u32, + binding: entry.binding, + }, + hlsl::BindTarget { + binding_array_size, + dynamic_storage_buffer_offsets_index, + ..*bt + }, + ); + ranges.push(Direct3D12::D3D12_DESCRIPTOR_RANGE { + RangeType: range_ty, + NumDescriptors: count, + BaseShaderRegister: bt.register, + RegisterSpace: bt.space as u32, + OffsetInDescriptorsFromTableStart: + Direct3D12::D3D12_DESCRIPTOR_RANGE_OFFSET_APPEND, + }); + bt.register += count; + } + } + + let mut sampler_index_within_bind_group = 0; + for entry in bgl.entries.iter() { + if let wgt::BindingType::Sampler(_) = entry.ty { + binding_map.insert( + naga::ResourceBinding { + group: index as u32, + binding: entry.binding, + }, + hlsl::BindTarget { + // Naga does not use the space field for samplers + space: 255, + register: sampler_index_within_bind_group, + binding_array_size: None, + dynamic_storage_buffer_offsets_index: None, + restrict_indexing: false, + }, + ); + sampler_index_within_bind_group += 1; + } + } + + if sampler_index_within_bind_group != 0 { + sampler_buffer_binding_map.insert( + hlsl::SamplerIndexBufferKey { + group: index as u32, + }, + bind_srv, + ); + ranges.push(Direct3D12::D3D12_DESCRIPTOR_RANGE { + RangeType: Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_SRV, + NumDescriptors: 1, + BaseShaderRegister: bind_srv.register, + RegisterSpace: bind_srv.space as u32, + OffsetInDescriptorsFromTableStart: + Direct3D12::D3D12_DESCRIPTOR_RANGE_OFFSET_APPEND, + }); + bind_srv.register += 1; + } + + if ranges.len() > range_base { + let range = &ranges[range_base..]; + parameters.push(Direct3D12::D3D12_ROOT_PARAMETER { + ParameterType: Direct3D12::D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE, + Anonymous: Direct3D12::D3D12_ROOT_PARAMETER_0 { + DescriptorTable: Direct3D12::D3D12_ROOT_DESCRIPTOR_TABLE { + NumDescriptorRanges: range.len() as u32, + pDescriptorRanges: range.as_ptr(), + }, + }, + ShaderVisibility: conv::map_visibility(visibility_view_static), + }); + info.tables |= super::TableTypes::SRV_CBV_UAV; + } + + // Root descriptors for dynamic uniform buffers + let dynamic_buffers_visibility = conv::map_visibility(visibility_view_dynamic_uniform); + for entry in bgl.entries.iter() { + match entry.ty { + wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Uniform, + has_dynamic_offset: true, + .. + } => {} + _ => continue, + }; + + binding_map.insert( + naga::ResourceBinding { + group: index as u32, + binding: entry.binding, + }, + hlsl::BindTarget { + binding_array_size: entry.count.map(NonZeroU32::get), + restrict_indexing: true, + ..bind_cbv + }, + ); + + parameters.push(Direct3D12::D3D12_ROOT_PARAMETER { + ParameterType: Direct3D12::D3D12_ROOT_PARAMETER_TYPE_CBV, + Anonymous: Direct3D12::D3D12_ROOT_PARAMETER_0 { + Descriptor: Direct3D12::D3D12_ROOT_DESCRIPTOR { + ShaderRegister: bind_cbv.register, + RegisterSpace: bind_cbv.space as u32, + }, + }, + ShaderVisibility: dynamic_buffers_visibility, + }); + + bind_cbv.register += entry.count.map_or(1, NonZeroU32::get); + } + + // Root constants for (offsets of) dynamic storage buffers + if dynamic_storage_buffers > 0 { + let parameter_index = parameters.len(); + + parameters.push(Direct3D12::D3D12_ROOT_PARAMETER { + ParameterType: Direct3D12::D3D12_ROOT_PARAMETER_TYPE_32BIT_CONSTANTS, + Anonymous: Direct3D12::D3D12_ROOT_PARAMETER_0 { + Constants: Direct3D12::D3D12_ROOT_CONSTANTS { + ShaderRegister: bind_cbv.register, + RegisterSpace: bind_cbv.space as u32, + Num32BitValues: dynamic_storage_buffers, + }, + }, + ShaderVisibility: conv::map_visibility(visibility_view_dynamic_storage), + }); + + let binding = hlsl::OffsetsBindTarget { + space: bind_cbv.space, + register: bind_cbv.register, + size: dynamic_storage_buffers, + }; + + bind_cbv.register += 1; + + dynamic_storage_buffer_offsets_targets.insert(index as u32, binding); + info.dynamic_storage_buffer_offsets = Some(DynamicStorageBufferOffsets { + root_index: parameter_index as u32, + range: total_dynamic_storage_buffers as usize + ..total_dynamic_storage_buffers as usize + dynamic_storage_buffers as usize, + }); + total_dynamic_storage_buffers += dynamic_storage_buffers; + } + + bind_group_infos[index] = Some(info); + } + + let sampler_heap_target = hlsl::SamplerHeapBindTargets { + standard_samplers: hlsl::BindTarget { + space: 0, + register: 0, + binding_array_size: None, + dynamic_storage_buffer_offsets_index: None, + restrict_indexing: false, + }, + comparison_samplers: hlsl::BindTarget { + space: 0, + register: 2048, + binding_array_size: None, + dynamic_storage_buffer_offsets_index: None, + restrict_indexing: false, + }, + }; + + let mut sampler_heap_root_index = None; + if sampler_in_any_bind_group { + // Sampler descriptor tables + // + // We bind two sampler ranges pointing to the same descriptor heap, using two different register ranges. + // + // We bind them as normal samplers in registers 0-2047 and comparison samplers in registers 2048-4095. + // Tier 2 hardware guarantees that the type of sampler only needs to match if the sampler is actually + // accessed in the shader. As such, we can bind the same array of samplers to both registers. + // + // We do this because HLSL does not allow you to alias registers at all. + let range_base = ranges.len(); + // Standard samplers, registers 0-2047 + ranges.push(Direct3D12::D3D12_DESCRIPTOR_RANGE { + RangeType: Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_SAMPLER, + NumDescriptors: 2048, + BaseShaderRegister: 0, + RegisterSpace: 0, + OffsetInDescriptorsFromTableStart: 0, + }); + // Comparison samplers, registers 2048-4095 + ranges.push(Direct3D12::D3D12_DESCRIPTOR_RANGE { + RangeType: Direct3D12::D3D12_DESCRIPTOR_RANGE_TYPE_SAMPLER, + NumDescriptors: 2048, + BaseShaderRegister: 2048, + RegisterSpace: 0, + OffsetInDescriptorsFromTableStart: 0, + }); + + let range = &ranges[range_base..]; + sampler_heap_root_index = Some(parameters.len() as super::RootIndex); + parameters.push(Direct3D12::D3D12_ROOT_PARAMETER { + ParameterType: Direct3D12::D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE, + Anonymous: Direct3D12::D3D12_ROOT_PARAMETER_0 { + DescriptorTable: Direct3D12::D3D12_ROOT_DESCRIPTOR_TABLE { + NumDescriptorRanges: range.len() as u32, + pDescriptorRanges: range.as_ptr(), + }, + }, + ShaderVisibility: Direct3D12::D3D12_SHADER_VISIBILITY_ALL, + }); + } + + // Ensure that we didn't reallocate! + debug_assert_eq!(ranges.len(), total_non_dynamic_entries); + + let (special_constants_root_index, special_constants_binding) = if desc.flags.intersects( + crate::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE + | crate::PipelineLayoutFlags::NUM_WORK_GROUPS, + ) { + let parameter_index = parameters.len(); + parameters.push(Direct3D12::D3D12_ROOT_PARAMETER { + ParameterType: Direct3D12::D3D12_ROOT_PARAMETER_TYPE_32BIT_CONSTANTS, + Anonymous: Direct3D12::D3D12_ROOT_PARAMETER_0 { + Constants: Direct3D12::D3D12_ROOT_CONSTANTS { + ShaderRegister: bind_cbv.register, + RegisterSpace: bind_cbv.space as u32, + Num32BitValues: 3, // 0 = first_vertex, 1 = first_instance, 2 = other + }, + }, + ShaderVisibility: Direct3D12::D3D12_SHADER_VISIBILITY_ALL, // really needed for VS and CS only, + }); + let binding = bind_cbv; + // This is the last time we use this, but lets increment + // it so if we add more later, the value behaves correctly. + + // This is an allow as it doesn't trigger on 1.90, hal's MSRV. + #[allow(unused_assignments)] + { + bind_cbv.register += 1; + } + (Some(parameter_index as u32), Some(binding)) + } else { + (None, None) + }; + + let blob = self.library.serialize_root_signature( + Direct3D12::D3D_ROOT_SIGNATURE_VERSION_1_0, + ¶meters, + &[], + Direct3D12::D3D12_ROOT_SIGNATURE_FLAG_ALLOW_INPUT_ASSEMBLER_INPUT_LAYOUT, + )?; + + let raw = unsafe { + self.raw + .CreateRootSignature::(0, blob.as_slice()) + } + .into_device_result("Root signature creation")?; + + let special_constants = if let Some(root_index) = special_constants_root_index { + let cmd_signatures = if desc + .flags + .contains(crate::PipelineLayoutFlags::INDIRECT_BUILTIN_UPDATE) + { + let constant_indirect_argument_desc = Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC { + Type: Direct3D12::D3D12_INDIRECT_ARGUMENT_TYPE_CONSTANT, + Anonymous: Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC_0 { + Constant: Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC_0_1 { + RootParameterIndex: root_index, + DestOffsetIn32BitValues: 0, + Num32BitValuesToSet: 3, + }, + }, + }; + let special_constant_buffer_args_len = { + // Hack: construct a dummy value of the special constants buffer value we need to + // fill, and calculate the size of each member. + let super::RootElement::SpecialConstantBuffer { + first_vertex, + first_instance, + other, + } = (super::RootElement::SpecialConstantBuffer { + first_vertex: 0, + first_instance: 0, + other: 0, + }) + else { + unreachable!(); + }; + size_of_val(&first_vertex) + size_of_val(&first_instance) + size_of_val(&other) + }; + + let draw_mesh = if self + .features + .features_wgpu + .contains(wgt::FeaturesWGPU::EXPERIMENTAL_MESH_SHADER) + { + Some(Self::create_command_signature( + &self.raw, + Some(&raw), + special_constant_buffer_args_len + size_of::(), + &[ + constant_indirect_argument_desc, + Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC { + Type: Direct3D12::D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH_MESH, + ..Default::default() + }, + ], + 0, + )?) + } else { + None + }; + + Some(super::CommandSignatures { + draw: Self::create_command_signature( + &self.raw, + Some(&raw), + special_constant_buffer_args_len + size_of::(), + &[ + constant_indirect_argument_desc, + Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC { + Type: Direct3D12::D3D12_INDIRECT_ARGUMENT_TYPE_DRAW, + ..Default::default() + }, + ], + 0, + )?, + draw_indexed: Self::create_command_signature( + &self.raw, + Some(&raw), + special_constant_buffer_args_len + + size_of::(), + &[ + constant_indirect_argument_desc, + Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC { + Type: Direct3D12::D3D12_INDIRECT_ARGUMENT_TYPE_DRAW_INDEXED, + ..Default::default() + }, + ], + 0, + )?, + draw_mesh, + dispatch: Self::create_command_signature( + &self.raw, + Some(&raw), + special_constant_buffer_args_len + size_of::(), + &[ + constant_indirect_argument_desc, + Direct3D12::D3D12_INDIRECT_ARGUMENT_DESC { + Type: Direct3D12::D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH, + ..Default::default() + }, + ], + 0, + )?, + }) + } else { + None + }; + Some(super::PipelineLayoutSpecialConstants { + root_index, + indirect_cmd_signatures: cmd_signatures, + }) + } else { + None + }; + + if let Some(label) = desc.label { + raw.set_name(label)?; + } + + self.counters.pipeline_layouts.add(1); + + Ok(super::PipelineLayout { + shared: super::PipelineLayoutShared { + signature: Some(raw), + total_root_elements: parameters.len() as super::RootIndex, + special_constants, + root_constant_info, + sampler_heap_root_index, + }, + bind_group_infos, + naga_options: hlsl::Options { + shader_model: self.shared.private_caps.shader_model, + binding_map, + fake_missing_bindings: false, + special_constants_binding, + immediates_target, + dynamic_storage_buffer_offsets_targets, + zero_initialize_workgroup_memory: true, + restrict_indexing: true, + sampler_heap_target, + sampler_buffer_binding_map, + external_texture_binding_map, + force_loop_bounding: true, + ray_query_initialization_tracking: true, + }, + }) + } + + unsafe fn destroy_pipeline_layout(&self, _pipeline_layout: super::PipelineLayout) { + self.counters.pipeline_layouts.sub(1); + } + + unsafe fn create_bind_group( + &self, + desc: &crate::BindGroupDescriptor< + super::BindGroupLayout, + super::Buffer, + super::Sampler, + super::TextureView, + super::AccelerationStructure, + >, + ) -> Result { + let mut cpu_views = desc + .layout + .cpu_heap_views + .as_ref() + .map(|cpu_heap| cpu_heap.inner.lock()); + if let Some(ref mut inner) = cpu_views { + inner.stage.clear(); + } + let mut dynamic_buffers = Vec::new(); + + let layout_and_entry_iter = desc.entries.iter().map(|entry| { + let layout = desc + .layout + .entries + .iter() + .find(|layout_entry| layout_entry.binding == entry.binding) + .expect("internal error: no layout entry found with binding slot"); + (layout, entry) + }); + let mut sampler_indexes: Vec = Vec::new(); + + for (layout, entry) in layout_and_entry_iter { + match layout.ty { + wgt::BindingType::Buffer { + ty, + has_dynamic_offset, + .. + } => { + let start = entry.resource_index as usize; + let end = start + entry.count as usize; + for data in &desc.buffers[start..end] { + let gpu_address = data.resolve_address(); + let mut size = data.resolve_size().try_into().unwrap(); + + if has_dynamic_offset { + match ty { + wgt::BufferBindingType::Uniform => { + dynamic_buffers.push(super::DynamicBuffer::Uniform( + Direct3D12::D3D12_GPU_DESCRIPTOR_HANDLE { + ptr: data.resolve_address(), + }, + )); + continue; + } + wgt::BufferBindingType::Storage { .. } => { + size = (data.buffer.size - data.offset) as u32; + dynamic_buffers.push(super::DynamicBuffer::Storage); + } + } + } + + let inner = cpu_views.as_mut().unwrap(); + let cpu_index = inner.stage.len() as u32; + let handle = desc.layout.cpu_heap_views.as_ref().unwrap().at(cpu_index); + match ty { + wgt::BufferBindingType::Uniform => { + let size_mask = + Direct3D12::D3D12_CONSTANT_BUFFER_DATA_PLACEMENT_ALIGNMENT - 1; + let raw_desc = Direct3D12::D3D12_CONSTANT_BUFFER_VIEW_DESC { + BufferLocation: gpu_address, + SizeInBytes: ((size - 1) | size_mask) + 1, + }; + unsafe { + self.raw.CreateConstantBufferView(Some(&raw_desc), handle) + }; + } + wgt::BufferBindingType::Storage { read_only: true } => { + let raw_desc = Direct3D12::D3D12_SHADER_RESOURCE_VIEW_DESC { + Format: Dxgi::Common::DXGI_FORMAT_R32_TYPELESS, + Shader4ComponentMapping: + Direct3D12::D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING, + ViewDimension: Direct3D12::D3D12_SRV_DIMENSION_BUFFER, + Anonymous: Direct3D12::D3D12_SHADER_RESOURCE_VIEW_DESC_0 { + Buffer: Direct3D12::D3D12_BUFFER_SRV { + FirstElement: data.offset / 4, + NumElements: size / 4, + StructureByteStride: 0, + Flags: Direct3D12::D3D12_BUFFER_SRV_FLAG_RAW, + }, + }, + }; + unsafe { + self.raw.CreateShaderResourceView( + &data.buffer.resource, + Some(&raw_desc), + handle, + ) + }; + } + wgt::BufferBindingType::Storage { read_only: false } => { + let raw_desc = Direct3D12::D3D12_UNORDERED_ACCESS_VIEW_DESC { + Format: Dxgi::Common::DXGI_FORMAT_R32_TYPELESS, + ViewDimension: Direct3D12::D3D12_UAV_DIMENSION_BUFFER, + Anonymous: Direct3D12::D3D12_UNORDERED_ACCESS_VIEW_DESC_0 { + Buffer: Direct3D12::D3D12_BUFFER_UAV { + FirstElement: data.offset / 4, + NumElements: size / 4, + StructureByteStride: 0, + CounterOffsetInBytes: 0, + Flags: Direct3D12::D3D12_BUFFER_UAV_FLAG_RAW, + }, + }, + }; + unsafe { + self.raw.CreateUnorderedAccessView( + &data.buffer.resource, + None, + Some(&raw_desc), + handle, + ) + }; + } + } + inner.stage.push(handle); + } + } + wgt::BindingType::Texture { .. } => { + let start = entry.resource_index as usize; + let end = start + entry.count as usize; + for data in &desc.textures[start..end] { + let handle = data.view.handle_srv.unwrap(); + cpu_views.as_mut().unwrap().stage.push(handle.raw); + } + } + wgt::BindingType::StorageTexture { .. } => { + let start = entry.resource_index as usize; + let end = start + entry.count as usize; + for data in &desc.textures[start..end] { + let handle = data.view.handle_uav.unwrap(); + cpu_views.as_mut().unwrap().stage.push(handle.raw); + } + } + wgt::BindingType::Sampler { .. } => { + let start = entry.resource_index as usize; + let end = start + entry.count as usize; + for &data in &desc.samplers[start..end] { + sampler_indexes.push(data.index); + } + } + wgt::BindingType::AccelerationStructure { .. } => { + let start = entry.resource_index as usize; + let end = start + entry.count as usize; + for data in &desc.acceleration_structures[start..end] { + let inner = cpu_views.as_mut().unwrap(); + let cpu_index = inner.stage.len() as u32; + let handle = desc.layout.cpu_heap_views.as_ref().unwrap().at(cpu_index); + let raw_desc = Direct3D12::D3D12_SHADER_RESOURCE_VIEW_DESC { + Format: Dxgi::Common::DXGI_FORMAT_UNKNOWN, + Shader4ComponentMapping: + Direct3D12::D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING, + ViewDimension: + Direct3D12::D3D12_SRV_DIMENSION_RAYTRACING_ACCELERATION_STRUCTURE, + Anonymous: Direct3D12::D3D12_SHADER_RESOURCE_VIEW_DESC_0 { + RaytracingAccelerationStructure: + Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_SRV { + Location: unsafe { data.resource.GetGPUVirtualAddress() }, + }, + }, + }; + unsafe { + self.raw + .CreateShaderResourceView(None, Some(&raw_desc), handle) + }; + inner.stage.push(handle); + } + } + wgt::BindingType::ExternalTexture => { + // We don't yet support binding arrays of external textures. + // https://github.com/gfx-rs/wgpu/issues/8027 + assert_eq!(entry.count, 1); + let external_texture = &desc.external_textures[entry.resource_index as usize]; + for plane in &external_texture.planes { + let plane_handle = plane.view.handle_srv.unwrap(); + cpu_views.as_mut().unwrap().stage.push(plane_handle.raw); + } + let gpu_address = external_texture.params.resolve_address(); + let size = external_texture.params.resolve_size() as u32; + let inner = cpu_views.as_mut().unwrap(); + let cpu_index = inner.stage.len() as u32; + let params_handle = desc.layout.cpu_heap_views.as_ref().unwrap().at(cpu_index); + let size_mask = Direct3D12::D3D12_CONSTANT_BUFFER_DATA_PLACEMENT_ALIGNMENT - 1; + let raw_desc = Direct3D12::D3D12_CONSTANT_BUFFER_VIEW_DESC { + BufferLocation: gpu_address, + SizeInBytes: ((size - 1) | size_mask) + 1, + }; + unsafe { + self.raw + .CreateConstantBufferView(Some(&raw_desc), params_handle) + }; + inner.stage.push(params_handle); + } + } + } + + let sampler_index_buffer = if !sampler_indexes.is_empty() { + let buffer_size = (sampler_indexes.len() * size_of::()) as u64; + + let label = if let Some(label) = desc.label { + Cow::Owned(format!("{label} (Internal Sampler Index Buffer)")) + } else { + Cow::Borrowed("Internal Sampler Index Buffer") + }; + + let buffer_desc = crate::BufferDescriptor { + label: Some(&label), + size: buffer_size, + usage: wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::MAP_WRITE, + // D3D12 backend doesn't care about the memory flags + memory_flags: crate::MemoryFlags::empty(), + }; + + let (buffer, allocation) = + suballocation::DeviceAllocationContext::from(self).create_buffer(&buffer_desc)?; + + let mut mapping = ptr::null_mut::(); + unsafe { buffer.Map(0, None, Some(&mut mapping)) }.into_device_result("Map")?; + + assert!(!mapping.is_null()); + assert_eq!(mapping as usize % 4, 0); + + unsafe { + ptr::copy_nonoverlapping( + sampler_indexes.as_ptr(), + mapping.cast(), + sampler_indexes.len(), + ) + }; + + // The unmapping is not needed, as all memory is coherent in d3d12, but lets be nice to our address space. + unsafe { buffer.Unmap(0, None) }; + + let srv_desc = Direct3D12::D3D12_SHADER_RESOURCE_VIEW_DESC { + Format: Dxgi::Common::DXGI_FORMAT_UNKNOWN, + ViewDimension: Direct3D12::D3D12_SRV_DIMENSION_BUFFER, + Anonymous: Direct3D12::D3D12_SHADER_RESOURCE_VIEW_DESC_0 { + Buffer: Direct3D12::D3D12_BUFFER_SRV { + FirstElement: 0, + NumElements: sampler_indexes.len() as u32, + StructureByteStride: 4, + Flags: Direct3D12::D3D12_BUFFER_SRV_FLAG_NONE, + }, + }, + Shader4ComponentMapping: Direct3D12::D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING, + }; + + let inner = cpu_views.as_mut().unwrap(); + let cpu_index = inner.stage.len() as u32; + let srv = desc.layout.cpu_heap_views.as_ref().unwrap().at(cpu_index); + + unsafe { + self.raw + .CreateShaderResourceView(&buffer, Some(&srv_desc), srv) + }; + + cpu_views.as_mut().unwrap().stage.push(srv); + + Some(super::SamplerIndexBuffer { buffer, allocation }) + } else { + None + }; + + let handle_views = match cpu_views { + Some(inner) => { + let dual = unsafe { + descriptor::upload( + &self.raw, + &inner, + &self.shared.heap_views, + &desc.layout.copy_counts, + ) + }?; + Some(dual) + } + None => None, + }; + + self.counters.bind_groups.add(1); + + Ok(super::BindGroup { + handle_views, + sampler_index_buffer, + dynamic_buffers, + }) + } + + unsafe fn destroy_bind_group(&self, group: super::BindGroup) { + if let Some(dual) = group.handle_views { + self.shared.heap_views.free_slice(dual); + } + + if let Some(sampler_buffer) = group.sampler_index_buffer { + suballocation::DeviceAllocationContext::from(self) + .free_resource(sampler_buffer.buffer, sampler_buffer.allocation); + } + + self.counters.bind_groups.sub(1); + } + + unsafe fn create_shader_module( + &self, + desc: &crate::ShaderModuleDescriptor, + shader: crate::ShaderInput, + ) -> Result { + self.counters.shader_modules.add(1); + + let raw_name = desc + .label + .and_then(|label| alloc::ffi::CString::new(label).ok()); + match shader { + crate::ShaderInput::Naga(naga) => Ok(super::ShaderModule { + source: super::ShaderModuleSource::Naga(naga), + raw_name, + runtime_checks: desc.runtime_checks, + }), + crate::ShaderInput::Dxil { + shader, + num_workgroups, + } => Ok(super::ShaderModule { + source: super::ShaderModuleSource::DxilPassthrough(super::DxilPassthroughShader { + shader: shader.to_vec(), + num_workgroups, + }), + raw_name, + runtime_checks: desc.runtime_checks, + }), + crate::ShaderInput::Hlsl { + shader, + num_workgroups, + } => Ok(super::ShaderModule { + source: super::ShaderModuleSource::HlslPassthrough(super::HlslPassthroughShader { + shader: shader.to_owned(), + num_workgroups, + }), + raw_name, + runtime_checks: desc.runtime_checks, + }), + crate::ShaderInput::SpirV(_) + | crate::ShaderInput::MetalLib { .. } + | crate::ShaderInput::Msl { .. } + | crate::ShaderInput::Glsl { .. } => { + unreachable!() + } + } + } + unsafe fn destroy_shader_module(&self, _module: super::ShaderModule) { + self.counters.shader_modules.sub(1); + // just drop + } + + unsafe fn create_render_pipeline( + &self, + desc: &crate::RenderPipelineDescriptor< + super::PipelineLayout, + super::ShaderModule, + super::PipelineCache, + >, + ) -> Result { + let mut shader_stages = wgt::ShaderStages::empty(); + let (topology_class, topology) = conv::map_topology(desc.primitive.topology); + let mut rtv_formats = [Dxgi::Common::DXGI_FORMAT_UNKNOWN; + Direct3D12::D3D12_SIMULTANEOUS_RENDER_TARGET_COUNT as usize]; + for (rtv_format, ct) in rtv_formats.iter_mut().zip(desc.color_targets) { + if let Some(ct) = ct.as_ref() { + *rtv_format = auxil::dxgi::conv::map_texture_format(ct.format); + } + } + + let bias = desc + .depth_stencil + .as_ref() + .map(|ds| ds.bias) + .unwrap_or_default(); + + let rasterizer_state = Direct3D12::D3D12_RASTERIZER_DESC { + FillMode: conv::map_polygon_mode(desc.primitive.polygon_mode), + CullMode: match desc.primitive.cull_mode { + None => Direct3D12::D3D12_CULL_MODE_NONE, + Some(wgt::Face::Front) => Direct3D12::D3D12_CULL_MODE_FRONT, + Some(wgt::Face::Back) => Direct3D12::D3D12_CULL_MODE_BACK, + }, + FrontCounterClockwise: match desc.primitive.front_face { + wgt::FrontFace::Cw => Foundation::FALSE, + wgt::FrontFace::Ccw => Foundation::TRUE, + }, + DepthBias: bias.constant, + DepthBiasClamp: bias.clamp, + SlopeScaledDepthBias: bias.slope_scale, + DepthClipEnable: windows_core::BOOL::from(!desc.primitive.unclipped_depth), + MultisampleEnable: windows_core::BOOL::from(desc.multisample.count > 1), + ForcedSampleCount: 0, + AntialiasedLineEnable: false.into(), + ConservativeRaster: if desc.primitive.conservative { + Direct3D12::D3D12_CONSERVATIVE_RASTERIZATION_MODE_ON + } else { + Direct3D12::D3D12_CONSERVATIVE_RASTERIZATION_MODE_OFF + }, + }; + + let blob_fs = match desc.fragment_stage { + Some(ref stage) => { + shader_stages |= wgt::ShaderStages::FRAGMENT; + Some(self.load_shader(stage, desc.layout, naga::ShaderStage::Fragment, None)?) + } + None => None, + }; + let pixel_shader = match &blob_fs { + Some(shader) => shader.create_native_shader(), + None => Direct3D12::D3D12_SHADER_BYTECODE::default(), + }; + let stream_output = Direct3D12::D3D12_STREAM_OUTPUT_DESC { + pSODeclaration: ptr::null(), + NumEntries: 0, + pBufferStrides: ptr::null(), + NumStrides: 0, + RasterizedStream: 0, + }; + let blend_state = Direct3D12::D3D12_BLEND_DESC { + AlphaToCoverageEnable: windows_core::BOOL::from( + desc.multisample.alpha_to_coverage_enabled, + ), + IndependentBlendEnable: true.into(), + RenderTarget: conv::map_render_targets(desc.color_targets), + }; + let depth_stencil_state = match desc.depth_stencil { + Some(ref ds) => conv::map_depth_stencil(ds), + None => Default::default(), + }; + let dsv_format = desc + .depth_stencil + .as_ref() + .map_or(Dxgi::Common::DXGI_FORMAT_UNKNOWN, |ds| { + auxil::dxgi::conv::map_texture_format(ds.format) + }); + let sample_desc = Dxgi::Common::DXGI_SAMPLE_DESC { + Count: desc.multisample.count, + Quality: 0, + }; + let cached_pso = Direct3D12::D3D12_CACHED_PIPELINE_STATE { + pCachedBlob: ptr::null(), + CachedBlobSizeInBytes: 0, + }; + let flags = Direct3D12::D3D12_PIPELINE_STATE_FLAG_NONE; + + let mut view_instancing = ArrayVec::::new(); + if let Some(mask) = desc.multiview_mask { + let mask = mask.get(); + // This array is just what _could_ be rendered to. We actually apply the mask at + // renderpass creation time. The `view_index` passed to the shader depends on the + // view's index in this array, so if we include every view in this array, `view_index` + // actually the texture array layer, like in vulkan. + for i in 0..32 - mask.leading_zeros() { + view_instancing.push(Direct3D12::D3D12_VIEW_INSTANCE_LOCATION { + ViewportArrayIndex: 0, + RenderTargetArrayIndex: i, + }); + } + } + + // Borrow view instancing slice, so we can be sure that it won't be moved while we have pointers into this buffer. + let view_instancing_slice = view_instancing.as_slice(); + + let mut stream_desc = RenderPipelineStateStreamDesc { + // Shared by vertex and mesh pipelines + root_signature: desc.layout.shared.signature.as_ref(), + pixel_shader, + blend_state, + sample_mask: desc.multisample.mask as u32, + rasterizer_state, + depth_stencil_state, + primitive_topology_type: topology_class, + rtv_formats: Direct3D12::D3D12_RT_FORMAT_ARRAY { + RTFormats: rtv_formats, + NumRenderTargets: desc.color_targets.len() as u32, + }, + dsv_format, + sample_desc, + node_mask: 0, + cached_pso, + flags, + view_instancing: if !view_instancing_slice.is_empty() { + Some(Direct3D12::D3D12_VIEW_INSTANCING_DESC { + ViewInstanceCount: view_instancing_slice.len() as u32, + pViewInstanceLocations: view_instancing_slice.as_ptr(), + // This lets us hide/mask certain values later, at renderpass creation time. + Flags: Direct3D12::D3D12_VIEW_INSTANCING_FLAG_ENABLE_VIEW_INSTANCE_MASKING, + }) + } else { + None + }, + + // Optional data that depends on the pipeline type (vertex vs mesh). + vertex_shader: Default::default(), + input_layout: Default::default(), + index_buffer_strip_cut_value: Default::default(), + stream_output, + task_shader: Default::default(), + mesh_shader: Default::default(), + }; + let mut input_element_descs = Vec::new(); + let blob_vs; + let blob_ts; + let blob_ms; + let mut vertex_strides = [None; crate::MAX_VERTEX_BUFFERS]; + match &desc.vertex_processor { + &crate::VertexProcessor::Standard { + vertex_buffers, + ref vertex_stage, + } => { + shader_stages |= wgt::ShaderStages::VERTEX; + blob_vs = Some(self.load_shader( + vertex_stage, + desc.layout, + naga::ShaderStage::Vertex, + desc.fragment_stage.as_ref(), + )?); + + for (i, (stride, vbuf)) in vertex_strides.iter_mut().zip(vertex_buffers).enumerate() + { + *stride = Some(vbuf.array_stride as u32); + let (slot_class, step_rate) = match vbuf.step_mode { + wgt::VertexStepMode::Vertex => { + (Direct3D12::D3D12_INPUT_CLASSIFICATION_PER_VERTEX_DATA, 0) + } + wgt::VertexStepMode::Instance => { + (Direct3D12::D3D12_INPUT_CLASSIFICATION_PER_INSTANCE_DATA, 1) + } + }; + for attribute in vbuf.attributes { + input_element_descs.push(Direct3D12::D3D12_INPUT_ELEMENT_DESC { + SemanticName: windows::core::PCSTR(NAGA_LOCATION_SEMANTIC.as_ptr()), + SemanticIndex: attribute.shader_location, + Format: auxil::dxgi::conv::map_vertex_format(attribute.format), + InputSlot: i as u32, + AlignedByteOffset: attribute.offset as u32, + InputSlotClass: slot_class, + InstanceDataStepRate: step_rate, + }); + } + } + stream_desc.vertex_shader = blob_vs.as_ref().unwrap().create_native_shader(); + stream_desc.input_layout = Direct3D12::D3D12_INPUT_LAYOUT_DESC { + pInputElementDescs: if input_element_descs.is_empty() { + ptr::null() + } else { + input_element_descs.as_ptr() + }, + NumElements: input_element_descs.len() as u32, + }; + stream_desc.index_buffer_strip_cut_value = match desc.primitive.strip_index_format { + Some(wgt::IndexFormat::Uint16) => { + Direct3D12::D3D12_INDEX_BUFFER_STRIP_CUT_VALUE_0xFFFF + } + Some(wgt::IndexFormat::Uint32) => { + Direct3D12::D3D12_INDEX_BUFFER_STRIP_CUT_VALUE_0xFFFFFFFF + } + None => Direct3D12::D3D12_INDEX_BUFFER_STRIP_CUT_VALUE_DISABLED, + }; + stream_desc.stream_output = Direct3D12::D3D12_STREAM_OUTPUT_DESC { + pSODeclaration: ptr::null(), + NumEntries: 0, + pBufferStrides: ptr::null(), + NumStrides: 0, + RasterizedStream: 0, + }; + } + crate::VertexProcessor::Mesh { + task_stage, + mesh_stage, + } => { + blob_ts = if let Some(ts) = task_stage { + shader_stages |= wgt::ShaderStages::TASK; + Some(self.load_shader( + ts, + desc.layout, + naga::ShaderStage::Task, + desc.fragment_stage.as_ref(), + )?) + } else { + None + }; + let task_shader = if let Some(ts) = &blob_ts { + ts.create_native_shader() + } else { + Default::default() + }; + shader_stages |= wgt::ShaderStages::MESH; + blob_ms = Some(self.load_shader( + mesh_stage, + desc.layout, + naga::ShaderStage::Mesh, + desc.fragment_stage.as_ref(), + )?); + stream_desc.task_shader = task_shader; + stream_desc.mesh_shader = blob_ms.as_ref().unwrap().create_native_shader(); + } + }; + let raw: Direct3D12::ID3D12PipelineState = + // If stream descriptors are available, use them as they are more flexible. + if let Ok(device) = self.raw.cast::() { + // Prefer stream descs where possible + let mut stream = stream_desc.to_stream(); + unsafe { + profiling::scope!("ID3D12Device2::CreatePipelineState"); + stream.create_pipeline_state(&device).map_err(|err| { + crate::PipelineError::Linkage(shader_stages, err.to_string()) + })? + } + } else { + unsafe { + // Safety: `stream_desc` entirely outlives the `desc`. + let desc = stream_desc.to_graphics_pipeline_descriptor(); + self.raw.CreateGraphicsPipelineState(&desc).map_err(|err| { + crate::PipelineError::Linkage(shader_stages, err.to_string()) + })? + } + }; + + if let Some(label) = desc.label { + raw.set_name(label)?; + } + + self.counters.render_pipelines.add(1); + + Ok(super::RenderPipeline { + raw, + layout: desc.layout.shared.clone(), + topology, + vertex_strides, + }) + } + + unsafe fn destroy_render_pipeline(&self, _pipeline: super::RenderPipeline) { + self.counters.render_pipelines.sub(1); + } + + unsafe fn create_compute_pipeline( + &self, + desc: &crate::ComputePipelineDescriptor< + super::PipelineLayout, + super::ShaderModule, + super::PipelineCache, + >, + ) -> Result { + let blob_cs = + self.load_shader(&desc.stage, desc.layout, naga::ShaderStage::Compute, None)?; + + let pair = { + profiling::scope!("ID3D12Device::CreateComputePipelineState"); + unsafe { + self.raw.CreateComputePipelineState( + &Direct3D12::D3D12_COMPUTE_PIPELINE_STATE_DESC { + pRootSignature: borrow_optional_interface_temporarily( + &desc.layout.shared.signature, + ), + CS: blob_cs.create_native_shader(), + NodeMask: 0, + CachedPSO: Direct3D12::D3D12_CACHED_PIPELINE_STATE::default(), + Flags: Direct3D12::D3D12_PIPELINE_STATE_FLAG_NONE, + }, + ) + } + }; + + let raw: Direct3D12::ID3D12PipelineState = pair.map_err(|err| { + crate::PipelineError::Linkage(wgt::ShaderStages::COMPUTE, err.to_string()) + })?; + + if let Some(label) = desc.label { + raw.set_name(label)?; + } + + self.counters.compute_pipelines.add(1); + + Ok(super::ComputePipeline { + raw, + layout: desc.layout.shared.clone(), + }) + } + + unsafe fn destroy_compute_pipeline(&self, _pipeline: super::ComputePipeline) { + self.counters.compute_pipelines.sub(1); + } + + unsafe fn create_pipeline_cache( + &self, + _desc: &crate::PipelineCacheDescriptor<'_>, + ) -> Result { + Ok(super::PipelineCache) + } + unsafe fn destroy_pipeline_cache(&self, _: super::PipelineCache) {} + + unsafe fn create_query_set( + &self, + desc: &wgt::QuerySetDescriptor, + ) -> Result { + let (heap_ty, raw_ty) = match desc.ty { + wgt::QueryType::Occlusion => ( + Direct3D12::D3D12_QUERY_HEAP_TYPE_OCCLUSION, + Direct3D12::D3D12_QUERY_TYPE_BINARY_OCCLUSION, + ), + wgt::QueryType::PipelineStatistics(_) => ( + Direct3D12::D3D12_QUERY_HEAP_TYPE_PIPELINE_STATISTICS, + Direct3D12::D3D12_QUERY_TYPE_PIPELINE_STATISTICS, + ), + wgt::QueryType::Timestamp => ( + Direct3D12::D3D12_QUERY_HEAP_TYPE_TIMESTAMP, + Direct3D12::D3D12_QUERY_TYPE_TIMESTAMP, + ), + }; + + if let Some(threshold) = self + .mem_allocator + .memory_budget_thresholds + .for_resource_creation + { + let info = self + .shared + .adapter + .query_video_memory_info(Dxgi::DXGI_MEMORY_SEGMENT_GROUP_LOCAL)?; + + // Assume each query is 256 bytes. + // On an AMD W6800 with driver version 32.0.12030.9, occlusion and pipeline statistics are 256, timestamp is 8. + + if info.CurrentUsage + desc.count as u64 * 256 >= info.Budget / 100 * threshold as u64 { + return Err(crate::DeviceError::OutOfMemory); + } + } + + let mut raw = None::; + unsafe { + self.raw.CreateQueryHeap( + &Direct3D12::D3D12_QUERY_HEAP_DESC { + Type: heap_ty, + Count: desc.count, + NodeMask: 0, + }, + &mut raw, + ) + } + .into_device_result("Query heap creation")?; + + let raw = raw.ok_or(crate::DeviceError::Unexpected)?; + + if let Some(label) = desc.label { + raw.set_name(label)?; + } + + self.counters.query_sets.add(1); + + Ok(super::QuerySet { raw, raw_ty }) + } + + unsafe fn destroy_query_set(&self, _set: super::QuerySet) { + self.counters.query_sets.sub(1); + } + + unsafe fn create_fence(&self) -> Result { + let raw: Direct3D12::ID3D12Fence = + unsafe { self.raw.CreateFence(0, Direct3D12::D3D12_FENCE_FLAG_SHARED) } + .into_device_result("Fence creation")?; + + self.counters.fences.add(1); + + Ok(super::Fence { raw }) + } + unsafe fn destroy_fence(&self, _fence: super::Fence) { + self.counters.fences.sub(1); + } + + unsafe fn get_fence_value( + &self, + fence: &super::Fence, + ) -> Result { + Ok(unsafe { fence.raw.GetCompletedValue() }) + } + unsafe fn wait( + &self, + fence: &super::Fence, + value: crate::FenceValue, + timeout: Option, + ) -> Result { + let timeout = timeout.unwrap_or(Duration::MAX); + + // We first check if the fence has already reached the value we're waiting for. + let mut fence_value = unsafe { fence.raw.GetCompletedValue() }; + if fence_value >= value { + return Ok(true); + } + + let event = Event::create(false, false)?; + + unsafe { fence.raw.SetEventOnCompletion(value, event.0) } + .into_device_result("Set event")?; + + let start_time = Instant::now(); + + // We need to loop to get correct behavior when timeouts are involved. + // + // wait(0): + // - We set the event from the fence value 0. + // - WaitForSingleObject times out, we return false. + // + // wait(1): + // - We set the event from the fence value 1. + // - WaitForSingleObject returns. However we do not know if the fence value is 0 or 1, + // just that _something_ triggered the event. We check the fence value, and if it is + // 1, we return true. Otherwise, we loop and wait again. + loop { + let elapsed = start_time.elapsed(); + + // We need to explicitly use checked_sub. Overflow with duration panics, and if the + // timing works out just right, we can get a negative remaining wait duration. + // + // This happens when a previous iteration WaitForSingleObject succeeded with a previous fence value, + // right before the timeout would have been hit. + let remaining_wait_duration = match timeout.checked_sub(elapsed) { + Some(remaining) => remaining, + None => { + log::trace!("Timeout elapsed in between waits!"); + break Ok(false); + } + }; + + log::trace!("Waiting for fence value {value} for {remaining_wait_duration:?}"); + + match unsafe { + Threading::WaitForSingleObject( + event.0, + remaining_wait_duration.as_millis().min(u32::MAX as u128) as u32, + ) + } { + Foundation::WAIT_OBJECT_0 => {} + Foundation::WAIT_ABANDONED | Foundation::WAIT_FAILED => { + log::error!("Wait failed!"); + break Err(crate::DeviceError::Lost); + } + Foundation::WAIT_TIMEOUT => { + log::trace!("Wait timed out!"); + break Ok(false); + } + other => { + log::error!("Unexpected wait status: 0x{other:?}"); + break Err(crate::DeviceError::Lost); + } + }; + + fence_value = unsafe { fence.raw.GetCompletedValue() }; + log::trace!("Wait complete! Fence actual value: {fence_value}"); + + if fence_value >= value { + break Ok(true); + } + } + } + + unsafe fn start_graphics_debugger_capture(&self) -> bool { + #[cfg(feature = "renderdoc")] + { + unsafe { + self.render_doc + .start_frame_capture(self.raw.as_raw(), ptr::null_mut()) + } + } + #[cfg(not(feature = "renderdoc"))] + false + } + + unsafe fn stop_graphics_debugger_capture(&self) { + #[cfg(feature = "renderdoc")] + unsafe { + self.render_doc + .end_frame_capture(self.raw.as_raw(), ptr::null_mut()) + } + } + + unsafe fn get_acceleration_structure_build_sizes<'a>( + &self, + desc: &crate::GetAccelerationStructureBuildSizesDescriptor<'a, super::Buffer>, + ) -> crate::AccelerationStructureBuildSizes { + let mut geometry_desc; + let device5 = self.raw.cast::().unwrap(); + let ty; + let inputs0; + let num_desc; + match desc.entries { + AccelerationStructureEntries::Instances(instances) => { + ty = Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL; + inputs0 = Direct3D12::D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS_0 { + InstanceDescs: 0, + }; + num_desc = instances.count; + } + AccelerationStructureEntries::Triangles(triangles) => { + geometry_desc = Vec::with_capacity(triangles.len()); + for triangle in triangles { + let index_format = triangle + .indices + .as_ref() + .map_or(Dxgi::Common::DXGI_FORMAT_UNKNOWN, |indices| { + auxil::dxgi::conv::map_index_format(indices.format) + }); + let index_count = triangle.indices.as_ref().map_or(0, |indices| indices.count); + + let triangle_desc = Direct3D12::D3D12_RAYTRACING_GEOMETRY_TRIANGLES_DESC { + // https://learn.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device5-getraytracingaccelerationstructureprebuildinfo + // It may not inspect/dereference any GPU virtual addresses, other than + // to check to see if a pointer is NULL or not, such as the optional + // transform in D3D12_RAYTRACING_GEOMETRY_TRIANGLES_DESC, without + // dereferencing it. + // + // This suggests we could pass a non-zero invalid address here if fetching the + // real address has significant overhead, but we pass the real one to be on the + // safe side for now. + Transform3x4: if desc + .flags + .contains(wgt::AccelerationStructureFlags::USE_TRANSFORM) + { + unsafe { + triangle + .transform + .as_ref() + .unwrap() + .buffer + .resource + .GetGPUVirtualAddress() + } + } else { + 0 + }, + IndexFormat: index_format, + VertexFormat: auxil::dxgi::conv::map_vertex_format(triangle.vertex_format), + IndexCount: index_count, + VertexCount: triangle.vertex_count, + IndexBuffer: 0, + VertexBuffer: Direct3D12::D3D12_GPU_VIRTUAL_ADDRESS_AND_STRIDE { + StartAddress: 0, + StrideInBytes: triangle.vertex_stride, + }, + }; + + geometry_desc.push(Direct3D12::D3D12_RAYTRACING_GEOMETRY_DESC { + Type: Direct3D12::D3D12_RAYTRACING_GEOMETRY_TYPE_TRIANGLES, + Flags: conv::map_acceleration_structure_geometry_flags(triangle.flags), + Anonymous: Direct3D12::D3D12_RAYTRACING_GEOMETRY_DESC_0 { + Triangles: triangle_desc, + }, + }) + } + ty = Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL; + inputs0 = Direct3D12::D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS_0 { + pGeometryDescs: geometry_desc.as_ptr(), + }; + num_desc = geometry_desc.len() as u32; + } + AccelerationStructureEntries::AABBs(aabbs) => { + geometry_desc = Vec::with_capacity(aabbs.len()); + for aabb in aabbs { + let aabb_desc = Direct3D12::D3D12_RAYTRACING_GEOMETRY_AABBS_DESC { + AABBCount: aabb.count as u64, + AABBs: Direct3D12::D3D12_GPU_VIRTUAL_ADDRESS_AND_STRIDE { + StartAddress: 0, + StrideInBytes: aabb.stride, + }, + }; + geometry_desc.push(Direct3D12::D3D12_RAYTRACING_GEOMETRY_DESC { + Type: Direct3D12::D3D12_RAYTRACING_GEOMETRY_TYPE_PROCEDURAL_PRIMITIVE_AABBS, + Flags: conv::map_acceleration_structure_geometry_flags(aabb.flags), + Anonymous: Direct3D12::D3D12_RAYTRACING_GEOMETRY_DESC_0 { + AABBs: aabb_desc, + }, + }) + } + ty = Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL; + inputs0 = Direct3D12::D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS_0 { + pGeometryDescs: geometry_desc.as_ptr(), + }; + num_desc = geometry_desc.len() as u32; + } + }; + let acceleration_structure_inputs = + Direct3D12::D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_INPUTS { + Type: ty, + Flags: conv::map_acceleration_structure_build_flags(desc.flags, None), + NumDescs: num_desc, + DescsLayout: Direct3D12::D3D12_ELEMENTS_LAYOUT_ARRAY, + Anonymous: inputs0, + }; + let mut info = Direct3D12::D3D12_RAYTRACING_ACCELERATION_STRUCTURE_PREBUILD_INFO::default(); + unsafe { + device5.GetRaytracingAccelerationStructurePrebuildInfo( + &acceleration_structure_inputs, + &mut info, + ) + }; + crate::AccelerationStructureBuildSizes { + acceleration_structure_size: info.ResultDataMaxSizeInBytes, + update_scratch_size: info.UpdateScratchDataSizeInBytes, + build_scratch_size: info.ScratchDataSizeInBytes, + } + } + + unsafe fn get_acceleration_structure_device_address( + &self, + acceleration_structure: &super::AccelerationStructure, + ) -> wgt::BufferAddress { + unsafe { acceleration_structure.resource.GetGPUVirtualAddress() } + } + + unsafe fn create_acceleration_structure( + &self, + desc: &crate::AccelerationStructureDescriptor, + ) -> Result { + // Create a D3D12 resource as per-usual. + let size = desc.size; + + let raw_desc = Direct3D12::D3D12_RESOURCE_DESC { + Dimension: Direct3D12::D3D12_RESOURCE_DIMENSION_BUFFER, + Alignment: 0, + Width: size, + Height: 1, + DepthOrArraySize: 1, + MipLevels: 1, + Format: Dxgi::Common::DXGI_FORMAT_UNKNOWN, + SampleDesc: Dxgi::Common::DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Layout: Direct3D12::D3D12_TEXTURE_LAYOUT_ROW_MAJOR, + // TODO: when moving to enhanced barriers use Direct3D12::D3D12_RESOURCE_FLAG_RAYTRACING_ACCELERATION_STRUCTURE + Flags: Direct3D12::D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, + }; + + let (resource, allocation) = suballocation::DeviceAllocationContext::from(self) + .create_acceleration_structure(desc, raw_desc)?; + + // for some reason there is no counter for acceleration structures + + Ok(super::AccelerationStructure { + resource, + allocation, + }) + } + + unsafe fn destroy_acceleration_structure( + &self, + acceleration_structure: super::AccelerationStructure, + ) { + suballocation::DeviceAllocationContext::from(self).free_resource( + acceleration_structure.resource, + acceleration_structure.allocation, + ); + } + + fn get_internal_counters(&self) -> wgt::HalCounters { + self.counters.as_ref().clone() + } + + fn generate_allocator_report(&self) -> Option { + Some(self.mem_allocator.generate_report()) + } + + fn tlas_instance_to_bytes(&self, instance: TlasInstance) -> Vec { + const MAX_U24: u32 = (1u32 << 24u32) - 1u32; + let temp = Direct3D12::D3D12_RAYTRACING_INSTANCE_DESC { + Transform: instance.transform, + _bitfield1: (instance.custom_data & MAX_U24) | (u32::from(instance.mask) << 24), + _bitfield2: 0, + AccelerationStructure: instance.blas_address, + }; + + wgt::bytemuck_wrapper!(unsafe struct Desc(Direct3D12::D3D12_RAYTRACING_INSTANCE_DESC)); + + bytemuck::bytes_of(&Desc::wrap(temp)).to_vec() + } + + fn check_if_oom(&self) -> Result<(), crate::DeviceError> { + let Some(threshold) = self.mem_allocator.memory_budget_thresholds.for_device_loss else { + return Ok(()); + }; + + let info = self + .shared + .adapter + .query_video_memory_info(Dxgi::DXGI_MEMORY_SEGMENT_GROUP_LOCAL)?; + + if info.CurrentUsage >= info.Budget / 100 * threshold as u64 { + return Err(crate::DeviceError::OutOfMemory); + } + + if matches!( + self.shared.private_caps.memory_architecture, + super::MemoryArchitecture::NonUnified + ) { + let info = self + .shared + .adapter + .query_video_memory_info(Dxgi::DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL)?; + + if info.CurrentUsage >= info.Budget / 100 * threshold as u64 { + return Err(crate::DeviceError::OutOfMemory); + } + } + + Ok(()) + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/device_creation.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/device_creation.rs new file mode 100644 index 00000000..dc5cdf9e --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/device_creation.rs @@ -0,0 +1,176 @@ +use alloc::sync::Arc; +use core::ops::Deref; + +use windows::core::Interface as _; +use windows::Win32::Graphics::{Direct3D, Direct3D12}; + +use super::D3D12Lib; +use crate::auxil::dxgi::factory::DxgiAdapter; + +/// Abstraction over D3D12 device creation. +/// +/// Supports two paths: +/// - **Independent**: Uses `ID3D12DeviceFactory` from the Agility SDK's Independent Devices API. +/// - **Legacy**: Uses the traditional `D3D12CreateDevice` export. +pub(super) enum DeviceFactory { + /// Uses `ID3D12DeviceFactory` from the Independent Devices API. + Independent(Direct3D12::ID3D12DeviceFactory), + /// Uses the traditional `D3D12CreateDevice` export. + Legacy, +} + +impl DeviceFactory { + /// Create a new `DeviceFactory`. + /// + /// If `agility_sdk` is `Some`, attempts to set up the Independent Devices API path. + /// On failure, the behavior depends on + /// [`on_load_failure`](wgt::Dx12AgilitySDKLoadFailure): + /// - [`Fallback`](wgt::Dx12AgilitySDKLoadFailure::Fallback): logs a warning and + /// returns `Ok(Legacy)`. + /// - [`Error`](wgt::Dx12AgilitySDKLoadFailure::Error): returns an `Err`. + pub(super) fn new( + lib: &D3D12Lib, + agility_sdk: Option<&wgt::Dx12AgilitySDK>, + ) -> Result { + let Some(agility_sdk) = agility_sdk else { + log::debug!("No D3D12 Agility SDK configuration provided; using system D3D12 runtime"); + return Ok(Self::Legacy); + }; + + match Self::try_create_independent(lib, agility_sdk) { + Ok(factory) => { + log::debug!( + "Using D3D12 Agility SDK v{} from '{}'", + agility_sdk.sdk_version, + agility_sdk.sdk_path + ); + Ok(Self::Independent(factory)) + } + Err(err) => { + let message = format!( + "Failed to initialize D3D12 Agility SDK (v{} at '{}'): {err}", + agility_sdk.sdk_version, agility_sdk.sdk_path + ); + + match agility_sdk.on_load_failure { + wgt::Dx12AgilitySDKLoadFailure::Fallback => { + log::warn!("{message}; falling back to system D3D12 runtime"); + Ok(Self::Legacy) + } + wgt::Dx12AgilitySDKLoadFailure::Error => { + Err(crate::InstanceError::new(message)) + } + } + } + } + } + + fn try_create_independent( + lib: &D3D12Lib, + agility_sdk: &wgt::Dx12AgilitySDK, + ) -> Result { + // Step 1: Get ID3D12SDKConfiguration1 via D3D12GetInterface + let sdk_config: Direct3D12::ID3D12SDKConfiguration1 = lib + .get_interface(&Direct3D12::CLSID_D3D12SDKConfiguration) + .map_err(DeviceFactoryError::GetInterface)?; + + // Step 2: Create device factory with the specified SDK version and path + let sdk_path = std::ffi::CString::new(agility_sdk.sdk_path.as_bytes()) + .map_err(|_| DeviceFactoryError::InvalidPath)?; + let factory: Direct3D12::ID3D12DeviceFactory = unsafe { + sdk_config.CreateDeviceFactory( + agility_sdk.sdk_version, + windows::core::PCSTR(sdk_path.as_ptr().cast::()), + ) + } + .map_err(DeviceFactoryError::CreateDeviceFactory)?; + + Ok(factory) + } + + /// Enable the D3D12 debug layer and optionally GPU-based validation. + /// + /// - **Legacy**: configures debug globally via `D3D12GetDebugInterface`. + /// - **Independent**: uses `GetConfigurationInterface` to get an + /// `ID3D12Debug` scoped to the factory. + pub(super) fn enable_debug_layer(&self, lib: &D3D12Lib, flags: wgt::InstanceFlags) { + if !flags + .intersects(wgt::InstanceFlags::VALIDATION | wgt::InstanceFlags::GPU_BASED_VALIDATION) + { + return; + } + + let debug_controller = match self { + Self::Independent(factory) => { + match unsafe { + factory.GetConfigurationInterface::( + &Direct3D12::CLSID_D3D12Debug, + ) + } { + Ok(debug) => debug, + Err(err) => { + log::warn!("Failed to get debug interface from device factory: {err}"); + return; + } + } + } + Self::Legacy => match lib.debug_interface() { + Ok(Some(debug)) => debug, + Ok(None) => return, + Err(err) => { + log::warn!("Failed to get debug interface: {err}"); + return; + } + }, + }; + + if flags.intersects(wgt::InstanceFlags::VALIDATION) { + unsafe { debug_controller.EnableDebugLayer() } + } + if flags.intersects(wgt::InstanceFlags::GPU_BASED_VALIDATION) { + if let Ok(debug1) = debug_controller.cast::() { + unsafe { debug1.SetEnableGPUBasedValidation(true) } + } else { + log::warn!("Failed to enable GPU-based validation"); + } + } + } + + /// Create a D3D12 device using the appropriate method. + pub(super) fn create_device( + &self, + lib: &Arc, + adapter: &DxgiAdapter, + feature_level: Direct3D::D3D_FEATURE_LEVEL, + ) -> Result { + match self { + Self::Independent(factory) => { + let mut result__: Option = None; + unsafe { factory.CreateDevice(adapter.deref(), feature_level, &mut result__) } + .map_err(|e| super::CreateDeviceError::D3D12CreateDevice(e.into()))?; + + result__.ok_or(super::CreateDeviceError::RetDeviceIsNull) + } + Self::Legacy => lib.create_device(adapter, feature_level), + } + } +} + +impl core::fmt::Debug for DeviceFactory { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Independent(_) => write!(f, "DeviceFactory::Independent"), + Self::Legacy => write!(f, "DeviceFactory::Legacy"), + } + } +} + +#[derive(Debug, thiserror::Error)] +enum DeviceFactoryError { + #[error("failed to get ID3D12SDKConfiguration1: {0}")] + GetInterface(super::GetInterfaceError), + #[error("SDK path contains null bytes")] + InvalidPath, + #[error("CreateDeviceFactory failed: {0}")] + CreateDeviceFactory(windows::core::Error), +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/instance.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/instance.rs new file mode 100644 index 00000000..6c3a33b5 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/instance.rs @@ -0,0 +1,188 @@ +use alloc::{string::String, sync::Arc, vec::Vec}; + +use parking_lot::RwLock; +use windows::Win32::{Foundation, Graphics::Dxgi}; + +use super::SurfaceTarget; +use crate::{ + auxil, + dx12::{ + device_creation::DeviceFactory, shader_compilation::CompilerContainer, D3D12Lib, DCompLib, + }, +}; + +impl crate::Instance for super::Instance { + type A = super::Api; + + unsafe fn init(desc: &crate::InstanceDescriptor<'_>) -> Result { + profiling::scope!("Init DX12 Backend"); + let lib_main = D3D12Lib::new().map_err(|e| { + crate::InstanceError::with_source(String::from("failed to load d3d12.dll"), e) + })?; + + // Create DeviceFactory first so we know which debug path to use + let device_factory = + DeviceFactory::new(&lib_main, desc.backend_options.dx12.agility_sdk.as_ref())?; + + device_factory.enable_debug_layer(&lib_main, desc.flags); + + let (lib_dxgi, factory) = auxil::dxgi::factory::create_factory(desc.flags)?; + + // Create IDXGIFactoryMedia + let factory_media = lib_dxgi.create_factory_media().ok(); + + let mut supports_allow_tearing = false; + if let Some(factory5) = factory.as_factory5() { + let mut allow_tearing = Foundation::FALSE; + let hr = unsafe { + factory5.CheckFeatureSupport( + Dxgi::DXGI_FEATURE_PRESENT_ALLOW_TEARING, + <*mut _>::cast(&mut allow_tearing), + size_of_val(&allow_tearing) as u32, + ) + }; + + match hr { + Err(err) => log::warn!("Unable to check for tearing support: {err}"), + Ok(()) => supports_allow_tearing = true, + } + } + + // Initialize the shader compiler + let compiler_container = match desc.backend_options.dx12.shader_compiler.clone() { + wgt::Dx12Compiler::DynamicDxc { dxc_path } => { + CompilerContainer::new_dynamic_dxc(dxc_path.into()).map_err(|e| { + crate::InstanceError::with_source(String::from("Failed to load dynamic DXC"), e) + })? + } + wgt::Dx12Compiler::StaticDxc => CompilerContainer::new_static_dxc().map_err(|e| { + crate::InstanceError::with_source(String::from("Failed to load static DXC"), e) + })?, + wgt::Dx12Compiler::Fxc => CompilerContainer::new_fxc().map_err(|e| { + crate::InstanceError::with_source(String::from("Failed to load FXC"), e) + })?, + wgt::Dx12Compiler::Auto => { + if cfg!(feature = "static-dxc") { + // Prefer static DXC if its compiled in + CompilerContainer::new_static_dxc().map_err(|e| { + crate::InstanceError::with_source( + String::from("Failed to load static DXC"), + e, + ) + })? + } else { + // Try to load dynamic DXC + let dynamic = CompilerContainer::new_dynamic_dxc("dxcompiler.dll".into()); + match dynamic { + Ok(v) => v, + Err(super::shader_compilation::GetContainerError::FailedToLoad(..)) => { + // If it can't be found load FXC + CompilerContainer::new_fxc().map_err(|e| { + crate::InstanceError::with_source( + String::from("Failed to load FXC"), + e, + ) + })? + } + Err(e) => { + // If another error occurs when loading static DXC return that error + return Err(crate::InstanceError::with_source( + String::from("Failed to load dynamic DXC"), + e, + )); + } + } + } + } + }; + + match compiler_container { + CompilerContainer::DynamicDxc(..) => { + log::debug!("Using dynamic DXC for shader compilation") + } + CompilerContainer::StaticDxc(..) => { + log::debug!("Using static DXC for shader compilation") + } + CompilerContainer::Fxc(..) => { + log::debug!("Using FXC for shader compilation") + } + } + + Ok(Self { + // The call to create_factory will only succeed if we get a factory4, so this is safe. + factory, + factory_media, + library: Arc::new(lib_main), + device_factory: Arc::new(device_factory), + dcomp_lib: Arc::new(DCompLib::new()), + presentation_system: desc.backend_options.dx12.presentation_system, + _lib_dxgi: lib_dxgi, + supports_allow_tearing, + flags: desc.flags, + memory_budget_thresholds: desc.memory_budget_thresholds, + compiler_container: Arc::new(compiler_container), + options: desc.backend_options.dx12.clone(), + telemetry: desc.telemetry, + }) + } + + unsafe fn create_surface( + &self, + display_handle: raw_window_handle::RawDisplayHandle, + window_handle: raw_window_handle::RawWindowHandle, + ) -> Result { + assert!(matches!( + display_handle, + raw_window_handle::RawDisplayHandle::Windows(_) + )); + match window_handle { + raw_window_handle::RawWindowHandle::Win32(handle) => { + // https://github.com/rust-windowing/raw-window-handle/issues/171 + let handle = Foundation::HWND(handle.hwnd.get() as *mut _); + let target = match self.presentation_system { + wgt::Dx12SwapchainKind::DxgiFromHwnd => SurfaceTarget::WndHandle(handle), + wgt::Dx12SwapchainKind::DxgiFromVisual => SurfaceTarget::VisualFromWndHandle { + handle, + dcomp_state: Default::default(), + }, + }; + + Ok(super::Surface { + factory: self.factory.clone(), + factory_media: self.factory_media.clone(), + target, + supports_allow_tearing: self.supports_allow_tearing, + swap_chain: RwLock::new(None), + options: self.options.clone(), + }) + } + _ => Err(crate::InstanceError::new(format!( + "window handle {window_handle:?} is not a Win32 handle" + ))), + } + } + + unsafe fn enumerate_adapters( + &self, + _surface_hint: Option<&super::Surface>, + ) -> Vec> { + let adapters = auxil::dxgi::factory::enumerate_adapters(self.factory.clone()); + + adapters + .into_iter() + .filter_map(|raw| { + super::Adapter::expose( + raw, + &self.library, + &self.device_factory, + &self.dcomp_lib, + self.flags, + self.memory_budget_thresholds, + self.compiler_container.clone(), + self.options.clone(), + self.telemetry, + ) + }) + .collect() + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/mod.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/mod.rs new file mode 100644 index 00000000..ac520b27 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/mod.rs @@ -0,0 +1,1698 @@ +/*! +# DirectX12 API internals. + +Generally the mapping is straightforward. + +## Resource transitions + +D3D12 API matches WebGPU internal states very well. The only +caveat here is issuing a special UAV barrier whenever both source +and destination states match, and they are for storage sync. + +## Memory + +For now, all resources are created with "committed" memory. + +## Sampler Descriptor Management + +At most one descriptor heap of each type can be bound at once. This +means that the descriptors from all bind groups need to be present +in the same heap, and they need to be contiguous within that heap. +This is not a problem for the SRV/CBV/UAV heap as it can be sized into +the millions of entries. However the sampler heap is limited to 2048 entries. + +In order to work around this limitation, we refer to samplers indirectly by index. +The entire sampler heap is bound at once and a buffer containing all sampler indexes +for that bind group is bound. The shader then uses the index to look up the sampler +in the heap. To help visualize this, the generated HLSL looks like this: + +```wgsl +@group(0) @binding(2) var myLinearSampler: sampler; +@group(1) @binding(1) var myAnisoSampler: sampler; +@group(1) @binding(4) var myCompSampler: sampler; +``` + +```cpp +// These bindings alias the same descriptors. Depending on the type, the shader will use the correct one. +SamplerState nagaSamplerHeap[2048]: register(s0, space0); +SamplerComparisonState nagaComparisonSamplerHeap[2048]: register(s2048, space1); + +StructuredBuffer nagaGroup0SamplerIndexArray : register(t0, space0); +StructuredBuffer nagaGroup1SamplerIndexArray : register(t1, space0); + +// Indexes into group 0 index array +static const SamplerState myLinearSampler = nagaSamplerHeap[nagaGroup0SamplerIndexArray[0]]; + +// Indexes into group 1 index array +static const SamplerState myAnisoSampler = nagaSamplerHeap[nagaGroup1SamplerIndexArray[0]]; +static const SamplerComparisonState myCompSampler = nagaComparisonSamplerHeap[nagaGroup1SamplerIndexArray[1]]; +``` + +Without this transform we would need separate set of sampler descriptors for each unique combination of samplers +in a bind group. This results in a lot of duplication and makes it easy to hit the 2048 limit. With the transform +the limit is merely 2048 unique samplers in existence, which is much more reasonable. + +## Resource binding + +See [`crate::Device::create_pipeline_layout`] documentation for the structure +of the root signature corresponding to WebGPU pipeline layout. + +Binding groups is mostly straightforward, with one big caveat: +all bindings have to be reset whenever the root signature changes. +This is the rule of D3D12, and we can do nothing to help it. + +We detect this change at both [`crate::CommandEncoder::set_bind_group`] +and [`crate::CommandEncoder::set_render_pipeline`] with +[`crate::CommandEncoder::set_compute_pipeline`]. + +For this reason, in order avoid repeating the binding code, +we are binding everything in `CommandEncoder::update_root_elements`. +When the pipeline layout is changed, we reset all bindings. +Otherwise, we pass a range corresponding only to the current bind group. + +!*/ + +mod adapter; +mod command; +mod conv; +mod dcomp; +mod descriptor; +mod device; +mod device_creation; +mod instance; +mod pipeline_desc; +mod sampler; +mod shader_compilation; +mod suballocation; +mod types; +mod view; + +use alloc::{borrow::ToOwned as _, string::String, sync::Arc, vec::Vec}; +use core::{ffi, fmt, mem, ops::Deref}; + +use arrayvec::ArrayVec; +use hashbrown::HashMap; +use parking_lot::{Mutex, RwLock}; +use suballocation::Allocator; +use windows::{ + core::{Free as _, Interface}, + Win32::{ + Foundation, + Graphics::{ + Direct3D, + Direct3D12::{self, D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT}, + DirectComposition, Dxgi, + }, + System::Threading, + }, +}; + +use self::dcomp::DCompLib; +use crate::auxil::{ + self, + dxgi::{ + factory::{DxgiAdapter, DxgiFactory}, + result::HResult, + }, +}; + +#[derive(Debug)] +struct DynLib { + inner: libloading::Library, +} + +impl DynLib { + unsafe fn new

Pure-Rust JPEG 2000 Codec

")); - assert!(landing.contains( - "" - )); - assert!(!home.contains("J2K: Rust JPEG 2000 / HTJ2K Codec")); - assert!(sitemap.contains("https://frames-sg.github.io/j2k/rust-jpeg2000-codec/\n 1.0")); -} +mod accelerator_evidence; +mod benchmark_publication; +mod environment; +mod metal_safety; +mod navigation_packaging; +mod repository_hygiene; diff --git a/xtask/tests/repo_lint_support/public_docs_policy/accelerator_evidence.rs b/xtask/tests/repo_lint_support/public_docs_policy/accelerator_evidence.rs new file mode 100644 index 00000000..a72f51eb --- /dev/null +++ b/xtask/tests/repo_lint_support/public_docs_policy/accelerator_evidence.rs @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::fs; + +use super::super::{assert_file_pattern_checks, repo_root, FilePatternCheck}; + +#[test] +fn public_docs_describe_public_crate_auto_and_cuda_runtime_surface_scope() { + assert_file_pattern_checks( + repo_root(), + &[ + FilePatternCheck::new("README.md") + .required(&[ + "public crate release", + "Runtime backend selection defaults to `Auto`", + "cuda-runtime", + "CUDA device memory", + "J2K-owned CUDA", + "NVIDIA performance", + ]) + .forbidden(&["compatibility-only with no runtime CUDA decode"]), + FilePatternCheck::new("docs/architecture.md").required(&[ + "public crate release", + "Runtime backend selection defaults to `Auto`", + ]), + FilePatternCheck::new("docs/release.md") + .required(&[ + "public crate release", + "Runtime backend selection defaults to `Auto`", + "cuda-runtime", + "CUDA device memory", + "J2K-owned CUDA", + "NVIDIA performance", + ]) + .forbidden(&["compatibility-only with no runtime CUDA decode"]), + FilePatternCheck::new("CHANGELOG.md") + .required(&[ + "cuda-runtime", + "CUDA device memory", + "J2K-owned CUDA", + "NVIDIA performance", + ]) + .forbidden(&["compatibility-only with no runtime CUDA decode"]), + ], + ); +} + +#[test] +fn accelerator_support_and_benchmark_evidence_have_single_document_owners() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read workspace README"); + let architecture = + fs::read_to_string(root.join("docs/architecture.md")).expect("read architecture docs"); + let public_support = fs::read_to_string(root.join("docs/public-support.md")) + .expect("read public support matrix"); + let ml = fs::read_to_string(root.join("docs/j2k-ml.md")).expect("read Burn integration docs"); + let evidence = fs::read_to_string(root.join("docs/benchmark-evidence.md")) + .expect("read benchmark evidence"); + let metal_telemetry = fs::read_to_string(root.join("crates/j2k-ml/benches/metal_telemetry.rs")) + .expect("read Metal telemetry source"); + let normalized_ml = ml.split_whitespace().collect::>().join(" "); + + for (name, document) in [("README", &readme), ("architecture", &architecture)] { + for canonical in ["public-support.md", "j2k-ml.md", "benchmark-evidence.md"] { + assert!(document.contains(canonical), "{name} must link {canonical}"); + } + for duplicated_evidence in [ + "RTX 4070 hardware validation", + "hardware-validated direct Metal batch matrix", + "A July 19, 2026 local M4 Pro diagnostic run", + "52 of 64 batch-32 cases", + ] { + assert!( + !document.contains(duplicated_evidence), + "{name} must not duplicate mutable accelerator evidence {duplicated_evidence:?}" + ); + } + } + + assert!(public_support.contains("## Owned batch codec boundary")); + assert!(public_support.contains("j2k-ml.md")); + assert!(normalized_ml.contains("not a Cartesian batch-greater-than-one validation")); + assert!(!ml.contains("A July 19, 2026 local M4 Pro diagnostic run")); + + for qualification in [ + "identical encoded content", + "decode-once broadcast", + "none of those rows is an acceptance baseline", + ] { + assert!( + evidence.contains(qualification), + "benchmark evidence must retain historical qualification {qualification:?}" + ); + } + + let snapshot = metal_telemetry + .find("let before = metal_snapshot(decoder);") + .expect("Metal snapshot before codec telemetry"); + let timer = metal_telemetry + .find("let start = Instant::now();") + .expect("Metal codec telemetry timer"); + assert!( + snapshot < timer, + "Metal telemetry snapshot must precede timing" + ); + for timing_boundary in [ + "runtime and pipeline initialization before the timed interval", + "cold for prepared-plan and execution-arena caches", + "first prepared decode includes its immutable-arena upload", + ] { + assert!( + normalized_ml.contains(timing_boundary), + "Burn integration docs must disclose {timing_boundary:?}" + ); + } +} + +#[test] +fn metal_batch_api_and_benchmark_docs_match_the_implemented_contract() { + let root = repo_root(); + let decoder = fs::read_to_string(root.join("crates/j2k-metal/src/batch_decoder/decoder.rs")) + .expect("read Metal batch decoder"); + let external = fs::read_to_string(root.join("crates/j2k-metal/src/batch_decoder/external.rs")) + .expect("read Metal external batch decoder"); + let ml = fs::read_to_string(root.join("docs/j2k-ml.md")).expect("read Burn integration docs"); + let architecture = + fs::read_to_string(root.join("docs/architecture.md")).expect("read architecture docs"); + + assert!(decoder.contains( + "Decode one homogeneous shared codec group using the preparation policy captured by the group." + )); + assert!(!decoder.contains("using strict session defaults")); + + assert!(external.contains( + "Decode one prepared homogeneous Gray, RGB, or RGBA group with native U8, U16, or I16 samples" + )); + assert!(!external.contains("Gray or unsigned RGB group")); + + for group_id in [ + "j2k_owned_batch_codec_cpu/input_{distinct|repeated}", + "j2k_owned_batch_burn_cpu/input_{distinct|repeated}", + ] { + assert!(ml.contains(group_id), "Burn docs must name {group_id}"); + } + assert!(ml.contains("`prepare_images` rows report images per second")); + assert!(ml.contains("Decode rows report decoded spatial pixels per second")); + assert!(!ml.contains("Criterion group names include `flex`")); + + assert!(architecture.contains("codec-side raw Objective-C resource-construction boundary")); + assert!(architecture.contains("separate audited raw-handle adoption boundary")); + assert!(!architecture.contains("sole raw Objective-C resource-construction boundary")); +} + +#[test] +fn metal_failure_attribution_is_documented_without_guessing_a_source() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read workspace README"); + let public_support = fs::read_to_string(root.join("docs/public-support.md")) + .expect("read public support matrix"); + let ml = fs::read_to_string(root.join("docs/j2k-ml.md")).expect("read Burn integration docs"); + let contracts = + fs::read_to_string(root.join("crates/j2k-metal/src/batch_decoder/contracts.rs")) + .expect("read Metal batch contracts"); + let submission = + fs::read_to_string(root.join("crates/j2k-metal/src/batch_decoder/submission.rs")) + .expect("read Metal batch submission contracts"); + + assert!(readme.contains("indexed preparation failures")); + assert!(readme.contains("homogeneous group execution failures")); + assert!(!readme.contains("returns source indices and indexed failures")); + for required in [ + "Device classic and HT codec jobs retain their original source identity", + "Metal command-buffer completion failure", + "remains a group-level failure", + "preserves every affected source index", + ] { + assert!(ml.contains(required), "j2k-ml docs must state {required:?}"); + } + assert!(public_support.contains("Preparation failures remain indexed per input")); + assert!(public_support.contains("command-buffer failures remain group-level")); + assert!(contracts.contains( + "Successful groups, indexed preparation failures, and homogeneous execution failures." + )); + assert!( + submission.contains("Command-buffer failures remain group-level at the batch boundary.") + ); +} diff --git a/xtask/tests/repo_lint_support/public_docs_policy/benchmark_publication.rs b/xtask/tests/repo_lint_support/public_docs_policy/benchmark_publication.rs new file mode 100644 index 00000000..386f402c --- /dev/null +++ b/xtask/tests/repo_lint_support/public_docs_policy/benchmark_publication.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::fs; + +use super::super::{assert_pattern_checks, repo_root, xtask_sources, PatternCheck}; + +#[test] +fn benchmark_docs_define_publication_gate_for_openjpeg_and_grok() { + let root = repo_root(); + let benchmark_corpora = fs::read_to_string(root.join("docs/benchmark-corpora.md")) + .expect("read benchmark corpus docs"); + let benchmark_evidence = fs::read_to_string(root.join("docs/benchmark-evidence.md")) + .expect("read benchmark evidence docs"); + let env_vars = fs::read_to_string(root.join("docs/env-vars.md")).expect("read env var docs"); + let benchmark_docs = format!("{benchmark_corpora}\n{benchmark_evidence}\n{env_vars}"); + let xtask = xtask_sources(root); + let ci = fs::read_to_string(root.join(".github/workflows/ci.yml")).expect("read CI workflow"); + + assert_pattern_checks(&[ + PatternCheck::new("benchmark publication docs", &benchmark_docs).required(&[ + "published benchmark", + "J2K_COMPARE_THREADS", + "J2K_REQUIRE_OPENJPEG=1", + "J2K_REQUIRE_GROK=1", + "comparator availability", + "comparator version", + "input source", + "j2k-generated", + ]), + PatternCheck::new("xtask benchmark signoff", &xtask).required(&[ + "\"j2k-bench-signoff\"", + "grok_parity", + "libjpeg_turbo_compare", + "bench-libjpeg-turbo", + "J2K_REQUIRE_LIBJPEG_TURBO", + "passed_test_count", + "expected at least", + ]), + PatternCheck::new("CI comparator parity job", &ci).required(&[ + "comparator-parity:", + "grokj2k-tools", + "libgrokj2k1-dev", + "libopenjp2-tools", + "libturbojpeg0-dev", + "pkg-config --modversion libgrokj2k", + "pkg-config --modversion libturbojpeg", + "J2K_REQUIRE_OPENJPEG: \"1\"", + "J2K_REQUIRE_GROK: \"1\"", + "J2K_REQUIRE_LIBJPEG_TURBO: \"1\"", + "cargo xtask j2k-bench-signoff", + ]), + ]); +} + +#[test] +fn adoption_starter_corpus_fallback_is_pinned() { + let root = repo_root(); + let workflow = fs::read_to_string(root.join(".github/workflows/gpu-validation.yml")) + .expect("read GPU validation workflow"); + let benchmark_docs = fs::read_to_string(root.join("docs/benchmark-corpora.md")) + .expect("read benchmark corpus docs"); + let openjpeg_commit = "39524bd3a601d90ed8e0177559400d23945f96a9"; + + let workflow_required = [ + "sha256sum -c - <<'SHA256'".to_string(), + "a56e27cbf5f843c048b6af1d6e090760e9c92fadba88b7dee0205918a37523bd kodim01.png".to_string(), + "1071c68372cc5a01435c2c225a5cf7d4bb803846ec08bb6b3d6721b156d7cb96 kodim24.png".to_string(), + "downloaded-from-r0k-us-kodak-lossless-true-color-sha256-pinned".to_string(), + format!("J2K_STARTER_OPENJPEG_DATA_COMMIT={openjpeg_commit}"), + "git -C target/j2k-public-corpora/openjpeg-data fetch --depth 1 origin".to_string(), + "git -C target/j2k-public-corpora/openjpeg-data checkout --detach".to_string(), + format!("source-native-openjpeg-data-conformance-dir@{openjpeg_commit}"), + format!("source-native-openjpeg-data-nonregression-dir@{openjpeg_commit}"), + ]; + let workflow_required = workflow_required + .iter() + .map(String::as_str) + .collect::>(); + let benchmark_docs_required = [ + "sha256sum -c - <<'SHA256'", + openjpeg_commit, + "downloaded-from-r0k-us-kodak-lossless-true-color-sha256-pinned", + "OpenJPEG-data commit", + "SHA-256-checked Kodak PNGs", + "fixed OpenJPEG-data commit", + ]; + + assert_pattern_checks(&[ + PatternCheck::new( + "GPU adoption fallback pinned starter corpus inputs", + &workflow, + ) + .required(&workflow_required) + .forbidden(&["git clone --depth 1 https://github.com/uclouvain/openjpeg-data"]), + PatternCheck::new( + "benchmark corpus docs pinned starter corpus inputs", + &benchmark_docs, + ) + .required(&benchmark_docs_required), + ]); +} + +#[test] +fn benchmark_publication_gate_rules_are_single_sourced() { + let root = repo_root(); + let gate = fs::read_to_string(root.join("xtask/src/publication_gate.rs")) + .expect("read publication gate module"); + let benchmark = fs::read_to_string(root.join("xtask/src/adoption_benchmark/support.rs")) + .expect("read adoption benchmark publication support module"); + let report = fs::read_to_string(root.join("xtask/src/adoption_report.rs")) + .expect("read adoption report module"); + + assert_pattern_checks(&[ + PatternCheck::new("publication gate module", &gate).required(&[ + "PUBLICATION_GATE_KEYS", + "PublicationGateEvaluation", + "pub(crate) fn collect_publication_gate_issues", + "publication_eligible", + "publication_blockers", + "benchmark_complete", + ]), + PatternCheck::new("adoption benchmark writer publication gate use", &benchmark) + .required(&[ + "PUBLICATION_GATE_KEYS", + "collect_publication_gate_issues", + "Some(&fixture_metadata)", + "Some(&encode_metadata)", + ]) + .forbidden(&["fn collect_publication_gate_issues("]), + PatternCheck::new("adoption report checker publication gate use", &report) + .required(&[ + "collect_publication_gate_issues", + "summary.get(\"cpu_fixture_compare\")", + "summary.get(\"cpu_encode_compare\")", + ]) + .forbidden(&["fn collect_gate_issue("]), + ]); +} diff --git a/xtask/tests/repo_lint_support/public_docs_policy/environment.rs b/xtask/tests/repo_lint_support/public_docs_policy/environment.rs new file mode 100644 index 00000000..4345622f --- /dev/null +++ b/xtask/tests/repo_lint_support/public_docs_policy/environment.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{collections::BTreeSet, fs}; + +use super::super::{ + assert_file_pattern_checks, documented_j2k_env_vars, is_archived_handoff, + is_internal_j2k_token, is_repo_lint_test_source, j2k_env_tokens, repo_root, repo_text_files, + FilePatternCheck, +}; + +#[test] +fn supported_j2k_env_vars_are_documented() { + let root = repo_root(); + let docs_path = root.join("docs/env-vars.md"); + let docs = fs::read_to_string(&docs_path) + .unwrap_or_else(|err| panic!("read {}: {err}", docs_path.display())); + assert_file_pattern_checks( + root, + &[ + FilePatternCheck::new("README.md") + .named("README environment-variable reference") + .required(&["docs/env-vars.md"]), + FilePatternCheck::new("docs/env-vars.md") + .named("supported environment-variable reference") + .required(&["| `J2K_SEMVER_TOOLCHAIN` | Rejected by `cargo xtask semver`; Rust `1.96` is pinned in source and CI. | Must not be set | Test/CI |"]) + .forbidden(&["J2K_JPEG_METAL_SPLIT_FAST420_BATCH"]), + ], + ); + let documented = documented_j2k_env_vars(&docs); + assert!( + !documented.is_empty(), + "docs/env-vars.md must document supported J2K_* environment variables" + ); + + let mut missing = Vec::new(); + let mut referenced = BTreeSet::new(); + for path in repo_text_files(root) { + if is_archived_handoff(&path) + || path.ends_with("docs/env-vars.md") + || path.ends_with("engineering/ai-codebase-audit-remediation-plan.md") + || is_repo_lint_test_source(root, &path) + { + continue; + } + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + for token in j2k_env_tokens(&source) { + referenced.insert(token.clone()); + if is_internal_j2k_token(&token) { + continue; + } + if !documented.contains(&token) { + missing.push(format!( + "{}: {token}", + path.strip_prefix(root).unwrap_or(&path).display() + )); + } + } + } + + assert!( + missing.is_empty(), + "supported J2K_* environment variables must be documented in docs/env-vars.md:\n{}", + missing.join("\n") + ); + let stale = documented + .difference(&referenced) + .cloned() + .collect::>(); + assert!( + stale.is_empty(), + "docs/env-vars.md documents J2K_* variables with no repo reference:\n{}", + stale.join("\n") + ); + for phantom in [ + "J2K_LEVEL1_CUDA_HT_MIN_MPS", + "J2K_LEVEL1_CUDA_HT_MIN_SPEEDUP_VS_NVIDIA", + "J2K_LEVEL2_CUDA_HT_MIN_MPS", + "J2K_LEVEL2_CUDA_HT_MIN_SPEEDUP_VS_NVIDIA", + ] { + assert!( + !documented.contains(phantom), + "phantom GPU validation env var `{phantom}` must not be documented without an implementation" + ); + } +} diff --git a/xtask/tests/repo_lint_support/public_docs_policy/metal_safety.rs b/xtask/tests/repo_lint_support/public_docs_policy/metal_safety.rs new file mode 100644 index 00000000..ebad806d --- /dev/null +++ b/xtask/tests/repo_lint_support/public_docs_policy/metal_safety.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{collections::BTreeSet, fs}; + +use super::super::{ + assert_file_pattern_checks, assert_pattern_checks, read_source_files, repo_root, rust_sources, + FilePatternCheck, PatternCheck, +}; + +#[test] +#[expect( + clippy::too_many_lines, + reason = "the cross-module Metal consistency checks are one public-contract policy" +)] +fn metal_consistency_cleanup_keeps_names_status_buffers_and_marker_sizes_single_sourced() { + let root = repo_root(); + let buffer_validation = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/buffer_validation.rs")) + .expect("read buffer validation"); + let decode_dispatch = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/decode_dispatch.rs")) + .expect("read decode dispatch"); + let lossless_prepare = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/lossless_prepare.rs")) + .expect("read lossless prepare"); + let tier1_encode = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/tier1_encode.rs")) + .expect("read tier1 encode"); + let resident_codestream = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/resident_codestream.rs")) + .expect("read resident codestream"); + let resident_tier1 = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/resident_tier1.rs")) + .expect("read resident tier1"); + let resident_tier1_types = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/resident_tier1/types.rs")) + .expect("read resident tier1 types"); + let direct_buffers = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_buffers.rs")) + .expect("read direct buffers"); + let direct_roi = fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_roi.rs")) + .expect("read direct ROI"); + let resident_types = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/resident_types.rs")) + .expect("read resident types"); + let resident_packet_plan = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/resident_packet_plan.rs")) + .expect("read resident packet plan"); + let encode_capacity = + fs::read_to_string(root.join("crates/j2k-metal/src/compute/encode_capacity.rs")) + .expect("read encode capacity"); + let jpeg_extended12 = read_source_files( + root, + &[ + "crates/j2k-jpeg/src/decoder/extended12.rs", + "crates/j2k-jpeg/src/decoder/extended12/upsample.rs", + ], + ); + let split_metal_status_users = [ + "crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup.rs", + "crates/j2k-metal/src/compute/decode_dispatch/classic_subband.rs", + "crates/j2k-metal/src/compute/decode_dispatch/ht_distinct.rs", + "crates/j2k-metal/src/compute/decode_dispatch/ht_subband.rs", + "crates/j2k-metal/src/compute/decode_dispatch/idwt.rs", + "crates/j2k-metal/src/compute/decode_dispatch/mct.rs", + "crates/j2k-metal/src/compute/decode_dispatch/store.rs", + "crates/j2k-metal/src/compute/lossless_prepare/batch.rs", + "crates/j2k-metal/src/compute/lossless_prepare/batch_item.rs", + "crates/j2k-metal/src/compute/lossless_prepare/commands.rs", + "crates/j2k-metal/src/compute/lossless_prepare/forward_encode.rs", + "crates/j2k-metal/src/compute/lossless_prepare/single.rs", + "crates/j2k-metal/src/compute/lossless_prepare/sizes.rs", + "crates/j2k-metal/src/compute/resident_tier1/profile_dispatch/analysis.rs", + "crates/j2k-metal/src/compute/resident_tier1/profile_dispatch/tokens.rs", + "crates/j2k-metal/src/compute/resident_tier1/readback.rs", + "crates/j2k-metal/src/compute/resident_tier1/result_harvest.rs", + ] + .into_iter() + .map(|relative| { + fs::read_to_string(root.join(relative)) + .unwrap_or_else(|error| panic!("read {relative}: {error}")) + }) + .collect::>() + .join("\n"); + let metal_status_users = [ + buffer_validation.as_str(), + decode_dispatch.as_str(), + lossless_prepare.as_str(), + tier1_encode.as_str(), + resident_codestream.as_str(), + resident_tier1.as_str(), + split_metal_status_users.as_str(), + ] + .join("\n"); + + assert_pattern_checks(&[ + PatternCheck::new( + "Metal resident tier1 component-count field", + &resident_tier1_types, + ) + .required(&["pub(crate) component_count: u8"]) + .forbidden(&["pub(crate) components: u8", "pub(crate) num_components: u8"]), + PatternCheck::new( + "Metal resident types component-count field", + &resident_types, + ) + .required(&["pub(crate) component_count: u8"]) + .forbidden(&["pub(crate) num_components: u8"]), + PatternCheck::new( + "Metal resident packet plan component-count field", + &resident_packet_plan, + ) + .required(&["pub(super) component_count: u8"]) + .forbidden(&["pub(super) num_components: u8"]), + PatternCheck::new("Metal direct buffer helper", &direct_buffers) + .required(&["pub(super) fn zeroed_shared_buffer", "early-returning"]), + PatternCheck::new("Metal status readback buffer users", &metal_status_users) + .required(&["zeroed_shared_buffer(&runtime.device"]) + .forbidden(&[ + "let status_buffer = runtime.device.new_buffer(", + "let status_buffer = runtime.device.new_buffer_with_data(", + "let status_buffer = runtime\n .device\n .new_buffer", + ]), + PatternCheck::new( + "codestream capacity marker-size constants", + &encode_capacity, + ) + .required(&[ + "JP2K_SIZ_FIXED_BYTES", + "JP2K_SIZ_BYTES_PER_COMPONENT", + "JP2K_CAP_MARKER_SEGMENT_BYTES", + "JP2K_COD_MARKER_SEGMENT_BYTES", + "JP2K_QCD_FIXED_BYTES", + "JP2K_TLM_MARKER_SEGMENT_BYTES", + "JP2K_SOT_MARKER_SEGMENT_BYTES", + "JP2K_SOD_MARKER_BYTES", + "JP2K_EOC_MARKER_BYTES", + ]) + .forbidden(&[ + "40usize", + "len.checked_add(14)", + "if job.write_tlm { 12", + "len.checked_add(12)", + "len.checked_add(2)", + ]), + PatternCheck::new("IDWT margin explanations", &direct_roi) + .required(&["16 samples", "40 for irreversible 9/7"]), + PatternCheck::new( + "extended 12-bit fancy upsample rounding explanation", + &jpeg_extended12, + ) + .required(&["IJG/libjpeg fancy h2v2 upsampling"]), + ]); +} + +#[test] +fn metal_raw_buffer_contents_access_stays_confined_to_checked_helpers() { + let root = repo_root(); + let allowed = BTreeSet::from([ + "crates/j2k-metal-support/src/buffer_access.rs", + "crates/j2k-metal/src/compute/direct_buffers.rs", + "crates/j2k-jpeg-metal/src/buffers.rs", + ]); + + for src_root in [ + "crates/j2k-metal-support/src", + "crates/j2k-metal/src", + "crates/j2k-jpeg-metal/src", + "crates/j2k-transcode-metal/src", + ] { + for path in rust_sources(&root.join(src_root)) { + let rel = path + .strip_prefix(root) + .expect("source path under repo root") + .to_string_lossy() + .replace('\\', "/"); + let source = + fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {rel}: {err}")); + if allowed.contains(rel.as_str()) { + continue; + } + assert!( + !source.contains(".contents()"), + "raw Metal buffer contents access must stay inside checked helpers; found in {rel}" + ); + } + } +} + +#[test] +fn j2k_metal_bench_surface_stays_clean_after_reset() { + let root = repo_root(); + let removed_j2k_metal_bench_command = ["cargo bench -p ", "j2k-metal", " --bench"].concat(); + assert_file_pattern_checks( + root, + &[ + FilePatternCheck::new("crates/j2k-metal/Cargo.toml") + .named("J2K Metal manifest") + .forbidden(&[ + "[[bench]]", + "criterion =", + "j2k-compare =", + "name = \"device_upload\"", + "name = \"compare\"", + "name = \"encode_stages\"", + "name = \"decode_stages\"", + ]), + FilePatternCheck::new("README.md") + .forbidden(&[removed_j2k_metal_bench_command.as_str()]), + FilePatternCheck::new("xtask/src/main.rs") + .forbidden(&[removed_j2k_metal_bench_command.as_str()]), + FilePatternCheck::new("crates/j2k-compare/src/openjpeg.rs") + .named("OpenJPEG comparator") + .required(&["pub fn version"]), + FilePatternCheck::new("crates/j2k-compare/src/grok.rs") + .named("Grok comparator") + .required(&["pub fn version", "pub fn library_path"]), + ], + ); + + let benches_dir = root.join("crates/j2k-metal/benches"); + if benches_dir.exists() { + let stale_entries: Vec<_> = fs::read_dir(&benches_dir) + .expect("read J2K Metal benches dir") + .map(|entry| { + let path = entry.expect("read J2K Metal bench entry").path(); + path.strip_prefix(root) + .expect("bench entry under repo root") + .display() + .to_string() + }) + .collect(); + assert!( + stale_entries.is_empty(), + "j2k-metal benches dir must stay empty after reset: {stale_entries:?}" + ); + } +} diff --git a/xtask/tests/repo_lint_support/public_docs_policy/navigation_packaging.rs b/xtask/tests/repo_lint_support/public_docs_policy/navigation_packaging.rs new file mode 100644 index 00000000..d0de386e --- /dev/null +++ b/xtask/tests/repo_lint_support/public_docs_policy/navigation_packaging.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{ffi::OsStr, fs}; + +use super::super::{ + assert_file_pattern_checks, assert_pattern_checks, normalize_path, publishable_crate_dirs, + repo_root, rust_include_paths, rust_sources, FilePatternCheck, PatternCheck, +}; + +#[test] +fn public_docs_route_users_to_current_crates() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + assert_file_pattern_checks( + root, + &[FilePatternCheck::new("README.md") + .named("README current crate routing") + .required(&[ + "Which crate should I use?", + "Fast Path For LLM-Assisted Use", + "cargo add j2k", + "statumen", + "wsi-dicom", + "j2k-jpeg", + "j2k", + "j2k-cli", + ])], + ); + + let legacy_terms = [ + format!("{}{}", "ash", "lar"), + format!("{}{}", "zig", "gurat"), + ]; + for legacy in &legacy_terms { + assert!( + !readme.to_ascii_lowercase().contains(legacy), + "README.md must use current package names only" + ); + } +} + +#[test] +fn published_crates_have_crates_io_landing_readmes() { + let root = repo_root(); + + for crate_dir in publishable_crate_dirs(root) { + let manifest_path = crate_dir.join("Cargo.toml"); + let manifest = fs::read_to_string(&manifest_path) + .unwrap_or_else(|err| panic!("read {}: {err}", manifest_path.display())); + let readme_path = crate_dir.join("README.md"); + let package = crate_dir + .file_name() + .and_then(OsStr::to_str) + .expect("publishable crate dir has UTF-8 name"); + + assert!( + manifest.contains("readme"), + "{} must declare a readme for crates.io landing pages", + manifest_path + .strip_prefix(root) + .unwrap_or(&manifest_path) + .display() + ); + assert!( + readme_path.exists(), + "{} must exist for crates.io landing pages", + readme_path + .strip_prefix(root) + .unwrap_or(&readme_path) + .display() + ); + + let readme = fs::read_to_string(&readme_path) + .unwrap_or_else(|err| panic!("read {}: {err}", readme_path.display())); + let readme_source_name = readme_path + .strip_prefix(root) + .unwrap_or(&readme_path) + .display() + .to_string(); + let docs_url = format!("https://docs.rs/{package}"); + assert_pattern_checks( + &[PatternCheck::new(&readme_source_name, &readme).required(&[ + docs_url.as_str(), + "https://github.com/frames-sg/j2k", + "docs/public-support.md", + ])], + ); + } +} + +#[test] +fn publishable_crates_configure_docs_rs_metadata() { + let root = repo_root(); + + for crate_dir in publishable_crate_dirs(root) { + let manifest_path = crate_dir.join("Cargo.toml"); + let manifest = fs::read_to_string(&manifest_path) + .unwrap_or_else(|err| panic!("read {}: {err}", manifest_path.display())); + + let manifest_source_name = manifest_path + .strip_prefix(root) + .unwrap_or(&manifest_path) + .display() + .to_string(); + assert_pattern_checks(&[ + PatternCheck::new(&manifest_source_name, &manifest).required(&[ + "[package.metadata.docs.rs]", + "all-features = true", + "targets = []", + ]), + ]); + } +} + +#[test] +fn support_matrix_is_linked_and_covers_adoption_surfaces() { + assert_file_pattern_checks( + repo_root(), + &[FilePatternCheck::new("README.md") + .named("README support matrix") + .required(&[ + "Stable APIs", + "Experimental APIs", + "BackendRequest::Auto", + "Security", + "Benchmark and parity policy", + "MSRV", + "OpenJPEG", + "Grok", + ])], + ); +} + +#[test] +fn public_codec_and_transcode_examples_are_publicly_linked() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + + let examples = [ + "crates/j2k/examples/decode_generated.rs", + "crates/j2k-jpeg/examples/inspect.rs", + "crates/j2k-metal/examples/decode_route_report.rs", + "crates/j2k-metal/examples/htj2k_encode_auto_report.rs", + "crates/j2k-metal/examples/resident_encode_buffer.rs", + "crates/j2k-tilecodec/examples/decompress.rs", + "crates/j2k-transcode/examples/jpeg_to_htj2k.rs", + "crates/j2k-transcode-metal/examples/jpeg_to_htj2k_route_report.rs", + ]; + for example in examples { + assert!( + root.join(example).exists(), + "expected runnable example `{example}`" + ); + } + assert_pattern_checks(&[ + PatternCheck::new("README public example links", &readme).required(&examples) + ]); +} + +#[test] +fn packaged_rust_sources_do_not_include_files_outside_their_crate() { + let root = repo_root(); + let workspace_crates = root.join("crates"); + let mut escaping = Vec::new(); + + for source_path in rust_sources(&workspace_crates) { + let Ok(relative_to_crates) = source_path.strip_prefix(&workspace_crates) else { + continue; + }; + let Some(crate_name) = relative_to_crates.components().next() else { + continue; + }; + let member_root = workspace_crates.join(crate_name.as_os_str()); + let source = fs::read_to_string(&source_path) + .unwrap_or_else(|err| panic!("read {}: {err}", source_path.display())); + + for include_path in rust_include_paths(&source) { + let resolved = normalize_path( + &source_path + .parent() + .expect("source file has parent") + .join(&include_path), + ); + if !resolved.starts_with(&member_root) { + escaping.push(format!( + "{} includes {} outside package root", + source_path + .strip_prefix(root) + .unwrap_or(&source_path) + .display(), + include_path + )); + } + } + } + + assert!( + escaping.is_empty(), + "package source include paths must stay inside their crate so packaged tests/benches/examples are not dead: {escaping:?}" +); +} + +#[test] +fn public_search_metadata_routes_generic_queries_to_one_landing_page() { + let root = repo_root(); + let read = |relative: &str| { + let path = root.join(relative); + fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {}: {err}", path.display())) + }; + + let root_readme = read("README.md"); + let workspace_manifest = read("Cargo.toml"); + let crate_manifest = read("crates/j2k/Cargo.toml"); + let crate_readme = read("crates/j2k/README.md"); + let crate_lib = read("crates/j2k/src/lib.rs"); + let home = read("docs/index.html"); + let landing = read("docs/rust-jpeg2000-codec/index.html"); + let sitemap = read("docs/sitemap.xml"); + + assert!(root_readme.starts_with("# J2K — Pure-Rust JPEG 2000 and HTJ2K Codec\n")); + assert!(root_readme.contains("[Pure-Rust JPEG 2000 codec documentation](https://frames-sg.github.io/j2k/rust-jpeg2000-codec/)")); + assert!(workspace_manifest + .contains("homepage = \"https://frames-sg.github.io/j2k/rust-jpeg2000-codec/\"")); + assert!(crate_manifest.contains("description = \"Pure-Rust JPEG 2000")); + assert!(crate_manifest.contains("homepage.workspace = true")); + assert!(crate_readme.contains("[Pure-Rust JPEG 2000 codec documentation](https://frames-sg.github.io/j2k/rust-jpeg2000-codec/)")); + assert!(crate_lib.contains("//! Pure-Rust JPEG 2000")); + assert!(landing.contains("Pure-Rust JPEG 2000 Codec — J2K")); + assert!(landing.contains("

Pure-Rust JPEG 2000 Codec

")); + assert!(landing.contains( + "" + )); + assert!(!home.contains("J2K: Rust JPEG 2000 / HTJ2K Codec")); + assert!(sitemap.contains("https://frames-sg.github.io/j2k/rust-jpeg2000-codec/\n 1.0")); +} diff --git a/xtask/tests/repo_lint_support/public_docs_policy/repository_hygiene.rs b/xtask/tests/repo_lint_support/public_docs_policy/repository_hygiene.rs new file mode 100644 index 00000000..3cf494a1 --- /dev/null +++ b/xtask/tests/repo_lint_support/public_docs_policy/repository_hygiene.rs @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{ffi::OsStr, fs}; + +use super::super::{ + is_allowed_legacy_name_history_reference, is_archived_handoff, is_repo_lint_test_source, + referenced_shell_scripts, repo_root, repo_text_files, +}; + +#[test] +fn active_repo_text_does_not_reintroduce_signinum_names() { + let root = repo_root(); + let mut offenders = Vec::new(); + + for path in repo_text_files(root) { + if is_archived_handoff(&path) + || is_allowed_legacy_name_history_reference(root, &path) + || is_repo_lint_test_source(root, &path) + { + continue; + } + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + for (line_idx, line) in source.lines().enumerate() { + let lower = line.to_ascii_lowercase(); + if lower.contains("signinum") { + offenders.push(format!( + "{}:{}:{}", + path.strip_prefix(root).unwrap_or(&path).display(), + line_idx + 1, + line + )); + } + } + } + + assert!( + offenders.is_empty(), + "active repo text must not reintroduce signinum names after the j2k rename:\n{}", + offenders.join("\n") + ); +} + +#[test] +fn public_repo_excludes_agent_private_artifacts() { + let root = repo_root(); + let private_docs_name = ["super", "powers"].concat(); + let private_dir = ["docs", private_docs_name.as_str()].join("/"); + let migration_doc = ["MIGRATION", ".md"].concat(); + let migration_doc_lower = migration_doc.to_ascii_lowercase(); + let mut offenders = Vec::new(); + + for path in repo_text_files(root) { + let relative = path.strip_prefix(root).unwrap_or(&path); + let relative_text = relative.to_string_lossy(); + let file_name = path + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + .to_ascii_lowercase(); + if relative_text.starts_with(&private_dir) || file_name == migration_doc_lower { + offenders.push(relative_text.to_string()); + } + } + + assert!( + offenders.is_empty(), + "public repo must not track agent-private planning docs or migration scratch files: {offenders:?}" +); +} + +#[test] +fn public_text_does_not_embed_local_user_home_paths() { + let root = repo_root(); + let mut offenders = Vec::new(); + + for path in repo_text_files(root) { + if is_archived_handoff(&path) { + continue; + } + if is_repo_lint_test_source(root, &path) { + continue; + } + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + if source.contains("/Users/") || source.contains("C:\\Users\\") { + offenders.push( + path.strip_prefix(root) + .unwrap_or(&path) + .display() + .to_string(), + ); + } + } + + assert!( + offenders.is_empty(), + "public text must not embed local user-home paths; use env vars or repo-relative defaults: {offenders:?}" +); +} + +#[test] +fn referenced_shell_scripts_exist() { + let root = repo_root(); + let mut missing = Vec::new(); + + for path in repo_text_files(root) { + if is_archived_handoff(&path) { + continue; + } + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + for script in referenced_shell_scripts(&source) { + let root_relative = root.join(&script); + let file_relative = path.parent().expect("text file has parent").join(&script); + if !root_relative.exists() && !file_relative.exists() { + missing.push(format!( + "{} references missing script {script}", + path.strip_prefix(root).unwrap_or(&path).display() + )); + } + } + } + + assert!( + missing.is_empty(), + "all referenced shell scripts must exist: {missing:?}" + ); +} + +#[test] +fn public_narrative_docs_do_not_carry_stale_zeiss_claims() { + let root = repo_root(); + let mut offenders = Vec::new(); + + for relative in [ + "README.md", + "docs/architecture.md", + "docs/release.md", + "paper/paper.md", + "paper/arxiv/main.tex", + ] { + let path = root.join(relative); + let Ok(source) = fs::read_to_string(&path) else { + if relative.starts_with("paper/") { + continue; + } + panic!("read {}: missing required narrative doc", path.display()); + }; + if source.contains("Zeiss") { + offenders.push(relative); + } + } + + assert!( + offenders.is_empty(), + "public narrative docs must not carry stale Zeiss integration claims: {offenders:?}" + ); +} diff --git a/xtask/tests/repo_lint_support/source_policy.rs b/xtask/tests/repo_lint_support/source_policy.rs deleted file mode 100644 index f035fccc..00000000 --- a/xtask/tests/repo_lint_support/source_policy.rs +++ /dev/null @@ -1,458 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::{ffi::OsStr, fs}; - -use super::{ - assert_file_pattern_checks, assert_rust_source_scan_checks, repo_root, rust_sources, - FilePatternCheck, RustSourceScanCheck, -}; - -#[test] -fn batch_work_keeps_referenced_plans_and_boundary_tests_in_focused_modules() { - let root = repo_root(); - let direct_plan = fs::read_to_string(root.join("crates/j2k-native/src/direct_plan.rs")) - .expect("read j2k-native direct plan"); - assert_file_pattern_checks( - root, - &[ - FilePatternCheck::new("crates/j2k-native/src/direct_plan.rs") - .required(&["mod referenced_ht;", "pub use referenced_ht::"]) - .forbidden(&["pub enum J2kReferencedHtj2kPlan"]), - FilePatternCheck::new("crates/j2k-native/src/direct_plan/referenced_ht.rs") - .required(&["pub enum J2kReferencedHtj2kPlan"]) - .forbidden(&["use super::*;"]), - ], - ); - assert!( - direct_plan.lines().count() < 250, - "direct_plan.rs must remain a focused public geometry owner" - ); - - for (owner, module_decl, child, symbol, max_lines) in [ - ( - "crates/j2k-native/src/tests.rs", - "mod workspace_reuse;", - "crates/j2k-native/src/tests/workspace_reuse.rs", - "fn decoder_workspace_reuses_component_owners_across_distinct_input_lifetimes", - 300usize, - ), - ( - "crates/j2k-cuda-runtime/src/tests.rs", - "mod context_diagnostics;", - "crates/j2k-cuda-runtime/src/tests/context_diagnostics.rs", - "fn runtime_diagnostics_count_device_to_host_transfers_when_required", - 100usize, - ), - ( - "crates/j2k-cuda-runtime/src/tests/pipeline.rs", - "mod native_store;", - "crates/j2k-cuda-runtime/src/tests/pipeline/native_store.rs", - "fn j2k_native_grayscale_batch_store_preserves_unsigned_and_signed_samples_when_runtime_required", - 220usize, - ), - ( - "crates/j2k-metal/src/compute/tests.rs", - "mod referenced_plan;", - "crates/j2k-metal/src/compute/tests/referenced_plan.rs", - "fn referenced_htj2k_payload_ranges_reconstruct_owned_direct_plan_bytes", - 100usize, - ), - ( - "crates/j2k-cuda/src/session.rs", - "mod tests;", - "crates/j2k-cuda/src/session/tests.rs", - "fn uninitialized_decode_pool_diagnostics_are_empty", - 120usize, - ), - ] { - let owner_source = fs::read_to_string(root.join(owner)) - .unwrap_or_else(|error| panic!("read {owner}: {error}")); - let child_source = fs::read_to_string(root.join(child)) - .unwrap_or_else(|error| panic!("read {child}: {error}")); - assert!(owner_source.contains(module_decl), "{owner} must declare {child}"); - assert!(!owner_source.contains(symbol), "{owner} must not retain {symbol}"); - assert!(child_source.contains(symbol), "{child} must own {symbol}"); - assert!( - child_source.lines().count() < max_lines, - "{child} exceeded its focused {max_lines}-line limit" - ); - assert!(!child_source.lines().any(|line| line.trim() == "use super::*;")); - } -} - -#[test] -fn metal_batch_policy_checks_live_in_their_focused_child_module() { - let root = repo_root(); - let owner_path = root.join("xtask/tests/repo_lint_support/gpu_adapter_policy.rs"); - let child_path = root - .join("xtask/tests/repo_lint_support/gpu_adapter_policy/metal_batch_structure_policy.rs"); - let owner = fs::read_to_string(&owner_path).expect("read GPU adapter policy root"); - let child = fs::read_to_string(&child_path).expect("read Metal batch structure policy"); - - assert!(owner.contains("mod metal_batch_structure_policy;")); - assert!(!owner.contains("fn metal_batch_heuristics_live_in_focused_module(")); - assert!(!owner.contains("fn metal_batch_routes_share_session_aware_implementations(")); - assert!(child.contains("fn metal_batch_heuristics_live_in_focused_module(")); - assert!(child.contains("fn metal_batch_routes_share_session_aware_implementations(")); - assert!( - owner.lines().count() < 1_550, - "GPU adapter policy root must not absorb focused Metal batch checks" - ); - assert!(child.lines().count() < 300); -} - -#[test] -fn metal_batch_execution_policy_checks_live_in_their_focused_child_module() { - let root = repo_root(); - let owner_path = root.join("xtask/tests/repo_lint_support/metal_compute_structure_policy.rs"); - let child_path = root - .join("xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution.rs"); - let owner = fs::read_to_string(&owner_path).expect("read Metal compute policy root"); - let child = fs::read_to_string(&child_path).expect("read Metal batch execution policy"); - - assert!(owner.contains("mod batch_execution;")); - assert!( - !owner.contains("fn metal_ht_chunk_tests_are_split_by_planning_cache_and_status_behavior(") - ); - assert!( - !owner.contains("fn metal_direct_destination_is_split_by_submission_and_group_encoding(") - ); - assert!(!owner - .contains("fn metal_distinct_classic_batch_execution_is_split_from_cleanup_dispatch(")); - assert!( - child.contains("fn metal_ht_chunk_tests_are_split_by_planning_cache_and_status_behavior(") - ); - assert!( - child.contains("fn metal_direct_destination_is_split_by_submission_and_group_encoding(") - ); - assert!( - child.contains("fn metal_distinct_classic_batch_execution_is_split_from_cleanup_dispatch(") - ); - assert!( - owner.lines().count() < 550, - "Metal compute policy root must not absorb batch execution structure checks" - ); - assert!(child.lines().count() < 250); -} - -#[test] -fn metal_multitile_device_tests_are_split_by_pixel_contract() { - let root = repo_root(); - let test_root = root.join("crates/j2k-metal/tests/device/multitile_color"); - let shell = fs::read_to_string(root.join("crates/j2k-metal/tests/device/multitile_color.rs")) - .expect("read Metal multi-tile test shell"); - - for module in ["batch_inputs", "classic", "gray12", "rgb", "signed"] { - assert!(shell.contains(&format!("mod {module};"))); - assert!(test_root.join(format!("{module}.rs")).exists()); - } - assert!(shell.lines().count() < 25); - assert!(!shell.contains("fn independent_openjph_multitile_gray12_decodes_exactly_on_metal(")); - assert!(!shell.contains("fn independent_openjph_multitile_rgb_decodes_exactly_on_metal(")); - assert!(!shell.contains("fn classic_multitile_rgb8_decodes_exactly_on_metal(")); - assert!(fs::read_to_string(test_root.join("gray12.rs")) - .expect("read Gray12 multi-tile tests") - .contains("fn independent_openjph_multitile_gray12_decodes_exactly_on_metal(")); - assert!(fs::read_to_string(test_root.join("rgb.rs")) - .expect("read RGB multi-tile tests") - .contains("fn independent_openjph_multitile_rgb_decodes_exactly_on_metal(")); -} - -#[test] -fn owned_batch_test_support_is_split_by_responsibility() { - let root = repo_root(); - let test_root = root.join("crates/j2k/tests/owned_batch"); - let shell = fs::read_to_string(root.join("crates/j2k/tests/owned_batch.rs")) - .expect("read owned-batch integration-test shell"); - - for (module, owned_symbol, max_lines) in [ - ("fixtures", "fn htj2k_gray8_fixture(", 260usize), - ("oracles", "fn native_request_oracle(", 140usize), - ( - "payload_plan", - "fn assert_prepared_ht_payload_ranges_reconstruct_owned_bytes(", - 180usize, - ), - ( - "native_types_and_requests", - "fn prepared_htj2k_gray_and_rgb_support_native_types_and_requests_exactly(", - 150usize, - ), - ] { - assert!(shell.contains(&format!("mod {module};"))); - let module_path = test_root.join(format!("{module}.rs")); - let source = fs::read_to_string(&module_path) - .unwrap_or_else(|error| panic!("read {}: {error}", module_path.display())); - assert!(source.contains(owned_symbol)); - assert!( - source.lines().count() < max_lines, - "{} exceeded its focused {max_lines}-line limit", - module_path - .strip_prefix(root) - .unwrap_or(&module_path) - .display() - ); - } - assert!( - shell.lines().count() < 40, - "owned_batch.rs must remain a focused integration-test module shell" - ); - assert!(!shell.contains("fn htj2k_gray8_fixture(")); - assert!(!shell.contains("fn native_request_oracle(")); - assert!(!shell.contains("fn assert_prepared_ht_payload_ranges_reconstruct_owned_bytes(")); - - let rgba_path = test_root.join("rgba.rs"); - let rgba = fs::read_to_string(&rgba_path).expect("read focused owned-batch RGBA tests"); - assert!( - rgba.lines().count() < 400, - "{} must remain focused on RGBA behavior", - rgba_path.strip_prefix(root).unwrap_or(&rgba_path).display() - ); - assert!( - !rgba.contains("fn prepared_htj2k_gray_and_rgb_support_native_types_and_requests_exactly(") - ); - - for path in rust_sources(&test_root) { - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - assert!( - !source.lines().any(|line| line.trim() == "use super::*;"), - "{} must import only the owned-batch contracts it uses", - path.strip_prefix(root).unwrap_or(&path).display() - ); - } -} - -#[test] -fn adapter_crates_do_not_import_codec_private_modules() { - assert_rust_source_scan_checks( - repo_root(), - &[RustSourceScanCheck::new( - "adapter codec-private module imports", - &[ - "crates/j2k-jpeg-metal", - "crates/j2k-jpeg-cuda", - "crates/j2k-metal", - "crates/j2k-cuda", - ], - ) - .forbidden(&["::__private", " __private::"])], - ); -} - -#[test] -fn production_j2k_cuda_code_does_not_reference_nvjpeg() { - assert_rust_source_scan_checks( - repo_root(), - &[RustSourceScanCheck::new( - "production J2K CUDA nvJPEG references", - &[ - "crates/j2k-cuda-runtime/src", - "crates/j2k-jpeg-cuda/src", - "crates/j2k-jpeg-cuda/benches", - ], - ) - .forbidden(&["nvjpeg", "nvJPEG", "Nvjpeg", "NVJPEG"])], - ); -} - -#[test] -fn cuda_runtime_rejects_product_cuda_c_and_checked_in_ptx() { - let root = repo_root(); - let cuda_runtime_src = root.join("crates/j2k-cuda-runtime/src"); - let mut forbidden = Vec::new(); - let mut pending = vec![cuda_runtime_src.clone()]; - - while let Some(dir) = pending.pop() { - for entry in - fs::read_dir(&dir).unwrap_or_else(|err| panic!("read {}: {err}", dir.display())) - { - let entry = entry.expect("read directory entry"); - let path = entry.path(); - if path.is_dir() { - pending.push(path); - continue; - } - if !matches!(path.extension().and_then(OsStr::to_str), Some("cu" | "ptx")) { - continue; - } - let rel = path - .strip_prefix(&cuda_runtime_src) - .unwrap_or(&path) - .to_string_lossy() - .replace('\\', "/"); - if rel.starts_with("fixtures/") || rel.starts_with("test-fixtures/") { - continue; - } - forbidden.push(rel); - } - } - - forbidden.sort(); - assert!( - forbidden.is_empty(), - "product CUDA C sources and checked-in product PTX are retired; use CUDA Oxide projects instead:\n{}", - forbidden.join("\n") - ); -} - -#[test] -fn cuda_runtime_dispatch_does_not_read_deprecated_oxide_route_selectors() { - let root = repo_root(); - let runtime_src = root.join("crates/j2k-cuda-runtime/src"); - let deprecated_selector = ["J2K", "CUDA", "USE", "OXIDE"].join("_"); - let mut violations = Vec::new(); - - for path in rust_sources(&runtime_src) { - let source = fs::read_to_string(&path) - .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); - if source.contains(&deprecated_selector) { - violations.push( - path.strip_prefix(root) - .unwrap_or(&path) - .display() - .to_string(), - ); - } - } - - assert!( - violations.is_empty(), - "CUDA Oxide route selection must be feature-driven, not runtime-env driven:\n{}", - violations.join("\n") - ); -} - -#[test] -fn cuda_trace_export_is_non_clobbering_and_documented() { - assert_file_pattern_checks( - repo_root(), - &[ - FilePatternCheck::new("crates/j2k-cuda/src/profile/trace.rs") - .named("CUDA profile trace writer") - .required(&[ - "OpenOptions::new().write(true).create_new(true).open(path)", - "fn write_trace_file", - "emit_trace_write_error(\"cuda_htj2k_trace_write\"", - "emit_trace_write_error(\"cuda_htj2k_encode_trace_write\"", - "std::io::ErrorKind::AlreadyExists", - "CUDA trace path already exists", - ]) - .forbidden(&["std::fs::write(&trace_path"]), - FilePatternCheck::new("crates/j2k-cuda/src/profile/tests.rs") - .named("CUDA trace non-clobber regression") - .required(&[ - "write_trace_file(&path, trace)", - "write_trace_file(&path, \"replace\")", - "ErrorKind::AlreadyExists", - ]), - FilePatternCheck::new("docs/env-vars.md") - .named("environment variable docs") - .required(&[ - "operator-supplied path", - "Existing files are not overwritten", - "parent directories are not created", - ]), - ], - ); -} - -#[test] -fn cuda_adapter_crates_keep_public_libs_as_module_shells() { - let root = repo_root(); - let expected_modules = [ - ( - "crates/j2k-jpeg-cuda", - [ - "codec.rs", - "decoder.rs", - "error.rs", - "runtime.rs", - "session.rs", - "surface.rs", - ] - .as_slice(), - ), - ( - "crates/j2k-cuda", - [ - "codec.rs", - "decoder.rs", - "encode.rs", - "error.rs", - "runtime.rs", - "session.rs", - "surface.rs", - ] - .as_slice(), - ), - ]; - - for (crate_dir, modules) in expected_modules { - let src_dir = root.join(crate_dir).join("src"); - let lib_path = src_dir.join("lib.rs"); - let lib = fs::read_to_string(&lib_path) - .unwrap_or_else(|err| panic!("read {}: {err}", lib_path.display())); - let line_count = lib.lines().count(); - assert!( - line_count <= 220, - "{} should stay a thin public module shell; found {line_count} lines", - lib_path.strip_prefix(root).unwrap_or(&lib_path).display() - ); - - for module in modules { - let module_path = src_dir.join(module); - assert!( - module_path.exists(), - "{} must exist to keep CUDA adapter responsibilities focused", - module_path - .strip_prefix(root) - .unwrap_or(&module_path) - .display() - ); - } - } -} - -#[test] -fn reusable_benchmark_generators_live_in_test_support() { - assert_file_pattern_checks( - repo_root(), - &[FilePatternCheck::new("crates/j2k-test-support/src/lib.rs") - .named("j2k-test-support") - .required(&[ - "pub fn gradient_u8", - "pub fn patterned_rgb8_tiles", - "pub fn gpu_bench_rgb8", - ])], - ); -} - -#[test] -fn gpu_runtime_tests_do_not_silently_return_on_missing_hardware_gates() { - assert_rust_source_scan_checks( - repo_root(), - &[RustSourceScanCheck::new( - "GPU runtime tests must use j2k-test-support gates so optional local skips are visible and CI require gates fail closed", - &[ - "crates/j2k-cuda-runtime/src", - "crates/j2k-cuda/src", - "crates/j2k-cuda/tests", - "crates/j2k-jpeg-cuda/tests", - "crates/j2k-transcode-cuda/tests", - "crates/j2k-metal/src", - "crates/j2k-metal/tests", - "crates/j2k-jpeg-metal/tests", - "crates/j2k-transcode-metal/tests", - ], - ) - .forbidden(&[ - "if !cuda_runtime_required() {\n return;", - "if !cuda_strict_oxide_required() {\n return;", - "if !cuda_jpeg_hardware_decode_required() {\n return;", - "if std::env::var_os(\"J2K_REQUIRE_CUDA_RUNTIME\").is_none() {\n return;", - "if Device::system_default().is_none() {\n eprintln!(\"skipping", - "no Metal device is available", - ])], - ); -} diff --git a/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_commands.rs b/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_commands.rs index 685d9a6c..8777c28e 100644 --- a/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_commands.rs +++ b/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_commands.rs @@ -16,6 +16,9 @@ pub(super) fn assert_regressions_stay_focused() { let orchestration = fs::read_to_string(root.join("xtask/src/release_commands/tests/orchestration.rs")) .expect("read release orchestration command tests"); + let path_patches = + fs::read_to_string(root.join("xtask/src/release_commands/tests/path_patches.rs")) + .expect("read release path-patch command tests"); let validation = fs::read_to_string(root.join("xtask/src/release_commands/tests/validation.rs")) .expect("read release validation command tests"); @@ -37,6 +40,11 @@ pub(super) fn assert_regressions_stay_focused() { &orchestration, 175, ), + ( + "xtask/src/release_commands/tests/path_patches.rs", + &path_patches, + 125, + ), ( "xtask/src/release_commands/tests/validation.rs", &validation, @@ -55,6 +63,7 @@ pub(super) fn assert_regressions_stay_focused() { "mod file_boundaries;", "mod integrity;", "mod orchestration;", + "mod path_patches;", "mod validation;", ]), PatternCheck::new("release file-boundary regressions", &file_boundaries) @@ -71,6 +80,10 @@ pub(super) fn assert_regressions_stay_focused() { "release_integrity_publish_mode_accepts_hermetic_final_metadata", "package_command_executes_list_and_dependency_aware_gates_hermetically", ]), + PatternCheck::new("release path-patch regressions", &path_patches).required(&[ + "path_patch_provenance_discovery_rejects_repository_escape", + "release_integrity_publish_mode_requires_approval_for_every_workspace_path_patch", + ]), PatternCheck::new("release validation command regressions", &validation).required(&[ "fn publish_workflow_validation_reports_parse_and_release_contract_failures()", "fn publish_script_validation_fails_closed_for_missing_and_drifted_contracts()", From b18592171748a495e003234c6a0ff6b2633ccfe8 Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 21 Jul 2026 17:42:37 -0400 Subject: [PATCH 13/22] fix: repair release CI validation --- crates/j2k-metal/src/batch/routes.rs | 9 +++++++-- crates/j2k-metal/src/batch_decoder.rs | 13 +++++++++---- crates/j2k-metal/src/batch_decoder/contracts.rs | 9 +++++++-- crates/j2k-metal/src/batch_decoder/decoder.rs | 8 ++++++-- crates/j2k-metal/src/batch_decoder/resident.rs | 5 +++-- crates/j2k-metal/src/decoder/direct_paths.rs | 1 + .../j2k-metal/src/store_native_color_batch.metal | 1 - crates/j2k-ml/Cargo.toml | 3 +++ docs/unsafe-audit.md | 10 +++++++--- third_party/cubecl-cuda-0.10.0-patched/Cargo.toml | 3 +++ .../cubecl-cuda-0.10.0-patched/Cargo.toml.orig | 2 ++ .../cubecl-runtime-0.10.0-patched/Cargo.toml | 11 +++++++++++ .../cubecl-runtime-0.10.0-patched/Cargo.toml.orig | 10 ++++++++++ third_party/wgpu-29.0.4-patched/README.md | 15 +++++++-------- third_party/wgpu-core-29.0.4-patched/Cargo.toml | 8 +++++++- .../wgpu-core-29.0.4-patched/Cargo.toml.orig | 8 +++++++- .../src/timestamp_normalization/common.wgsl | 2 +- third_party/wgpu-hal-29.0.4-patched/Cargo.toml | 2 +- .../wgpu-hal-29.0.4-patched/Cargo.toml.orig | 2 +- third_party/wgpu-hal-29.0.4-patched/README.md | 1 - 20 files changed, 93 insertions(+), 30 deletions(-) diff --git a/crates/j2k-metal/src/batch/routes.rs b/crates/j2k-metal/src/batch/routes.rs index c5461942..83551393 100644 --- a/crates/j2k-metal/src/batch/routes.rs +++ b/crates/j2k-metal/src/batch/routes.rs @@ -1,8 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +#[cfg(target_os = "macos")] use std::sync::Arc; -use j2k_core::{BackendRequest, PixelFormat}; +use j2k_core::BackendRequest; +#[cfg(target_os = "macos")] +use j2k_core::PixelFormat; use crate::{Error, J2kDecoder, MetalBackendSession, MetalDecodeRequest, Surface}; @@ -11,7 +14,9 @@ use super::heuristics::{ is_region_scaled_direct_batch_candidate, is_repeated_full_color_candidate, is_repeated_full_grayscale_candidate, should_auto_use_metal_for_region_scaled_direct_batch, }; -use super::request::{batch_scheduler_invariant, BatchOp, QueuedRequest}; +#[cfg(target_os = "macos")] +use super::request::batch_scheduler_invariant; +use super::request::{BatchOp, QueuedRequest}; pub(super) fn decode_repeated_full_grayscale( request: &QueuedRequest, diff --git a/crates/j2k-metal/src/batch_decoder.rs b/crates/j2k-metal/src/batch_decoder.rs index 39270e87..36fdb1b3 100644 --- a/crates/j2k-metal/src/batch_decoder.rs +++ b/crates/j2k-metal/src/batch_decoder.rs @@ -1,12 +1,17 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +#[cfg(all(test, target_os = "macos"))] use std::sync::Arc; +#[cfg(any(test, target_os = "macos"))] +use j2k::BatchLayout; use j2k::{ - BatchColor, BatchDecodeOptions, BatchGroupInfo, BatchLayout, DecodeRequest, EncodedImage, - IndexedBatchError, J2kDecodeWarning, PreparedBatch, PreparedBatchGroup, PreparedImage, + BatchDecodeOptions, BatchGroupInfo, EncodedImage, IndexedBatchError, J2kDecodeWarning, + PreparedBatch, PreparedBatchGroup, PreparedImage, }; -use j2k_core::{DeviceSubmission, PixelFormat, Rect}; +#[cfg(any(test, target_os = "macos"))] +use j2k_core::PixelFormat; +use j2k_core::Rect; #[cfg(target_os = "macos")] use j2k_metal_support::{MetalImageDestination, MetalImageLayout, ResidentMetalImage}; @@ -26,7 +31,6 @@ mod external; mod plan_cache; #[cfg(all(test, target_os = "macos"))] mod queue_ordering_tests; -#[cfg(target_os = "macos")] mod resident; #[cfg(target_os = "macos")] mod submission; @@ -57,6 +61,7 @@ pub use self::decoder::MetalBatchDecoder; #[cfg(target_os = "macos")] pub use self::submission::{SubmittedMetalGroupDecodeInto, SubmittedMetalPreparedBatch}; +#[cfg(any(test, target_os = "macos"))] use self::contracts::validate_group_contract; #[cfg(target_os = "macos")] use self::plan_cache::{PreparedColorPlanCache, PreparedGrayPlanCache}; diff --git a/crates/j2k-metal/src/batch_decoder/contracts.rs b/crates/j2k-metal/src/batch_decoder/contracts.rs index 1619f51b..3e03a764 100644 --- a/crates/j2k-metal/src/batch_decoder/contracts.rs +++ b/crates/j2k-metal/src/batch_decoder/contracts.rs @@ -2,11 +2,15 @@ //! Shared Metal batch result and metadata contracts. +#[cfg(target_os = "macos")] +use super::{BatchDecodeOptions, Buffer, ResidentMetalImage}; use super::{ - BatchDecodeOptions, BatchGroupInfo, BatchLayout, Buffer, Error, IndexedBatchError, - J2kDecodeWarning, PixelFormat, PreparedBatchGroup, Rect, ResidentMetalImage, Surface, + BatchGroupInfo, Error, IndexedBatchError, J2kDecodeWarning, PreparedBatchGroup, Rect, Surface, }; +#[cfg(any(test, target_os = "macos"))] +use super::{BatchLayout, PixelFormat}; +#[cfg(any(test, target_os = "macos"))] pub(super) fn validate_group_contract(info: &BatchGroupInfo) -> Result { if !matches!(info.layout, BatchLayout::Nchw | BatchLayout::Nhwc) { return Err(Error::UnsupportedMetalRequest { @@ -28,6 +32,7 @@ pub struct MetalBatchGroupCompletion { } impl MetalBatchGroupCompletion { + #[cfg(target_os = "macos")] pub(super) fn from_prepared(group: &PreparedBatchGroup, options: BatchDecodeOptions) -> Self { let decoded_rects = group .images() diff --git a/crates/j2k-metal/src/batch_decoder/decoder.rs b/crates/j2k-metal/src/batch_decoder/decoder.rs index 42ddb138..59532d15 100644 --- a/crates/j2k-metal/src/batch_decoder/decoder.rs +++ b/crates/j2k-metal/src/batch_decoder/decoder.rs @@ -4,8 +4,11 @@ use super::{ BatchDecodeOptions, EncodedImage, Error, MetalBackendSession, MetalBatchDecodeResult, - MetalBatchGroup, MetalBatchGroupError, MetalImageDestination, PixelFormat, PreparedBatch, - PreparedBatchGroup, PreparedColorPlanCache, PreparedGrayPlanCache, PreparedImage, + MetalBatchGroup, MetalBatchGroupError, PreparedBatch, PreparedBatchGroup, PreparedImage, +}; +#[cfg(target_os = "macos")] +use super::{ + MetalImageDestination, PixelFormat, PreparedColorPlanCache, PreparedGrayPlanCache, SubmittedMetalPreparedBatch, }; @@ -103,6 +106,7 @@ impl MetalBatchDecoder { Ok(self.submission_count) } + #[cfg(target_os = "macos")] pub(super) fn record_submission(&mut self) { self.submission_count = self.submission_count.saturating_add(1); } diff --git a/crates/j2k-metal/src/batch_decoder/resident.rs b/crates/j2k-metal/src/batch_decoder/resident.rs index 4e265cd7..5a850547 100644 --- a/crates/j2k-metal/src/batch_decoder/resident.rs +++ b/crates/j2k-metal/src/batch_decoder/resident.rs @@ -2,11 +2,12 @@ //! Codec-owned resident Metal group submission. +#[cfg(target_os = "macos")] use super::{ allocate_codec_owned_group_destination, validate_codec_owned_resident_group, BatchColor, - BatchDecodeOptions, Error, MetalBatchDecoder, MetalBatchGroup, MetalResidentGroupMetadata, - PreparedBatchGroup, SubmittedMetalResidentGroup, + MetalResidentGroupMetadata, SubmittedMetalResidentGroup, }; +use super::{BatchDecodeOptions, Error, MetalBatchDecoder, MetalBatchGroup, PreparedBatchGroup}; impl MetalBatchDecoder { pub(super) fn decode_prepared_group_with_options( diff --git a/crates/j2k-metal/src/decoder/direct_paths.rs b/crates/j2k-metal/src/decoder/direct_paths.rs index 3ab13ea0..c4f81e79 100644 --- a/crates/j2k-metal/src/decoder/direct_paths.rs +++ b/crates/j2k-metal/src/decoder/direct_paths.rs @@ -9,6 +9,7 @@ use j2k_core::{BackendRequest, PixelFormat}; use j2k_native::{ DecodeSettings as NativeDecodeSettings, Image as NativeImage, J2kDirectGrayscalePlan, }; +#[cfg(target_os = "macos")] use metal::Device; #[cfg(target_os = "macos")] diff --git a/crates/j2k-metal/src/store_native_color_batch.metal b/crates/j2k-metal/src/store_native_color_batch.metal index 81f36ea1..da015ceb 100644 --- a/crates/j2k-metal/src/store_native_color_batch.metal +++ b/crates/j2k-metal/src/store_native_color_batch.metal @@ -211,4 +211,3 @@ kernel void j2k_store_native_rgba_batch_i16( float((1u << (params.bit_depths[3] - 1u)) - 1u) ); } - diff --git a/crates/j2k-ml/Cargo.toml b/crates/j2k-ml/Cargo.toml index 49090ac2..2e950e65 100644 --- a/crates/j2k-ml/Cargo.toml +++ b/crates/j2k-ml/Cargo.toml @@ -10,6 +10,9 @@ publish = false readme = "README.md" autobenches = false +[package.metadata.cargo-machete] +ignored = ["criterion"] + [features] default = [] cpu = [] diff --git a/docs/unsafe-audit.md b/docs/unsafe-audit.md index f9899c08..742ddae3 100644 --- a/docs/unsafe-audit.md +++ b/docs/unsafe-audit.md @@ -87,13 +87,13 @@ moved, or removed. | `crates/j2k-cuda/tests/batch_decoder_api/session_soak.rs` | Test-only repeated external CUDA allocation handoff during the 1,000-batch reuse soak. | The one allocation remains live across the soak but is exclusively borrowed for each submission; every pending owner is waited before the next view or final readback, and pool diagnostics are observed only at completion boundaries. | The resident/external 1,000-batch stable-memory and reusable-session regression on NVIDIA hardware. | | `crates/j2k-cuda/src/decoder.rs` | Adapter-side unsafe release of queued CUDA pool holds after synchronous downstream completion. | Session-private pools remain confined to the owning context; IDWT target owners stay live until context synchronization or a synchronously completed MCT/store launch proves all preceding default-stream work complete. | CUDA decoder structure/accounting tests, queued-resource source policy, host-surface tests, and strict GPU validation. | | `crates/j2k-cuda/src/batch/decoder.rs` | Unsafe public direct-to-external CUDA decode and submission entry points for exact-native Gray, RGB, and RGBA groups. | Group format/layout and the destination view are validated against the decoder context before dispatch. The external allocation stays live and exclusively inaccessible; the returned owner retains codec scratch and descriptors, downstream consumers are stream-ordered after it, and pixels are not exposed until `wait` validates entropy status. Uncertain completion requires allocation quarantine. | Exact RGB/RGBA/signed/classic resident-versus-external parity, `external_batch_handles_roi_reduction_and_drop_safe_session_reuse_when_runtime_required`, and the 1,000-batch external reuse soak. | -| `crates/j2k-cuda/src/decoder/color_batch/native_batch/store/submission.rs` | Adapter dispatch to the six unsafe external enqueue boundaries for native `U8`/`U16`/`I16` RGB and RGBA stores. | Store targets borrow decoded plane owners and are matched to the requested channel count and native format; runtime preflight validates layout and destination coverage. The high-level pending completion retains all planes, uploaded job metadata, and external destination ownership until group status and final-store completion retire together. | Exact RGB, signed RGB, and RGBA external/resident CPU-oracle matrices, pending-drop session reuse, and external lifecycle tests. | +| `crates/j2k-cuda/src/decoder/color_batch/native_batch/store/submission/external.rs` | Adapter dispatch to the six unsafe external enqueue boundaries for native `U8`/`U16`/`I16` RGB and RGBA stores. | Store targets borrow decoded plane owners and are matched to the requested channel count and native format; runtime preflight validates layout and destination coverage. The high-level pending completion retains all planes, uploaded job metadata, and external destination ownership until group status and final-store completion retire together. | Exact RGB, signed RGB, and RGBA external/resident CPU-oracle matrices, pending-drop session reuse, and external lifecycle tests. | | `crates/j2k-cuda/src/decoder/grayscale_batch/store.rs` | Adapter calls to synchronous and queued unsafe external Gray8, Gray16, and GrayI16 final-store APIs. | Targets borrow the decoded coefficient owners and carry plan-derived geometry and native precision. The synchronous route retains inputs and destination through its wait; the queued route transfers the store guard into `GrayscalePendingCompletion`, which retains decoded work and the caller-owned allocation until status validation and store retirement, with quarantine required on uncertain completion. | Classic/HT native grayscale external parity, ROI/reduction and multi-tile regressions, mixed-decomposition lifetime coverage, and the external session-reuse soak. | | `crates/j2k-cuda/src/decoder/pending_completion.rs` | Unsafe release of queued final-store metadata after a later group status readback establishes same-stream completion. | The optimization is used only when a nonempty cleanup or classic status readback was ordered after the final store and returned no completion-uncertain error; otherwise the store event is synchronized directly. IDWT guards, decoded buffers, tables, and pool owners remain retained, and uncertain store completion abandons rather than frees reachable resources. | `resident_and_external_decode_pools_stabilize_for_one_thousand_batches_when_runtime_required` verifies one status transfer replaces the store-event wait; pending-drop lifecycle tests and `error_retirement_attempts_cleanup_and_preserves_both_errors` cover retirement failures. | | `crates/j2k-cuda/src/decoder/resident/cleanup_dequant.rs` | Adapter calls to the unsafe HTJ2K-cleanup enqueue API. | Component work owns every coefficient target, decode resources and session pools stay live and context-matched, and queued handles are retained through status completion, explicit synchronization, or synchronously completed dependent work. | Resident decode unit tests, queued-resource source policy, host-surface tests, and strict GPU validation. | | `crates/j2k-cuda/src/decoder/resident/cleanup_dequant/classic/queued.rs` | Adapter construction and unsafe submission of cross-image classic Tier-1 targets with indexed error mapping. | Source and component counts are checked before submission; component work retains disjoint coefficient destinations, while the surrounding batch owner retains decode payload and session table resources until the queued guard completes. The runtime guard owns copied descriptors, scratch, statuses, and pool holds, and device status indexes are translated through retained source identities. | Classic source-index mapping, runtime preflight and deferred-status tests, exact classic RGB parity, pending-completion lifecycle tests, focused-module policy, and strict CUDA validation. | | `crates/j2k-cuda/src/decoder/resident/cleanup_dequant/enqueue.rs` | Adapter-side selection and chaining of unsafe cleanup, refinement, and dequantization enqueues. | Component work retains all coefficient targets; prepared payload/table resources and session pools remain context-matched and live; the queued guard survives through the ordered final store or another proven same-context completion point. | Cleanup-only/refinement route tests, queued ownership source policy, exact batch parity, and strict CUDA validation. | -| `crates/j2k-cuda/src/decoder/resident/chunked_cleanup/enqueue.rs` | Unsafe grouped-status HTJ2K cleanup/cleanup-dequant enqueue and ordered dequantization chaining for bounded chunks. | Chunk planning validates source identity, payload slices, job counts, coalesced disjoint coefficient targets, status offsets, and the shared status group before submission. Each retained chunk owns uploaded payload/tables and descriptors; component work owns outputs, the pool and group remain context-matched, and the same default stream orders dequantization before later IDWT. Partial failure finishes every already-submitted chunk. | `chunk_local_kernel_failure_maps_to_original_job_and_source`, `chunk_jobs_for_one_coefficient_band_are_adjacent_for_one_runtime_target`, grouped-status index preservation, tiny-cap multi-chunk parity, and strict CUDA validation. | +| `crates/j2k-cuda/src/decoder/resident/chunked_cleanup/enqueue/kernel.rs` | Unsafe grouped-status HTJ2K cleanup/cleanup-dequant enqueue and ordered dequantization chaining for bounded chunks. | Chunk planning validates source identity, payload slices, job counts, coalesced disjoint coefficient targets, status offsets, and the shared status group before submission. Each retained chunk owns uploaded payload/tables and descriptors; component work owns outputs, the pool and group remain context-matched, and the same default stream orders dequantization before later IDWT. Partial failure finishes every already-submitted chunk. | `chunk_local_kernel_failure_maps_to_original_job_and_source`, `chunk_jobs_for_one_coefficient_band_are_adjacent_for_one_runtime_target`, grouped-status index preservation, tiny-cap multi-chunk parity, and strict CUDA validation. | | `crates/j2k-cuda/src/decoder/resident/idwt.rs` | Adapter calls to unsafe IDWT enqueue APIs. | Component work owns every coefficient target, decode resources and session pools stay live and context-matched, and queued handles are retained through status completion, explicit synchronization, or synchronously completed dependent work. | Resident decode unit tests, queued-resource source policy, host-surface tests, and strict GPU validation. | | `crates/j2k-cuda/tests/batch_decoder_api/support.rs` | Unsafe test helper that forms a lifetime-bound external view over an owned CUDA buffer and forwards it to `submit_batch_into`. | The caller supplies the exact live same-context pointer and length under an exclusive mutable borrow and must retain the allocation without host or unordered device access until the returned submission is waited or dropped; the view itself never takes allocation ownership. | Multi-tile external parity, mixed-decomposition lifetime tests, the 1,000-batch reuse soak, and drop-safe external lifecycle tests all exercise the helper contract. | | `crates/j2k-cuda/tests/batch_decoder_api/signed_rgb.rs` | Test-only external allocation handoff for exact signed RGB batch decoding. | The CUDA allocation remains live and exclusively borrowed while the destination view exists; submitted work is waited before host readback or allocation reuse; every request/layout case compares the completed destination against the CPU integer oracle. | Independent OpenJPH signed 8/12/16-bit RGB Full/ROI/reduced parity and resident/external destination regressions on NVIDIA hardware. | @@ -101,7 +101,10 @@ moved, or removed. | `crates/j2k-metal/src/batch_decoder.rs` | Codec-owned/external Metal batch destination adoption and immutable resident-batch raw access. | Codec-owned destinations wrap fresh same-device buffers exclusively and remain retained by pending work; external destinations have validated layout, range, alignment, and device identity. Completed buffers are adopted only after producer completion, and raw resident access is read-only while an owner remains live and no writer overlaps it. | Gray/RGB/RGBA NCHW/NHWC resident and external parity, destination bounds/device tests, continuation/drop-safe session tests, and strict Metal hardware validation. | | `crates/j2k-metal/src/batch_decoder/contracts.rs` | Unsafe read-only Metal buffer access from a completed `MetalResidentBatch`. | A resident batch is constructed only after successful producer completion and is logically immutable; callers bind the handle only for reads, retain the batch or a clone through consumer completion, and exclude every CPU or GPU writer while any resident owner exists. | `prepared_htj2k_rgba_nhwc_resident_group_is_exact_and_uses_one_allocation`, NCHW resident parity, prepared-session resident output tests, and the Metal raw-resource policy. | | `crates/j2k-metal/src/batch_decoder/encoder_count_tests.rs` | Test-only exclusive destination adoption and checked host readback for the one-command-buffer/one-encoder structural matrix. | Each allocation is fresh and has one logical codec writer; the pending group retains the destination through successful completion, and the checked read occurs only after that guard is released. The selected range covers the complete Gray/RGB/RGBA batch. | `external_groups_use_one_producer_command_buffer_and_compute_encoder` checks classic and HT groups of one and eight images for exact CPU parity and one producer encoder. | -| `crates/j2k-metal/src/batch_decoder/queue_ordering_tests.rs` | Test-only exclusive Metal destination construction and raw readback across exact-queue, cross-queue, overlapping, dropped, and concurrent submission scenarios. | Every destination is fresh and retained by its own pending owner; wrong-sized layouts fail before commit, overlapping submissions use distinct status/scratch/output owners, and host reads occur only after producer completion plus any event-ordered consumer command. Concurrent session clones use distinct buffers and one monotonic shared-event timeline. | Exact-queue no-bridge, deferred compatibility bridge, monotonic cross-queue timeline, overlapping-owner, consumer-copy ordering, drop-reuse, and cloned-session commit-order tests in this module. | +| `crates/j2k-metal/src/batch_decoder/queue_ordering_tests/exact_queue.rs` | Test-only exclusive Metal destination construction and raw readback for exact producer/consumer queue submission. | The destination is fresh and retained by its pending owner; the host read occurs only after producer completion releases the exclusive codec write, and exact-queue ordering requires no event bridge. | `known_exact_consumer_queue_submits_without_an_event_bridge`, synchronous external no-bridge, and codec-owned resident no-bridge regressions. | +| `crates/j2k-metal/src/batch_decoder/queue_ordering_tests/cross_queue.rs` | Test-only exclusive Metal destinations and raw readback across cross-queue and concurrent cloned-session submissions. | Every destination and consumer-copy buffer is distinct and retained through producer completion; reads occur only after both the event-ordered producer and consumer commands complete, and cloned sessions serialize values on one monotonic event timeline. | Monotonic cross-queue timeline reuse, consumer-copy ordering, different-device rejection, and cloned-session commit-order tests. | +| `crates/j2k-metal/src/batch_decoder/queue_ordering_tests/fixtures.rs` | Test-only construction of exclusive valid and deliberately undersized Metal destinations. | Each buffer is freshly allocated with one exclusive owner; valid destinations transfer that ownership to pending submission, while the undersized destination is rejected during preflight before codec work commits. | Exact/cross-queue submission tests and `dropping_exact_queue_work_leaves_the_session_reusable`. | +| `crates/j2k-metal/src/batch_decoder/queue_ordering_tests/lifecycle.rs` | Test-only exclusive Metal destination construction and raw readback for overlapping and dropped submission lifecycles. | Overlapping work uses distinct status, scratch, and output owners; wrong-sized layouts fail before commit, and host reads occur only after each pending owner has completed and released its exclusive destination write. | Deferred compatibility bridge, overlapping-owner isolation, preflight rejection, drop retirement, and session-reuse regressions. | | `crates/j2k-metal/src/batch_decoder/submission.rs` | Unsafe adoption of a fresh codec-owned destination and conversion of its completed output into immutable resident batch storage. | Checked size arithmetic and `MetalImageLayout` construction precede adoption; the pending group retains the exclusive destination and raw output owner until submission and status validation succeed. The destination guard is then dropped before `ResidentMetalImage::from_completed_buffer`, and no other raw writable owner or later writer remains. | Resident Gray/RGB/RGBA parity, `dropped_shared_prepared_batch_retires_work_and_reuses_decoder`, nonfatal group continuation, and exact/cross-queue ownership tests. | | `crates/j2k-metal/src/compute/direct_buffers.rs` | MetalDirect host slice buffer uploads and reusable shared-buffer zeroing. | Typed host slices are copied into Metal-owned buffers with validated byte lengths, and reusable buffers are allocated to at least the requested byte count before writes. | MetalDirect decode/encode and resident batch tests. | | `crates/j2k-metal/src/compute/abi.rs` | Unsafe `GpuAbi` marker implementations for Metal compute parameter and status structs. | Every marked type is `repr(C)`, `Copy`, contains only numeric scalar/array fields, accepts every bit pattern, and is mirrored by the Metal shader declaration. | Exact size/alignment/offset tests cover host-read status layouts; Metal compilation checks shader sources and route-specific runtime parity exercises parameter and job layouts. | @@ -126,6 +129,7 @@ moved, or removed. | `crates/j2k-metal/tests/batch_rgba.rs` | Test-only external destination adoption and immutable resident-buffer readback for exact RGBA batches. | Fresh external ranges remain exclusively retained until completion; codec-owned resident groups are exposed only after successful producer completion. Checked raw reads cover validated ranges and occur without any concurrent CPU/GPU writer or escaped handle. | Classic/HT RGBA native-type/request/layout external and NCHW resident parity on the Metal release lane. | | `crates/j2k-metal/tests/device.rs` | Raw Metal surface/codestream constructor and consuming-handoff regression tests. | Test allocations have completed or have no writers; raw ranges are valid; tests retain no concurrently mutating alias. | The device integration test target on the Metal release lane. | | `crates/j2k-metal/tests/device/color_batch.rs` | Test-only exclusive external Metal destinations and checked host reads for exact native `U8`, `U16`, and `I16` RGB batches, including dropped pending work. | Each offset layout is aligned for its native sample type and covers the complete dense group; fresh buffers remain exclusively retained through wait or drop, and only a successfully completed destination is read. Dropping one pending group retires its command before the prepared plan and session are reused. | Exact U8 request/layout, U16 layout, independent signed-RGB request/layout, and `dropped_pending_prepared_ht_rgb_group_reuses_session_and_prepared_plan` tests. | +| `crates/j2k-metal/tests/device/color_mct_group.rs` | Test-only raw readback of completed signed RGB resident batches grouped by color-transform mode. | Each resident allocation is exposed only after codec completion; the test performs a read-only checked host copy while retaining the immutable resident owner and never submits a concurrent writer. | `mixed_signed_rgb_mct_modes_preserve_each_images_samples` compares separated MCT and non-MCT groups against the CPU integer oracle. | | `crates/j2k-metal/tests/device/decode.rs` | Test-only unsafe construction and consuming handoff of raw Metal codestream descriptors. | Fresh buffers have no prior or concurrent writers; construction validates the selected range against capacity, the valid descriptor remains immutable, and consuming raw-buffer handoff occurs only when it is the allocation's sole owner with no cloned handle. | `metal_encoded_raw_parts_validate_ranges_and_support_consuming_handoff`. | | `crates/j2k-metal/tests/device/external_batch.rs` | Test-only external destination adoption and checked reads for subrange validation, odd Gray8 strides, signed Gray12, drop retirement, and consumer-queue ordering. | Each fresh layout is bounds- and format-checked and remains exclusively retained through synchronous completion, pending wait, or drop. Host reads follow release of the destination guard; consumer-copy reads additionally wait for both the producer signal and consumer command, and signed output uses the validated two-byte layout. | Destination-subrange rejection, distinct/unaligned group writes, pending drop/reuse, same-device consumer ordering, and signed Gray12 tests in this module. | | `crates/j2k-metal/tests/device/grayscale_external.rs` | Test-only exclusive Metal destinations and checked host reads for HT ROI/reduction, overlap, classic grayscale, and signed classic Gray12 outputs. | Every buffer range is fresh, correctly pitched and aligned for its pixel format, and retained exclusively by the pending codec work until explicit completion; checked reads use only the completed selected range with no concurrent CPU or GPU writer. | HT ROI/reduction CPU parity, independent SigProp-overlap parity, classic no-staging Gray8, and classic signed Gray12 exact-sample tests. | diff --git a/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml b/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml index 21a144d9..10c6c6ea 100644 --- a/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml +++ b/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml @@ -31,6 +31,9 @@ license = "MIT OR Apache-2.0" repository = "https://github.com/tracel-ai/cubecl/tree/main/crates/cubecl-cuda" resolver = "2" +[package.metadata.cargo-machete] +ignored = ["paste", "pretty_assertions", "test-log"] + [features] default = [ "std", diff --git a/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml.orig b/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml.orig index efe149f6..8b77ee00 100644 --- a/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml.orig +++ b/third_party/cubecl-cuda-0.10.0-patched/Cargo.toml.orig @@ -10,6 +10,8 @@ readme.workspace = true repository = "https://github.com/tracel-ai/cubecl/tree/main/crates/cubecl-cuda" version.workspace = true +[package.metadata.cargo-machete] +ignored = ["paste", "pretty_assertions", "test-log"] [lints] workspace = true diff --git a/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml b/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml index 869c82a8..aa31d47c 100644 --- a/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml +++ b/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml @@ -35,6 +35,17 @@ license = "MIT OR Apache-2.0" repository = "https://github.com/tracel-ai/cubecl/tree/main/crates/cubecl-runtime" resolver = "2" +[package.metadata.cargo-machete] +ignored = [ + "cfg-if", + "cfg_aliases", + "enumset", + "rand", + "serde_json", + "serial_test", + "toml", +] + [features] autotune-checks = [] channel-cell = [] diff --git a/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml.orig b/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml.orig index 01c3efcf..a82a0a93 100644 --- a/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml.orig +++ b/third_party/cubecl-runtime-0.10.0-patched/Cargo.toml.orig @@ -10,6 +10,16 @@ readme.workspace = true repository = "https://github.com/tracel-ai/cubecl/tree/main/crates/cubecl-runtime" version.workspace = true +[package.metadata.cargo-machete] +ignored = [ + "cfg-if", + "cfg_aliases", + "enumset", + "rand", + "serde_json", + "serial_test", + "toml", +] [lints] workspace = true diff --git a/third_party/wgpu-29.0.4-patched/README.md b/third_party/wgpu-29.0.4-patched/README.md index 5060ed39..68119397 100644 --- a/third_party/wgpu-29.0.4-patched/README.md +++ b/third_party/wgpu-29.0.4-patched/README.md @@ -24,9 +24,9 @@ Additionally, [WebGPU Fundamentals] is a tutorial for WebGPU which is very simil ### Wiki -We have a [wiki](https://github.com/gfx-rs/wgpu/wiki) which has information on useful architecture patterns, debugging tips, and more getting started information. +We have a [wiki](https://github.com/gfx-rs/wgpu/wiki) which has information on useful architecture patterns, debugging tips, and more getting started information. -### Need Help? Want to Contribute? +### Need Help? Want to Contribute? The wgpu community uses Matrix and Discord to discuss. @@ -38,7 +38,7 @@ The wgpu community uses Matrix and Discord to discuss. ### Other Languages -To use wgpu in C or dozens of other languages, look at [wgpu-native](https://github.com/gfx-rs/wgpu-native). These are C bindings to wgpu and has an up-to-date list of libraries bringing support to other languages. +To use wgpu in C or dozens of other languages, look at [wgpu-native](https://github.com/gfx-rs/wgpu-native). These are C bindings to wgpu and has an up-to-date list of libraries bringing support to other languages. [Learn WebGPU (for C++)] is a good resource for learning how to use wgpu-native from C++. @@ -72,10 +72,10 @@ Contributors are welcome! See [CONTRIBUTING.md][contrib] for more information. | OpenGL | 🆗 (GL 3.3+) | 🆗 (GL ES 3.0+) | 📐 | 🆗 (WebGL2) | | WebGPU | | | | ✅ | -✅ = First Class Support -🆗 = Downlevel/Best Effort Support -📐 = Requires the [ANGLE](https://github.com/gfx-rs/wgpu/wiki/Running-on-ANGLE) translation layer (GL ES 3.0 only) -🌋 = Requires the [MoltenVK](https://vulkan.lunarg.com/sdk/home#mac) translation layer +✅ = First Class Support\ +🆗 = Downlevel/Best Effort Support\ +📐 = Requires the [ANGLE](https://github.com/gfx-rs/wgpu/wiki/Running-on-ANGLE) translation layer (GL ES 3.0 only)\ +🌋 = Requires the [MoltenVK](https://vulkan.lunarg.com/sdk/home#mac) translation layer\ 🛠️ = Unsupported, though open to contributions ## Environment Variables @@ -165,4 +165,3 @@ Exactly which WGSL features `wgpu` supports depends on how you are using it: [wgsl spec]: https://gpuweb.github.io/gpuweb/wgsl/ [naga]: https://github.com/gfx-rs/wgpu/tree/trunk/naga/ [naga bugs]: https://github.com/gfx-rs/wgpu/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22naga%22 - diff --git a/third_party/wgpu-core-29.0.4-patched/Cargo.toml b/third_party/wgpu-core-29.0.4-patched/Cargo.toml index 94284eb6..e07beecc 100644 --- a/third_party/wgpu-core-29.0.4-patched/Cargo.toml +++ b/third_party/wgpu-core-29.0.4-patched/Cargo.toml @@ -42,7 +42,13 @@ targets = [ ] [package.metadata.cargo-machete] -ignored = ["cfg_aliases"] +ignored = [ + "cfg_aliases", + "wgpu-core-deps-apple", + "wgpu-core-deps-emscripten", + "wgpu-core-deps-wasm", + "wgpu-core-deps-windows-linux-android", +] [features] angle = ["wgpu-core-deps-apple/angle"] diff --git a/third_party/wgpu-core-29.0.4-patched/Cargo.toml.orig b/third_party/wgpu-core-29.0.4-patched/Cargo.toml.orig index 9fa3daee..2aca370c 100644 --- a/third_party/wgpu-core-29.0.4-patched/Cargo.toml.orig +++ b/third_party/wgpu-core-29.0.4-patched/Cargo.toml.orig @@ -27,7 +27,13 @@ targets = [ [package.metadata.cargo-machete] # Cargo machete can't check build.rs dependencies. See https://github.com/bnjbvr/cargo-machete/issues/100 -ignored = ["cfg_aliases"] +ignored = [ + "cfg_aliases", + "wgpu-core-deps-apple", + "wgpu-core-deps-emscripten", + "wgpu-core-deps-wasm", + "wgpu-core-deps-windows-linux-android", +] [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(wgpu_validate_locks)'] } diff --git a/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/common.wgsl b/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/common.wgsl index cb25220e..1acf205a 100644 --- a/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/common.wgsl +++ b/third_party/wgpu-core-29.0.4-patched/src/timestamp_normalization/common.wgsl @@ -42,7 +42,7 @@ fn high(a: u32) -> u32 { return a >> 16; } -/// Combines two 16bit words into a single 32bit word. +/// Combines two 16bit words into a single 32bit word. /// `w1` is the upper 16 bits and `w0` is the lower 16 bits. /// /// The high 16 bits of each argument are discarded. diff --git a/third_party/wgpu-hal-29.0.4-patched/Cargo.toml b/third_party/wgpu-hal-29.0.4-patched/Cargo.toml index f2546186..2832503e 100644 --- a/third_party/wgpu-hal-29.0.4-patched/Cargo.toml +++ b/third_party/wgpu-hal-29.0.4-patched/Cargo.toml @@ -47,7 +47,7 @@ targets = [ ] [package.metadata.cargo-machete] -ignored = ["cfg_aliases"] +ignored = ["cfg_aliases", "windows-result"] [features] device_lost_panic = [] diff --git a/third_party/wgpu-hal-29.0.4-patched/Cargo.toml.orig b/third_party/wgpu-hal-29.0.4-patched/Cargo.toml.orig index 30265268..c58977ae 100644 --- a/third_party/wgpu-hal-29.0.4-patched/Cargo.toml.orig +++ b/third_party/wgpu-hal-29.0.4-patched/Cargo.toml.orig @@ -31,7 +31,7 @@ targets = [ [package.metadata.cargo-machete] # Cargo machete can't check build.rs dependencies. See https://github.com/bnjbvr/cargo-machete/issues/100 -ignored = ["cfg_aliases"] +ignored = ["cfg_aliases", "windows-result"] [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = [ diff --git a/third_party/wgpu-hal-29.0.4-patched/README.md b/third_party/wgpu-hal-29.0.4-patched/README.md index c1abf60a..e2ab9080 100644 --- a/third_party/wgpu-hal-29.0.4-patched/README.md +++ b/third_party/wgpu-hal-29.0.4-patched/README.md @@ -117,4 +117,3 @@ page still applies to this API, with the exception of API tracing/replay functionality, which is only available in `wgpu-core`. [wiki-debug]: https://github.com/gfx-rs/wgpu/wiki/Debugging-wgpu-Applications - From 0c9f46a6701356714a42f2b9dca1673226306a67 Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 21 Jul 2026 18:23:44 -0400 Subject: [PATCH 14/22] fix: preserve Metal batch module imports --- crates/j2k-metal/src/batch_decoder.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/j2k-metal/src/batch_decoder.rs b/crates/j2k-metal/src/batch_decoder.rs index 36fdb1b3..5b96821f 100644 --- a/crates/j2k-metal/src/batch_decoder.rs +++ b/crates/j2k-metal/src/batch_decoder.rs @@ -3,12 +3,16 @@ #[cfg(all(test, target_os = "macos"))] use std::sync::Arc; +#[cfg(target_os = "macos")] +use j2k::DecodeRequest; #[cfg(any(test, target_os = "macos"))] -use j2k::BatchLayout; +use j2k::{BatchColor, BatchLayout}; use j2k::{ BatchDecodeOptions, BatchGroupInfo, EncodedImage, IndexedBatchError, J2kDecodeWarning, PreparedBatch, PreparedBatchGroup, PreparedImage, }; +#[cfg(target_os = "macos")] +use j2k_core::DeviceSubmission; #[cfg(any(test, target_os = "macos"))] use j2k_core::PixelFormat; use j2k_core::Rect; From 9fb1dd27218328965b75a820d9ab47e4cc2aab31 Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 21 Jul 2026 18:29:11 -0400 Subject: [PATCH 15/22] fix: expose Arc to Metal batch plan cache --- crates/j2k-metal/src/batch_decoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/j2k-metal/src/batch_decoder.rs b/crates/j2k-metal/src/batch_decoder.rs index 5b96821f..c04b570e 100644 --- a/crates/j2k-metal/src/batch_decoder.rs +++ b/crates/j2k-metal/src/batch_decoder.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -#[cfg(all(test, target_os = "macos"))] +#[cfg(target_os = "macos")] use std::sync::Arc; #[cfg(target_os = "macos")] From 87d624afc479924dc7084fbe36c1e6025a1c3265 Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 21 Jul 2026 21:50:03 -0400 Subject: [PATCH 16/22] fix: complete cross-platform release validation --- crates/j2k-ml/src/lib.rs | 10 +++++++--- crates/j2k-ml/src/metal/batch.rs | 16 +++++++++------- .../PATCH_PROVENANCE.md | 2 +- .../PATCH_PROVENANCE.md | 2 +- .../wgpu-29.0.4-patched/PATCH_PROVENANCE.md | 2 +- .../wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md | 2 +- .../wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md | 2 +- .../dependency_policy/path_patches.rs | 10 +++++----- 8 files changed, 26 insertions(+), 20 deletions(-) diff --git a/crates/j2k-ml/src/lib.rs b/crates/j2k-ml/src/lib.rs index 4656f887..64a09f01 100644 --- a/crates/j2k-ml/src/lib.rs +++ b/crates/j2k-ml/src/lib.rs @@ -12,9 +12,13 @@ use burn_core::tensor::{backend::Backend, Int, Tensor}; use j2k::{BatchGroupInfo, IndexedBatchError, J2kDecodeWarning, Rect}; -#[cfg(any(feature = "cpu", feature = "cuda", feature = "metal"))] +#[cfg(any( + feature = "cpu", + feature = "cuda", + all(feature = "metal", target_os = "macos") +))] mod batch_contract; -#[cfg(any(feature = "cuda", feature = "metal", test))] +#[cfg(any(feature = "cuda", all(feature = "metal", target_os = "macos"), test))] mod completion; #[cfg(feature = "cpu")] @@ -89,7 +93,7 @@ pub struct BurnBatchGroupError { } impl BurnBatchGroupError { - #[cfg(any(feature = "cuda", feature = "metal", test))] + #[cfg(any(feature = "cuda", all(feature = "metal", target_os = "macos"), test))] pub(crate) fn new(source_indices: Vec, source: BurnDecodeError) -> Self { Self { source_indices, diff --git a/crates/j2k-ml/src/metal/batch.rs b/crates/j2k-ml/src/metal/batch.rs index 94d5511b..135f0e99 100644 --- a/crates/j2k-ml/src/metal/batch.rs +++ b/crates/j2k-ml/src/metal/batch.rs @@ -1,17 +1,18 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +#[cfg(target_os = "macos")] use burn_core::tensor::backend::Backend; use burn_wgpu::{Wgpu, WgpuDevice}; -use j2k::{ - BatchDecodeOptions, BatchGroupInfo, EncodedImage, IndexedBatchError, NativeSampleType, - PreparedBatch, PreparedBatchGroup, PreparedImage, -}; +use j2k::{BatchDecodeOptions, EncodedImage, PreparedBatch, PreparedImage}; +#[cfg(target_os = "macos")] +use j2k::{BatchGroupInfo, IndexedBatchError, NativeSampleType, PreparedBatchGroup}; use j2k_metal::MetalBatchDecoder as CodecDecoder; +#[cfg(target_os = "macos")] use crate::batch_contract::{dtype, tensor_shape}; -use crate::{ - BurnBatchDecode, BurnBatchGroup, BurnBatchGroupError, BurnBatchTensor, BurnDecodeError, -}; +use crate::{BurnBatchDecode, BurnDecodeError}; +#[cfg(target_os = "macos")] +use crate::{BurnBatchGroup, BurnBatchGroupError, BurnBatchTensor}; #[cfg(target_os = "macos")] use super::interop::{ @@ -78,6 +79,7 @@ impl SubmittedMetalBurnBatch { #[cfg(not(target_os = "macos"))] #[derive(Debug)] +/// Uninhabited pending Metal batch returned only for cross-platform API compatibility. pub struct SubmittedMetalBurnBatch { _private: (), } diff --git a/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md b/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md index a950e740..03d25f50 100644 --- a/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md +++ b/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md @@ -9,7 +9,7 @@ Pinned SHA-256 digests: - patched local `src/compute/command.rs`: `d71de50b6035d5b98895e21b0fd6994a389b25240f48812e8a1231a0b6cdbe5c` - upstream `src/compute/server.rs`: `7411535c6ae5c72efac9a85ec42d451a8f7ece4e890c1d4ecf35ff4ff4cd2a1d` - patched local `src/compute/server.rs`: `ad81a690415f228c5a43f9972f06a3ae1bc7ba8673e3a545d79e15a6655d5a3c` -- patched tree inventory, excluding this provenance file: `7ea9844b39b5402a2f0afec42cbe92aa48f3f6a46c8e05af948e711a555eff73` +- patched tree inventory, excluding this provenance file: `7af55592787c15a3ac4647339dcd638cd8040588ded8aeae3c96a7398d8261de` Local change: implement CubeCL runtime's hidden external-write hook by resolving the allocation binding on its owning CUDA stream and returning that diff --git a/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md b/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md index 61b35bac..f21a500d 100644 --- a/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md +++ b/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md @@ -9,7 +9,7 @@ Pinned SHA-256 digests: - patched local `src/client.rs`: `2ba96df7c895b975d9e171f3c2452fc61e887091af565384061bbd816d471e1b` - upstream `src/server/base.rs`: `c6292c16d5a73d7a1570776ffd25ec10afc339b942dbc5a693d8c791feb8ef5b` - patched local `src/server/base.rs`: `93ad450ed4aba9c39478be65afd428110118f6014dd080f79ced701612baafe2` -- patched tree inventory, excluding this provenance file: `e7d20369aff883f81057356bce8b48eb1c33fb35195759d9e0fbd8e836021a35` +- patched tree inventory, excluding this provenance file: `125c6b04cf8d43fabc00360d9c79ea61141362946528cf376c8b631ac2b21106` Local change: add one hidden `ExternalWriteServer` hook and one unsafe `ComputeClient::external_write_stream` method. The hook resolves the normal diff --git a/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md b/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md index fbebf3b5..f95d3652 100644 --- a/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md +++ b/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md @@ -9,7 +9,7 @@ Pinned SHA-256 digests: - patched local `src/api/buffer.rs`: `19ba97b38b63fd491e480cfcc773fa2055f691a97ee727d7541a7255bbf82ba4` - upstream `src/backend/wgpu_core.rs`: `8c0e1fab257ef93a86f6d6c93f1970f9756dececcaf3cebaa398a90a9d58efa9` - patched local `src/backend/wgpu_core.rs`: `efdb2d9a3351718d29ca0a4a843b1e61cc778dcb5fb46a0c365e9dd79507101c` -- patched tree inventory, excluding this provenance file: `21872d88aa2b24f4a24f40148d3cc6182067e783885762fc91a8844cd934e3ba` +- patched tree inventory, excluding this provenance file: `d3baa4fa75e494319ee92b7849ebaacbd7ae88d62743df728c0dc779c5fdf839` Local change: add a hidden, unsafe, range-scoped `Buffer::mark_external_write_initialized` hook and its private-construction diff --git a/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md b/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md index b7aa6c4d..25bef269 100644 --- a/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md +++ b/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md @@ -7,7 +7,7 @@ Pinned SHA-256 digests: - crates.io archive: `2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7` - upstream `src/device/global.rs`: `8d8554b2400b46b595af68b0d2fab472da06b8c11fa1823299728b5bc2cf2caa` - patched local `src/device/global.rs`: `ceab63ff6129a22e1bea8e9e1e691edb745080541d9c65b3ec043981d0a21978` -- patched tree inventory, excluding this provenance file: `e494cac012a4503763ffb461980967eedf23b5ac00b8cde28020a558dcef04a5` +- patched tree inventory, excluding this provenance file: `fd09d50213becd3f1b4a2bb5c714049d0be68d2b30cc9bc75ddb99d98bdc2296` Local change: add the hidden core half of the audited external-write handoff. It validates that the requested byte range belongs to a live buffer and drains diff --git a/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md b/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md index ade4a0d5..e76e26c8 100644 --- a/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md +++ b/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md @@ -7,7 +7,7 @@ Pinned SHA-256 digests: - crates.io archive: `97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d` - upstream `src/metal/mod.rs`: `daed7a2d7c1cd7b9431b3009b5b7feb2ca8e45648ffd7eb0185c46cfcbeb2ada` - patched local `src/metal/mod.rs`: `ffa9a5a9767b5e458d50001a66bdf904be5a3199adfb213cac033a9765d0e5a6` -- patched tree inventory, excluding this provenance file: `fd7b218ab74d8ad48090c3e03dd2f738ada2c1644a96d2932b5b145c24762449` +- patched tree inventory, excluding this provenance file: `3ea41da6b01f5c50a996cd7c5208820eae48a76b5227f25a4fa8b20e8ddf7700` Local change: add three Metal-only retained raw-handle accessors, for the selected `MTLDevice`, its `MTLCommandQueue`, and an `MTLBuffer`. Each accessor diff --git a/xtask/tests/repo_lint_support/dependency_policy/path_patches.rs b/xtask/tests/repo_lint_support/dependency_policy/path_patches.rs index 4f3617bf..787f0e71 100644 --- a/xtask/tests/repo_lint_support/dependency_policy/path_patches.rs +++ b/xtask/tests/repo_lint_support/dependency_policy/path_patches.rs @@ -41,7 +41,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "cubecl-cuda", directory: "third_party/cubecl-cuda-0.10.0-patched", archive_sha256: "b6b0a69ff45688d322ad8e92c8bf645167b9ca490fa8fa087fc6adac8c5e46be", - tree_sha256: "7ea9844b39b5402a2f0afec42cbe92aa48f3f6a46c8e05af948e711a555eff73", + tree_sha256: "7af55592787c15a3ac4647339dcd638cd8040588ded8aeae3c96a7398d8261de", files: &[ PatchedFile { path: "src/compute/command.rs", @@ -59,7 +59,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "cubecl-runtime", directory: "third_party/cubecl-runtime-0.10.0-patched", archive_sha256: "b68491bf5b3e997ae36bdc4e63b4ccd6d2f0e86b3b596a5d7a48d2b9e92622a0", - tree_sha256: "e7d20369aff883f81057356bce8b48eb1c33fb35195759d9e0fbd8e836021a35", + tree_sha256: "125c6b04cf8d43fabc00360d9c79ea61141362946528cf376c8b631ac2b21106", files: &[ PatchedFile { path: "src/client.rs", @@ -77,7 +77,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "wgpu", directory: "third_party/wgpu-29.0.4-patched", archive_sha256: "76e8840e1ba2881d4cbb18d2147627a56af426ff064c0401eb0c8410c6325d07", - tree_sha256: "21872d88aa2b24f4a24f40148d3cc6182067e783885762fc91a8844cd934e3ba", + tree_sha256: "d3baa4fa75e494319ee92b7849ebaacbd7ae88d62743df728c0dc779c5fdf839", files: &[ PatchedFile { path: "src/api/buffer.rs", @@ -95,7 +95,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "wgpu-core", directory: "third_party/wgpu-core-29.0.4-patched", archive_sha256: "2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7", - tree_sha256: "e494cac012a4503763ffb461980967eedf23b5ac00b8cde28020a558dcef04a5", + tree_sha256: "fd09d50213becd3f1b4a2bb5c714049d0be68d2b30cc9bc75ddb99d98bdc2296", files: &[PatchedFile { path: "src/device/global.rs", upstream_sha256: "8d8554b2400b46b595af68b0d2fab472da06b8c11fa1823299728b5bc2cf2caa", @@ -106,7 +106,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "wgpu-hal", directory: "third_party/wgpu-hal-29.0.4-patched", archive_sha256: "97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d", - tree_sha256: "fd7b218ab74d8ad48090c3e03dd2f738ada2c1644a96d2932b5b145c24762449", + tree_sha256: "3ea41da6b01f5c50a996cd7c5208820eae48a76b5227f25a4fa8b20e8ddf7700", files: &[PatchedFile { path: "src/metal/mod.rs", upstream_sha256: "daed7a2d7c1cd7b9431b3009b5b7feb2ca8e45648ffd7eb0185c46cfcbeb2ada", From e44d56304c4241d0d12a8dd632472c275d42edb2 Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 21 Jul 2026 21:57:59 -0400 Subject: [PATCH 17/22] fix: make release gates reproducible on Linux --- .../j2k-metal/src/batch_decoder/resident.rs | 1 + .../block-0.1.6-patched/PATCH_PROVENANCE.md | 2 +- .../PATCH_PROVENANCE.md | 2 +- .../PATCH_PROVENANCE.md | 2 +- .../wgpu-29.0.4-patched/PATCH_PROVENANCE.md | 2 +- .../PATCH_PROVENANCE.md | 2 +- .../PATCH_PROVENANCE.md | 2 +- .../dependency_policy/path_patches.rs | 38 +++++++++++++++---- 8 files changed, 38 insertions(+), 13 deletions(-) diff --git a/crates/j2k-metal/src/batch_decoder/resident.rs b/crates/j2k-metal/src/batch_decoder/resident.rs index 5a850547..91f38f45 100644 --- a/crates/j2k-metal/src/batch_decoder/resident.rs +++ b/crates/j2k-metal/src/batch_decoder/resident.rs @@ -23,6 +23,7 @@ impl MetalBatchDecoder { #[cfg(not(target_os = "macos"))] { + let _ = self; let _ = group; let _ = options; Err(Error::MetalUnavailable) diff --git a/third_party/block-0.1.6-patched/PATCH_PROVENANCE.md b/third_party/block-0.1.6-patched/PATCH_PROVENANCE.md index 88bea84e..04ab982a 100644 --- a/third_party/block-0.1.6-patched/PATCH_PROVENANCE.md +++ b/third_party/block-0.1.6-patched/PATCH_PROVENANCE.md @@ -8,7 +8,7 @@ Pinned SHA-256 digests: - crates.io `block-0.1.6.crate` archive: `0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a` - upstream `block-0.1.6/src/lib.rs`: `eb31678adf63b53109d9b94eba23699fd5f9ebfdb950f6e1a57ad51bb6a146fa` - patched local `src/lib.rs`: `bf799f4d01bb497fdcffe7a5e28d998e721ed45c1be866ed1b454df39ce876a9` -- patched tree inventory, excluding this provenance file: `b560316824dcbd524cac358019017f31c65ee1664cfdc15177e807f1132d1f48` +- patched tree inventory, excluding this provenance file and generated root lockfile: `ed11b5084e7c790c36466b1ea4033b9b8b1378739c38346b888d7c55178b3214` Documented ABI deltas from upstream are intentionally limited to: diff --git a/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md b/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md index 03d25f50..7cb00b6f 100644 --- a/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md +++ b/third_party/cubecl-cuda-0.10.0-patched/PATCH_PROVENANCE.md @@ -9,7 +9,7 @@ Pinned SHA-256 digests: - patched local `src/compute/command.rs`: `d71de50b6035d5b98895e21b0fd6994a389b25240f48812e8a1231a0b6cdbe5c` - upstream `src/compute/server.rs`: `7411535c6ae5c72efac9a85ec42d451a8f7ece4e890c1d4ecf35ff4ff4cd2a1d` - patched local `src/compute/server.rs`: `ad81a690415f228c5a43f9972f06a3ae1bc7ba8673e3a545d79e15a6655d5a3c` -- patched tree inventory, excluding this provenance file: `7af55592787c15a3ac4647339dcd638cd8040588ded8aeae3c96a7398d8261de` +- patched tree inventory, excluding this provenance file and generated root lockfile: `bc1d7a26f591d748244ac3b18d4d641969d03506dff9d0f1c3ce73e10086166d` Local change: implement CubeCL runtime's hidden external-write hook by resolving the allocation binding on its owning CUDA stream and returning that diff --git a/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md b/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md index f21a500d..bc36a1db 100644 --- a/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md +++ b/third_party/cubecl-runtime-0.10.0-patched/PATCH_PROVENANCE.md @@ -9,7 +9,7 @@ Pinned SHA-256 digests: - patched local `src/client.rs`: `2ba96df7c895b975d9e171f3c2452fc61e887091af565384061bbd816d471e1b` - upstream `src/server/base.rs`: `c6292c16d5a73d7a1570776ffd25ec10afc339b942dbc5a693d8c791feb8ef5b` - patched local `src/server/base.rs`: `93ad450ed4aba9c39478be65afd428110118f6014dd080f79ced701612baafe2` -- patched tree inventory, excluding this provenance file: `125c6b04cf8d43fabc00360d9c79ea61141362946528cf376c8b631ac2b21106` +- patched tree inventory, excluding this provenance file and generated root lockfile: `0ae2be14bb10fc27c633c99b0a862dce8b237fc32768fd63eff0b6ad53ef0047` Local change: add one hidden `ExternalWriteServer` hook and one unsafe `ComputeClient::external_write_stream` method. The hook resolves the normal diff --git a/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md b/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md index f95d3652..ecfc2c10 100644 --- a/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md +++ b/third_party/wgpu-29.0.4-patched/PATCH_PROVENANCE.md @@ -9,7 +9,7 @@ Pinned SHA-256 digests: - patched local `src/api/buffer.rs`: `19ba97b38b63fd491e480cfcc773fa2055f691a97ee727d7541a7255bbf82ba4` - upstream `src/backend/wgpu_core.rs`: `8c0e1fab257ef93a86f6d6c93f1970f9756dececcaf3cebaa398a90a9d58efa9` - patched local `src/backend/wgpu_core.rs`: `efdb2d9a3351718d29ca0a4a843b1e61cc778dcb5fb46a0c365e9dd79507101c` -- patched tree inventory, excluding this provenance file: `d3baa4fa75e494319ee92b7849ebaacbd7ae88d62743df728c0dc779c5fdf839` +- patched tree inventory, excluding this provenance file and generated root lockfile: `c958c8a0308bc8c1b3753f32f5696cc5d15ef50f3dec56b64eb1936d9a22a1f5` Local change: add a hidden, unsafe, range-scoped `Buffer::mark_external_write_initialized` hook and its private-construction diff --git a/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md b/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md index 25bef269..707652f4 100644 --- a/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md +++ b/third_party/wgpu-core-29.0.4-patched/PATCH_PROVENANCE.md @@ -7,7 +7,7 @@ Pinned SHA-256 digests: - crates.io archive: `2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7` - upstream `src/device/global.rs`: `8d8554b2400b46b595af68b0d2fab472da06b8c11fa1823299728b5bc2cf2caa` - patched local `src/device/global.rs`: `ceab63ff6129a22e1bea8e9e1e691edb745080541d9c65b3ec043981d0a21978` -- patched tree inventory, excluding this provenance file: `fd09d50213becd3f1b4a2bb5c714049d0be68d2b30cc9bc75ddb99d98bdc2296` +- patched tree inventory, excluding this provenance file and generated root lockfile: `13631d9cf9230ea52ae5458c440d60ccf23988839b9d89ff48cb026e7c324411` Local change: add the hidden core half of the audited external-write handoff. It validates that the requested byte range belongs to a live buffer and drains diff --git a/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md b/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md index e76e26c8..215b8233 100644 --- a/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md +++ b/third_party/wgpu-hal-29.0.4-patched/PATCH_PROVENANCE.md @@ -7,7 +7,7 @@ Pinned SHA-256 digests: - crates.io archive: `97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d` - upstream `src/metal/mod.rs`: `daed7a2d7c1cd7b9431b3009b5b7feb2ca8e45648ffd7eb0185c46cfcbeb2ada` - patched local `src/metal/mod.rs`: `ffa9a5a9767b5e458d50001a66bdf904be5a3199adfb213cac033a9765d0e5a6` -- patched tree inventory, excluding this provenance file: `3ea41da6b01f5c50a996cd7c5208820eae48a76b5227f25a4fa8b20e8ddf7700` +- patched tree inventory, excluding this provenance file and generated root lockfile: `c6cbbd822bd03017303b9270cad39adc20ebc97b3b0765bd8b088c5957674269` Local change: add three Metal-only retained raw-handle accessors, for the selected `MTLDevice`, its `MTLCommandQueue`, and an `MTLBuffer`. Each accessor diff --git a/xtask/tests/repo_lint_support/dependency_policy/path_patches.rs b/xtask/tests/repo_lint_support/dependency_policy/path_patches.rs index 787f0e71..d36e5542 100644 --- a/xtask/tests/repo_lint_support/dependency_policy/path_patches.rs +++ b/xtask/tests/repo_lint_support/dependency_policy/path_patches.rs @@ -30,7 +30,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "block", directory: "third_party/block-0.1.6-patched", archive_sha256: "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a", - tree_sha256: "b560316824dcbd524cac358019017f31c65ee1664cfdc15177e807f1132d1f48", + tree_sha256: "ed11b5084e7c790c36466b1ea4033b9b8b1378739c38346b888d7c55178b3214", files: &[PatchedFile { path: "src/lib.rs", upstream_sha256: "eb31678adf63b53109d9b94eba23699fd5f9ebfdb950f6e1a57ad51bb6a146fa", @@ -41,7 +41,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "cubecl-cuda", directory: "third_party/cubecl-cuda-0.10.0-patched", archive_sha256: "b6b0a69ff45688d322ad8e92c8bf645167b9ca490fa8fa087fc6adac8c5e46be", - tree_sha256: "7af55592787c15a3ac4647339dcd638cd8040588ded8aeae3c96a7398d8261de", + tree_sha256: "bc1d7a26f591d748244ac3b18d4d641969d03506dff9d0f1c3ce73e10086166d", files: &[ PatchedFile { path: "src/compute/command.rs", @@ -59,7 +59,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "cubecl-runtime", directory: "third_party/cubecl-runtime-0.10.0-patched", archive_sha256: "b68491bf5b3e997ae36bdc4e63b4ccd6d2f0e86b3b596a5d7a48d2b9e92622a0", - tree_sha256: "125c6b04cf8d43fabc00360d9c79ea61141362946528cf376c8b631ac2b21106", + tree_sha256: "0ae2be14bb10fc27c633c99b0a862dce8b237fc32768fd63eff0b6ad53ef0047", files: &[ PatchedFile { path: "src/client.rs", @@ -77,7 +77,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "wgpu", directory: "third_party/wgpu-29.0.4-patched", archive_sha256: "76e8840e1ba2881d4cbb18d2147627a56af426ff064c0401eb0c8410c6325d07", - tree_sha256: "d3baa4fa75e494319ee92b7849ebaacbd7ae88d62743df728c0dc779c5fdf839", + tree_sha256: "c958c8a0308bc8c1b3753f32f5696cc5d15ef50f3dec56b64eb1936d9a22a1f5", files: &[ PatchedFile { path: "src/api/buffer.rs", @@ -95,7 +95,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "wgpu-core", directory: "third_party/wgpu-core-29.0.4-patched", archive_sha256: "2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7", - tree_sha256: "fd09d50213becd3f1b4a2bb5c714049d0be68d2b30cc9bc75ddb99d98bdc2296", + tree_sha256: "13631d9cf9230ea52ae5458c440d60ccf23988839b9d89ff48cb026e7c324411", files: &[PatchedFile { path: "src/device/global.rs", upstream_sha256: "8d8554b2400b46b595af68b0d2fab472da06b8c11fa1823299728b5bc2cf2caa", @@ -106,7 +106,7 @@ const PATH_PATCHES: &[PathPatch] = &[ name: "wgpu-hal", directory: "third_party/wgpu-hal-29.0.4-patched", archive_sha256: "97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d", - tree_sha256: "3ea41da6b01f5c50a996cd7c5208820eae48a76b5227f25a4fa8b20e8ddf7700", + tree_sha256: "c6cbbd822bd03017303b9270cad39adc20ebc97b3b0765bd8b088c5957674269", files: &[PatchedFile { path: "src/metal/mod.rs", upstream_sha256: "daed7a2d7c1cd7b9431b3009b5b7feb2ca8e45648ffd7eb0185c46cfcbeb2ada", @@ -139,6 +139,28 @@ fn patched_tree_digest_includes_nested_provenance_named_files() { assert_ne!(first, second, "only the root provenance record is excluded"); } +#[test] +fn patched_tree_digest_excludes_generated_root_lockfile() { + let directory = std::env::temp_dir().join(format!( + "j2k-patched-tree-lockfile-{}-{}", + std::process::id(), + NEXT_PATCH_TREE_TEST_ID.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&directory).expect("create patched-tree test directory"); + fs::write(directory.join("src.rs"), "payload").expect("write patched payload"); + let without_lockfile = patched_tree_sha256(&directory); + + fs::write(directory.join("Cargo.lock"), "generated lockfile") + .expect("write generated root lockfile"); + let with_lockfile = patched_tree_sha256(&directory); + fs::remove_dir_all(&directory).expect("remove patched-tree test directory"); + + assert_eq!( + without_lockfile, with_lockfile, + "an ignored generated root lockfile is not part of a reproducible patch tree" + ); +} + #[test] fn all_workspace_path_patches_have_pinned_provenance_and_local_digests() { let root = repo_root(); @@ -241,7 +263,9 @@ fn patched_tree_sha256(directory: &Path) -> String { let path = entry.expect("read patched tree entry").path(); if path.is_dir() { pending.push(path); - } else if path != directory.join("PATCH_PROVENANCE.md") { + } else if path != directory.join("PATCH_PROVENANCE.md") + && path != directory.join("Cargo.lock") + { let relative = path .strip_prefix(directory) .expect("patched file below patch directory") From 25ee3fae43293f1379c6c19a93334751492bdf91 Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 21 Jul 2026 22:05:43 -0400 Subject: [PATCH 18/22] fix: complete Linux Metal validation --- crates/j2k-metal/src/batch/tests.rs | 5 ++++- crates/j2k-metal/src/batch_decoder/contracts.rs | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/j2k-metal/src/batch/tests.rs b/crates/j2k-metal/src/batch/tests.rs index 07c31b0e..1d3369bf 100644 --- a/crates/j2k-metal/src/batch/tests.rs +++ b/crates/j2k-metal/src/batch/tests.rs @@ -6,10 +6,13 @@ use j2k_core::{BackendRequest, Downscale, PixelFormat, Rect}; use crate::{Error, MetalSession}; +#[cfg(target_os = "macos")] use super::execute::process_batch; +#[cfg(target_os = "macos")] +use super::heuristics::GroupedRequests; use super::heuristics::{ auto_region_scaled_direct_metal_min_dim, can_decode_requests_as_repeated_region_scaled_batch, - group_metal_requests, profile_route_label, same_input_bytes, BatchRoute, GroupedRequests, + group_metal_requests, profile_route_label, same_input_bytes, BatchRoute, AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT, AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM, AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT, AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, diff --git a/crates/j2k-metal/src/batch_decoder/contracts.rs b/crates/j2k-metal/src/batch_decoder/contracts.rs index 3e03a764..21570411 100644 --- a/crates/j2k-metal/src/batch_decoder/contracts.rs +++ b/crates/j2k-metal/src/batch_decoder/contracts.rs @@ -246,8 +246,8 @@ impl MetalBatchGroup { /// /// Gray groups and NHWC color groups expose one convenience [`Surface`] /// per image. NCHW color groups return an empty slice because a [`Surface`] - /// promises interleaved pixel semantics; use [`Self::resident_batch`] for - /// their dense planar allocation. + /// promises interleaved pixel semantics; on macOS, use `resident_batch` + /// for their dense planar allocation. pub fn surfaces(&self) -> &[Surface] { &self.surfaces } From 7a7c90bdab387c90d24e3d3bcfc215d74cce7805 Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 21 Jul 2026 22:19:08 -0400 Subject: [PATCH 19/22] fix: close remaining Linux release checks --- crates/j2k-metal/src/batch/session.rs | 1 + crates/j2k-ml/benches/batch_decode_metal.rs | 5 +++++ xtask/src/coverage/source_analysis/workspace.rs | 3 +++ xtask/src/coverage/source_analysis/workspace/tests.rs | 2 ++ 4 files changed, 11 insertions(+) diff --git a/crates/j2k-metal/src/batch/session.rs b/crates/j2k-metal/src/batch/session.rs index d464315f..7fa3b326 100644 --- a/crates/j2k-metal/src/batch/session.rs +++ b/crates/j2k-metal/src/batch/session.rs @@ -53,6 +53,7 @@ impl SharedSession { } } +/// Pending surface decode submitted through [`MetalSession`](crate::MetalSession). pub struct MetalSubmission { session: SharedSession, pub(super) slot: Option, diff --git a/crates/j2k-ml/benches/batch_decode_metal.rs b/crates/j2k-ml/benches/batch_decode_metal.rs index 9b1968bf..a5fbe8a9 100644 --- a/crates/j2k-ml/benches/batch_decode_metal.rs +++ b/crates/j2k-ml/benches/batch_decode_metal.rs @@ -9,7 +9,9 @@ use j2k_ml::{CpuBurnDecoder, MetalBurnDecoder}; #[path = "batch_decode_metal/instrumentation.rs"] mod instrumentation; +#[cfg(target_os = "macos")] mod metal_telemetry; +#[cfg(target_os = "macos")] #[path = "batch_decode_metal/profile.rs"] mod profile; mod support; @@ -37,7 +39,10 @@ fn main() { bench_burn_direct(&mut criterion, &workload_specs, input_mode); criterion.final_summary(); } + #[cfg(target_os = "macos")] ProcessMode::Profile => profile::run(&workload_specs, input_mode), + #[cfg(not(target_os = "macos"))] + ProcessMode::Profile => panic!("Metal profiling requires macOS"), } } diff --git a/xtask/src/coverage/source_analysis/workspace.rs b/xtask/src/coverage/source_analysis/workspace.rs index 1e8f056b..0c95539b 100644 --- a/xtask/src/coverage/source_analysis/workspace.rs +++ b/xtask/src/coverage/source_analysis/workspace.rs @@ -275,6 +275,9 @@ pub(super) fn classify_unreached_source(root: &Path, path: &str) -> Result Date: Tue, 21 Jul 2026 22:30:29 -0400 Subject: [PATCH 20/22] fix: classify pinned GPU interop sources --- xtask/src/coverage/source_analysis.rs | 1 + .../src/coverage/source_analysis/workspace.rs | 21 ++++++++++++++++++- .../source_analysis/workspace/tests.rs | 17 +++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/xtask/src/coverage/source_analysis.rs b/xtask/src/coverage/source_analysis.rs index 5bf67779..c4402e16 100644 --- a/xtask/src/coverage/source_analysis.rs +++ b/xtask/src/coverage/source_analysis.rs @@ -32,6 +32,7 @@ use workspace::{ pub(super) const GENERATED_DWT_DISPOSITION: &str = "generated-codec-math-fragment"; pub(super) const VENDORED_BLOCK_DISPOSITION: &str = "vendored-block-ffi-binding"; +pub(super) const VENDORED_GPU_INTEROP_DISPOSITION: &str = "vendored-gpu-interop-patch"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum SourceRole { diff --git a/xtask/src/coverage/source_analysis/workspace.rs b/xtask/src/coverage/source_analysis/workspace.rs index 0c95539b..5cab64b5 100644 --- a/xtask/src/coverage/source_analysis/workspace.rs +++ b/xtask/src/coverage/source_analysis/workspace.rs @@ -11,7 +11,10 @@ use crate::process::{self, cargo, CommandContext}; use super::ast::validate_source; use super::cfg_eval::CoverageCfgContext; use super::graph::ReachKind; -use super::{SourceRole, GENERATED_DWT_DISPOSITION, VENDORED_BLOCK_DISPOSITION}; +use super::{ + SourceRole, GENERATED_DWT_DISPOSITION, VENDORED_BLOCK_DISPOSITION, + VENDORED_GPU_INTEROP_DISPOSITION, +}; mod fuzz_manifests; #[cfg(test)] mod tests; @@ -62,6 +65,14 @@ struct SelectedPackage { has_build_script: bool, } +const VENDORED_GPU_INTEROP_PREFIXES: &[&str] = &[ + "third_party/cubecl-cuda-0.10.0-patched/", + "third_party/cubecl-runtime-0.10.0-patched/", + "third_party/wgpu-29.0.4-patched/", + "third_party/wgpu-core-29.0.4-patched/", + "third_party/wgpu-hal-29.0.4-patched/", +]; + pub(super) fn discover_source_roots( root: &Path, lane: CoverageLane, @@ -275,6 +286,14 @@ pub(super) fn classify_unreached_source(root: &Path, path: &str) -> Result Date: Tue, 21 Jul 2026 22:44:09 -0400 Subject: [PATCH 21/22] fix: audit vendored GPU coverage exclusions --- xtask/src/coverage/exclusion_policy.rs | 44 ++++++++ xtask/src/coverage/source_analysis.rs | 7 ++ .../src/coverage/source_analysis/workspace.rs | 12 +-- xtask/src/coverage/tests.rs | 75 +------------ xtask/src/coverage/tests/exclusion_policy.rs | 102 ++++++++++++++++++ .../line_ratchets.rs | 1 + .../source_analysis_tests.rs | 7 +- 7 files changed, 163 insertions(+), 85 deletions(-) create mode 100644 xtask/src/coverage/tests/exclusion_policy.rs diff --git a/xtask/src/coverage/exclusion_policy.rs b/xtask/src/coverage/exclusion_policy.rs index d8b42919..58cfb588 100644 --- a/xtask/src/coverage/exclusion_policy.rs +++ b/xtask/src/coverage/exclusion_policy.rs @@ -6,6 +6,8 @@ use std::path::Path; use syn::{Attribute, Item}; +use super::source_analysis::VENDORED_GPU_INTEROP_ROOTS; + use self::evidence_modules::collect_evidence_symbols_from_file; mod evidence_modules; @@ -76,6 +78,11 @@ const VENDORED_BLOCK_EVIDENCE: &[EvidenceTest] = &[ ), ]; +const VENDORED_GPU_INTEROP_EVIDENCE: &[EvidenceTest] = &[primary_evidence( + "xtask/tests/repo_lint_support/dependency_policy/path_patches.rs", + "all_workspace_path_patches_have_pinned_provenance_and_local_digests", +)]; + const fn primary_evidence(path: &'static str, name: &'static str) -> EvidenceTest { EvidenceTest { path, @@ -159,6 +166,15 @@ pub(super) const COVERAGE_EXCLUSIONS: &[CoverageExclusion] = &[ }, evidence: VENDORED_BLOCK_EVIDENCE, }, + CoverageExclusion { + id: "vendored-gpu-interop-patch", + reason: "the provenance-pinned GPU interop dependencies are outside workspace instrumentation and their exact patched trees are integrity-checked", + matcher: ExclusionMatcher::PathTrees { + roots: VENDORED_GPU_INTEROP_ROOTS, + suffix: ".rs", + }, + evidence: VENDORED_GPU_INTEROP_EVIDENCE, + }, ]; #[derive(Clone, Copy, Debug)] @@ -193,6 +209,10 @@ pub(super) enum ExclusionMatcher { excludes: Option<&'static str>, suffix: &'static str, }, + PathTrees { + roots: &'static [&'static str], + suffix: &'static str, + }, MarkerSpan { path: &'static str, start: &'static str, @@ -230,6 +250,9 @@ fn exclusion_matches( && contains.is_none_or(|needle| path.contains(needle)) && excludes.is_none_or(|needle| !path.contains(needle)) && path.ends_with(suffix)), + ExclusionMatcher::PathTrees { roots, suffix } => Ok(roots + .iter() + .any(|root| path.starts_with(root) && path.ends_with(suffix))), ExclusionMatcher::MarkerSpan { path: exact, start, @@ -471,6 +494,27 @@ fn validate_exclusion_matcher(root: &Path, exclusion: &CoverageExclusion) -> Res )); } } + ExclusionMatcher::PathTrees { roots, .. } => { + if roots.is_empty() { + return Err(format!( + "coverage exclusion `{}` path trees are empty", + exclusion.id + )); + } + for tree_root in roots { + let directory = root.join(tree_root); + let matched = directory.is_dir() + && collect_rust_files(&directory, root)? + .into_iter() + .any(|path| exclusion_matches(exclusion, &path, 1, &[]).unwrap_or(false)); + if !matched { + return Err(format!( + "coverage exclusion `{}` path tree `{tree_root}` matches no Rust file", + exclusion.id + )); + } + } + } ExclusionMatcher::MarkerSpan { path, .. } => { let source_path = root.join(path); let source = fs::read_to_string(&source_path).map_err(|err| { diff --git a/xtask/src/coverage/source_analysis.rs b/xtask/src/coverage/source_analysis.rs index c4402e16..c2c91514 100644 --- a/xtask/src/coverage/source_analysis.rs +++ b/xtask/src/coverage/source_analysis.rs @@ -33,6 +33,13 @@ use workspace::{ pub(super) const GENERATED_DWT_DISPOSITION: &str = "generated-codec-math-fragment"; pub(super) const VENDORED_BLOCK_DISPOSITION: &str = "vendored-block-ffi-binding"; pub(super) const VENDORED_GPU_INTEROP_DISPOSITION: &str = "vendored-gpu-interop-patch"; +pub(super) const VENDORED_GPU_INTEROP_ROOTS: &[&str] = &[ + "third_party/cubecl-cuda-0.10.0-patched/", + "third_party/cubecl-runtime-0.10.0-patched/", + "third_party/wgpu-29.0.4-patched/", + "third_party/wgpu-core-29.0.4-patched/", + "third_party/wgpu-hal-29.0.4-patched/", +]; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum SourceRole { diff --git a/xtask/src/coverage/source_analysis/workspace.rs b/xtask/src/coverage/source_analysis/workspace.rs index 5cab64b5..27dd6b5c 100644 --- a/xtask/src/coverage/source_analysis/workspace.rs +++ b/xtask/src/coverage/source_analysis/workspace.rs @@ -13,7 +13,7 @@ use super::cfg_eval::CoverageCfgContext; use super::graph::ReachKind; use super::{ SourceRole, GENERATED_DWT_DISPOSITION, VENDORED_BLOCK_DISPOSITION, - VENDORED_GPU_INTEROP_DISPOSITION, + VENDORED_GPU_INTEROP_DISPOSITION, VENDORED_GPU_INTEROP_ROOTS, }; mod fuzz_manifests; #[cfg(test)] @@ -65,14 +65,6 @@ struct SelectedPackage { has_build_script: bool, } -const VENDORED_GPU_INTEROP_PREFIXES: &[&str] = &[ - "third_party/cubecl-cuda-0.10.0-patched/", - "third_party/cubecl-runtime-0.10.0-patched/", - "third_party/wgpu-29.0.4-patched/", - "third_party/wgpu-core-29.0.4-patched/", - "third_party/wgpu-hal-29.0.4-patched/", -]; - pub(super) fn discover_source_roots( root: &Path, lane: CoverageLane, @@ -286,7 +278,7 @@ pub(super) fn classify_unreached_source(root: &Path, path: &str) -> Result>(); - let embedded_shader_line = lines - .iter() - .position(|line| line.contains("#include ")) - .expect("embedded shader marker") - + 1; - let included_shader_line = lines - .iter() - .position(|line| line.contains("include_str!(\"../store.metal\")")) - .expect("included shader marker") - + 1; - - assert_eq!( - matching_exclusion(path, embedded_shader_line, &lines) - .unwrap() - .map(|rule| rule.id), - Some("metal-embedded-shader-body") - ); - assert!(matching_exclusion(path, included_shader_line, &lines) - .unwrap() - .is_none()); -} - -#[test] -fn cuda_simt_exclusion_covers_split_device_modules_only() { - let split_device_module = - "crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/exports.rs"; - assert_eq!( - matching_exclusion(split_device_module, 1, &[]) - .unwrap() - .map(|rule| rule.id), - Some("cuda-simt-device-rust") - ); - - let host_module = "crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/src/main.rs"; - assert_eq!( - matching_exclusion(host_module, 1, &[]) - .unwrap() - .map(|rule| rule.id), - Some("cuda-generated-host-scaffold") - ); - assert!( - matching_exclusion("crates/j2k-cuda-runtime/src/j2k_encode.rs", 1, &[]) - .unwrap() - .is_none() - ); -} - -#[test] -fn exclusion_policy_maps_every_narrow_rule_to_existing_tests() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")).join(".."); - validate_exclusion_policy(&root).unwrap(); - assert!(COVERAGE_EXCLUSIONS - .iter() - .all(|rule| !rule.evidence.is_empty())); - assert!(!COVERAGE_EXCLUSIONS.iter().any(|rule| { - matches!( - rule.matcher, - ExclusionMatcher::WholeFile { - path: "crates/j2k-cuda/" | "crates/j2k-metal/" - } - ) - })); -} - #[test] fn coverage_cli_defaults_to_host_and_accepts_explicit_lanes() { let default = parse_options(std::iter::empty()).unwrap(); diff --git a/xtask/src/coverage/tests/exclusion_policy.rs b/xtask/src/coverage/tests/exclusion_policy.rs new file mode 100644 index 00000000..760df07b --- /dev/null +++ b/xtask/src/coverage/tests/exclusion_policy.rs @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::fs; +use std::path::Path; + +use super::super::exclusion_policy::{ + matching_exclusion, validate_exclusion_policy, ExclusionMatcher, COVERAGE_EXCLUSIONS, +}; + +#[test] +fn metal_raw_shader_span_is_narrower_than_the_host_source_file() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join(".."); + let path = "crates/j2k-metal/src/compute/shader_source.rs"; + let source = fs::read_to_string(root.join(path)).unwrap(); + let lines = source.lines().collect::>(); + let embedded_shader_line = lines + .iter() + .position(|line| line.contains("#include ")) + .expect("embedded shader marker") + + 1; + let included_shader_line = lines + .iter() + .position(|line| line.contains("include_str!(\"../store.metal\")")) + .expect("included shader marker") + + 1; + + assert_eq!( + matching_exclusion(path, embedded_shader_line, &lines) + .unwrap() + .map(|rule| rule.id), + Some("metal-embedded-shader-body") + ); + assert!(matching_exclusion(path, included_shader_line, &lines) + .unwrap() + .is_none()); +} + +#[test] +fn cuda_simt_exclusion_covers_split_device_modules_only() { + let split_device_module = + "crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/exports.rs"; + assert_eq!( + matching_exclusion(split_device_module, 1, &[]) + .unwrap() + .map(|rule| rule.id), + Some("cuda-simt-device-rust") + ); + + let host_module = "crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/src/main.rs"; + assert_eq!( + matching_exclusion(host_module, 1, &[]) + .unwrap() + .map(|rule| rule.id), + Some("cuda-generated-host-scaffold") + ); + assert!( + matching_exclusion("crates/j2k-cuda-runtime/src/j2k_encode.rs", 1, &[]) + .unwrap() + .is_none() + ); +} + +#[test] +fn vendored_gpu_interop_exclusion_covers_only_pinned_patch_roots() { + for path in [ + "third_party/cubecl-cuda-0.10.0-patched/build.rs", + "third_party/cubecl-runtime-0.10.0-patched/src/client.rs", + "third_party/wgpu-29.0.4-patched/src/api/buffer.rs", + "third_party/wgpu-core-29.0.4-patched/src/device/global.rs", + "third_party/wgpu-hal-29.0.4-patched/src/metal/mod.rs", + ] { + assert_eq!( + matching_exclusion(path, 1, &[]) + .unwrap() + .map(|rule| rule.id), + Some("vendored-gpu-interop-patch"), + "{path}" + ); + } + assert!( + matching_exclusion("third_party/unreviewed-0.1.0-patched/src/lib.rs", 1, &[]) + .unwrap() + .is_none() + ); +} + +#[test] +fn exclusion_policy_maps_every_narrow_rule_to_existing_tests() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join(".."); + validate_exclusion_policy(&root).unwrap(); + assert!(COVERAGE_EXCLUSIONS + .iter() + .all(|rule| !rule.evidence.is_empty())); + assert!(!COVERAGE_EXCLUSIONS.iter().any(|rule| { + matches!( + rule.matcher, + ExclusionMatcher::WholeFile { + path: "crates/j2k-cuda/" | "crates/j2k-metal/" + } + ) + })); +} diff --git a/xtask/tests/repo_lint_support/coverage_structure_policy/line_ratchets.rs b/xtask/tests/repo_lint_support/coverage_structure_policy/line_ratchets.rs index 015a4593..c45439e3 100644 --- a/xtask/tests/repo_lint_support/coverage_structure_policy/line_ratchets.rs +++ b/xtask/tests/repo_lint_support/coverage_structure_policy/line_ratchets.rs @@ -156,6 +156,7 @@ fn coverage_source_modules_stay_within_structural_ratchets() { ), ("xtask/src/coverage/tests/deferred_bodies.rs", 200), ("xtask/src/coverage/tests/evaluation.rs", 250), + ("xtask/src/coverage/tests/exclusion_policy.rs", 150), ("xtask/src/coverage/tests/evaluation/non_executable.rs", 100), ( "xtask/src/coverage/tests/evaluation/compiler_line_evidence.rs", diff --git a/xtask/tests/repo_lint_support/coverage_structure_policy/source_analysis_tests.rs b/xtask/tests/repo_lint_support/coverage_structure_policy/source_analysis_tests.rs index 01d59228..9e999058 100644 --- a/xtask/tests/repo_lint_support/coverage_structure_policy/source_analysis_tests.rs +++ b/xtask/tests/repo_lint_support/coverage_structure_policy/source_analysis_tests.rs @@ -169,6 +169,7 @@ fn coverage_source_analysis_regression_ownership_stays_explicit() { let tests = read("xtask/src/coverage/tests.rs"); let attributes = read("xtask/src/coverage/tests/attributes.rs"); let cfg_provenance = read("xtask/src/coverage/tests/cfg_provenance.rs"); + let exclusion_policy = read("xtask/src/coverage/tests/exclusion_policy.rs"); let source = read("xtask/src/coverage/tests/source_analysis.rs"); assert_pattern_checks(&[ @@ -176,12 +177,12 @@ fn coverage_source_analysis_regression_ownership_stays_explicit() { "mod attributes;", "mod cfg_provenance;", "mod deferred_bodies;", + "mod exclusion_policy;", "mod executable_evidence;", "fn parses_added_diff_hunks_without_counting_deletions()", "fn untracked_rust_sources_fail_the_local_coverage_preflight()", "fn lcov_parser_merges_duplicate_line_records_by_max_count()", "fn eighty_percent_changed_line_coverage_passes_exactly()", - "fn exclusion_policy_maps_every_narrow_rule_to_existing_tests()", "fn coverage_cli_defaults_to_host_and_accepts_explicit_lanes()", ]), PatternCheck::new("coverage attribute-disposition regressions", &attributes).required(&[ @@ -191,6 +192,10 @@ fn coverage_source_analysis_regression_ownership_stays_explicit() { ]), PatternCheck::new("coverage cfg provenance regressions", &cfg_provenance) .required(&["fn cfg_active_changed_source_cannot_evade_coverage_gate()"]), + PatternCheck::new("coverage exclusion regressions", &exclusion_policy).required(&[ + "fn exclusion_policy_maps_every_narrow_rule_to_existing_tests()", + "fn vendored_gpu_interop_exclusion_covers_only_pinned_patch_roots()", + ]), PatternCheck::new("coverage source-analysis regressions", &source).required(&[ "fn body_bearing_function_forms_have_item_and_body_spans()", "fn nested_inline_module_uses_its_real_module_directory()", From 65c98eea724ebf512857f31cbe05f53398904526 Mon Sep 17 00:00:00 2001 From: GF Date: Tue, 21 Jul 2026 23:03:42 -0400 Subject: [PATCH 22/22] fix: preserve binary codec fixtures on Windows --- .gitattributes | 8 ++++++++ .../decoder_fixture_policy.rs | 1 + .../binary_attributes.rs | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 .gitattributes create mode 100644 xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy/binary_attributes.rs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f5550762 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +*.ppm binary +*.pgm binary +*.raw binary +*.gray binary +*.j2k binary +*.j2c binary +*.jp2 binary +*.jph binary diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy.rs index 20feb23b..e5cd8f3f 100644 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy.rs +++ b/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy.rs @@ -4,6 +4,7 @@ use std::fs; use crate::repo_lint_support::{assert_pattern_checks, read_source_files, repo_root, PatternCheck}; +mod binary_attributes; mod shared_codec; #[test] diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy/binary_attributes.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy/binary_attributes.rs new file mode 100644 index 00000000..85cf2a3f --- /dev/null +++ b/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy/binary_attributes.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::fs; + +use crate::repo_lint_support::repo_root; + +#[test] +fn binary_codec_fixtures_disable_git_text_conversion() { + let attributes = fs::read_to_string(repo_root().join(".gitattributes")) + .expect("read repository Git attributes"); + + for extension in ["ppm", "pgm", "raw", "gray", "j2k", "j2c", "jp2", "jph"] { + let rule = format!("*.{extension} binary"); + assert!( + attributes.lines().any(|line| line.trim() == rule), + "{rule} must disable text conversion for binary codec fixtures" + ); + } +}

(filename: P) -> Result + where + P: AsRef, + { + unsafe { libloading::Library::new(filename) }.map(|inner| Self { inner }) + } + + unsafe fn get( + &self, + symbol: &[u8], + ) -> Result, crate::DeviceError> { + unsafe { self.inner.get(symbol) }.map_err(|e| match e { + libloading::Error::GetProcAddress { .. } | libloading::Error::GetProcAddressUnknown => { + crate::DeviceError::Unexpected + } + libloading::Error::IncompatibleSize + | libloading::Error::CreateCString { .. } + | libloading::Error::CreateCStringWithTrailing { .. } => crate::hal_internal_error(e), + _ => crate::DeviceError::Unexpected, // could be unreachable!() but we prefer to be more robust + }) + } +} + +#[derive(Debug)] +struct D3D12Lib { + lib: DynLib, +} + +#[derive(Clone, Copy)] +pub enum CreateDeviceError { + GetProcAddress, + D3D12CreateDevice(windows_core::HRESULT), + RetDeviceIsNull, +} + +impl D3D12Lib { + fn new() -> Result { + unsafe { DynLib::new("d3d12.dll").map(|lib| Self { lib }) } + } + + fn create_device( + &self, + adapter: &DxgiAdapter, + feature_level: Direct3D::D3D_FEATURE_LEVEL, + ) -> Result { + // Calls windows::Win32::Graphics::Direct3D12::D3D12CreateDevice on d3d12.dll + type Fun = extern "system" fn( + padapter: *mut ffi::c_void, + minimumfeaturelevel: Direct3D::D3D_FEATURE_LEVEL, + riid: *const windows_core::GUID, + ppdevice: *mut *mut ffi::c_void, + ) -> windows_core::HRESULT; + let func: libloading::Symbol = + unsafe { self.lib.get(c"D3D12CreateDevice".to_bytes()) } + .map_err(|_| CreateDeviceError::GetProcAddress)?; + + let mut result__: Option = None; + + let res = (func)( + adapter.as_raw(), + feature_level, + // TODO: Generic? + &Direct3D12::ID3D12Device::IID, + <*mut _>::cast(&mut result__), + ); + + if res.is_err() { + return Err(CreateDeviceError::D3D12CreateDevice(res)); + } + + result__.ok_or(CreateDeviceError::RetDeviceIsNull) + } + + fn serialize_root_signature( + &self, + version: Direct3D12::D3D_ROOT_SIGNATURE_VERSION, + parameters: &[Direct3D12::D3D12_ROOT_PARAMETER], + static_samplers: &[Direct3D12::D3D12_STATIC_SAMPLER_DESC], + flags: Direct3D12::D3D12_ROOT_SIGNATURE_FLAGS, + ) -> Result { + // Calls windows::Win32::Graphics::Direct3D12::D3D12SerializeRootSignature on d3d12.dll + type Fun = extern "system" fn( + prootsignature: *const Direct3D12::D3D12_ROOT_SIGNATURE_DESC, + version: Direct3D12::D3D_ROOT_SIGNATURE_VERSION, + ppblob: *mut *mut ffi::c_void, + pperrorblob: *mut *mut ffi::c_void, + ) -> windows_core::HRESULT; + let func: libloading::Symbol = + unsafe { self.lib.get(c"D3D12SerializeRootSignature".to_bytes()) }?; + + let desc = Direct3D12::D3D12_ROOT_SIGNATURE_DESC { + NumParameters: parameters.len() as _, + pParameters: parameters.as_ptr(), + NumStaticSamplers: static_samplers.len() as _, + pStaticSamplers: static_samplers.as_ptr(), + Flags: flags, + }; + + let mut blob = None; + let mut error = None::; + (func)( + &desc, + version, + <*mut _>::cast(&mut blob), + <*mut _>::cast(&mut error), + ) + .ok() + .into_device_result("Root signature serialization")?; + + if let Some(error) = error { + let error = D3DBlob(error); + log::error!( + "Root signature serialization error: {:?}", + unsafe { error.as_c_str() }.unwrap().to_str().unwrap() + ); + return Err(crate::DeviceError::Unexpected); // could be hal_usage_error or hal_internal_error + } + + blob.ok_or(crate::DeviceError::Unexpected) + } + + fn debug_interface(&self) -> Result, crate::DeviceError> { + // Calls windows::Win32::Graphics::Direct3D12::D3D12GetDebugInterface on d3d12.dll + type Fun = extern "system" fn( + riid: *const windows_core::GUID, + ppvdebug: *mut *mut ffi::c_void, + ) -> windows_core::HRESULT; + let func: libloading::Symbol = + unsafe { self.lib.get(c"D3D12GetDebugInterface".to_bytes()) }?; + + let mut result__ = None; + + let res = (func)(&Direct3D12::ID3D12Debug::IID, <*mut _>::cast(&mut result__)).ok(); + + if let Err(ref err) = res { + match err.code() { + Dxgi::DXGI_ERROR_SDK_COMPONENT_MISSING => return Ok(None), + _ => {} + } + } + + res.into_device_result("GetDebugInterface")?; + + result__.ok_or(crate::DeviceError::Unexpected).map(Some) + } + + /// Calls D3D12GetInterface to obtain a COM interface by CLSID and IID. + /// + /// This is used by the Independent Devices API to obtain `ID3D12SDKConfiguration1`. + fn get_interface( + &self, + clsid: &windows_core::GUID, + ) -> Result { + // Calls windows::Win32::Graphics::Direct3D12::D3D12GetInterface on d3d12.dll + type Fun = extern "system" fn( + rclsid: *const windows_core::GUID, + riid: *const windows_core::GUID, + ppvdebug: *mut *mut ffi::c_void, + ) -> windows_core::HRESULT; + let func: libloading::Symbol = + unsafe { self.lib.get(c"D3D12GetInterface".to_bytes()) } + .map_err(|_| GetInterfaceError::GetProcAddress)?; + + let mut result__: Option = None; + + let res = (func)(clsid, &T::IID, <*mut _>::cast(&mut result__)); + + if res.is_err() { + return Err(GetInterfaceError::D3D12GetInterface(res)); + } + + result__.ok_or(GetInterfaceError::RetIsNull) + } +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum GetInterfaceError { + GetProcAddress, + D3D12GetInterface(windows_core::HRESULT), + RetIsNull, +} + +impl fmt::Display for GetInterfaceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::GetProcAddress => write!(f, "D3D12GetInterface not found in d3d12.dll"), + Self::D3D12GetInterface(hr) => write!(f, "D3D12GetInterface failed: {hr}"), + Self::RetIsNull => write!(f, "D3D12GetInterface returned null"), + } + } +} + +impl core::error::Error for GetInterfaceError {} + +#[derive(Debug)] +pub(super) struct DxgiLib { + lib: DynLib, +} + +impl DxgiLib { + pub fn new() -> Result { + unsafe { DynLib::new("dxgi.dll").map(|lib| Self { lib }) } + } + + /// Will error with crate::DeviceError::Unexpected if DXGI 1.3 is not available. + pub fn debug_interface1(&self) -> Result, crate::DeviceError> { + // Calls windows::Win32::Graphics::Dxgi::DXGIGetDebugInterface1 on dxgi.dll + type Fun = extern "system" fn( + flags: u32, + riid: *const windows_core::GUID, + pdebug: *mut *mut ffi::c_void, + ) -> windows_core::HRESULT; + let func: libloading::Symbol = + unsafe { self.lib.get(c"DXGIGetDebugInterface1".to_bytes()) }?; + + let mut result__ = None; + + let res = (func)(0, &Dxgi::IDXGIInfoQueue::IID, <*mut _>::cast(&mut result__)).ok(); + + if let Err(ref err) = res { + match err.code() { + Dxgi::DXGI_ERROR_SDK_COMPONENT_MISSING => return Ok(None), + _ => {} + } + } + + res.into_device_result("debug_interface1")?; + + result__.ok_or(crate::DeviceError::Unexpected).map(Some) + } + + /// Will error with crate::DeviceError::Unexpected if DXGI 1.4 is not available. + pub fn create_factory4( + &self, + factory_flags: Dxgi::DXGI_CREATE_FACTORY_FLAGS, + ) -> Result { + // Calls windows::Win32::Graphics::Dxgi::CreateDXGIFactory2 on dxgi.dll + type Fun = extern "system" fn( + flags: Dxgi::DXGI_CREATE_FACTORY_FLAGS, + riid: *const windows_core::GUID, + ppfactory: *mut *mut ffi::c_void, + ) -> windows_core::HRESULT; + let func: libloading::Symbol = + unsafe { self.lib.get(c"CreateDXGIFactory2".to_bytes()) }?; + + let mut result__ = None; + + (func)( + factory_flags, + &Dxgi::IDXGIFactory4::IID, + <*mut _>::cast(&mut result__), + ) + .ok() + .into_device_result("create_factory4")?; + + result__.ok_or(crate::DeviceError::Unexpected) + } + + /// Will error with crate::DeviceError::Unexpected if DXGI 1.3 is not available. + pub fn create_factory_media(&self) -> Result { + // Calls windows::Win32::Graphics::Dxgi::CreateDXGIFactory1 on dxgi.dll + type Fun = extern "system" fn( + riid: *const windows_core::GUID, + ppfactory: *mut *mut ffi::c_void, + ) -> windows_core::HRESULT; + let func: libloading::Symbol = + unsafe { self.lib.get(c"CreateDXGIFactory1".to_bytes()) }?; + + let mut result__ = None; + + // https://learn.microsoft.com/en-us/windows/win32/api/dxgi1_3/nn-dxgi1_3-idxgifactorymedia + (func)(&Dxgi::IDXGIFactoryMedia::IID, <*mut _>::cast(&mut result__)) + .ok() + .into_device_result("create_factory_media")?; + + result__.ok_or(crate::DeviceError::Unexpected) + } +} + +/// Create a temporary "owned" copy inside a [`mem::ManuallyDrop`] without increasing the refcount or +/// moving away the source variable. +/// +/// This is a common pattern when needing to pass interface pointers ("borrows") into Windows +/// structs. Moving/cloning ownership is impossible/inconvenient because: +/// +/// - The caller does _not_ assume ownership (and decrement the refcount at a later time); +/// - Unnecessarily increasing and decrementing the refcount; +/// - [`Drop`] destructors cannot run inside `union` structures (when the created structure is +/// implicitly dropped after a call). +/// +/// See also and +/// . +/// +/// # Safety +/// Performs a [`mem::transmute_copy()`] on a refcounted [`Interface`] type. The returned +/// [`mem::ManuallyDrop`] should _not_ be dropped. +pub unsafe fn borrow_interface_temporarily(src: &I) -> mem::ManuallyDrop> { + unsafe { mem::transmute_copy(src) } +} + +/// See [`borrow_interface_temporarily()`] +pub unsafe fn borrow_optional_interface_temporarily( + src: &Option, +) -> mem::ManuallyDrop> { + unsafe { mem::transmute_copy(src) } +} + +struct D3DBlob(Direct3D::ID3DBlob); + +impl Deref for D3DBlob { + type Target = Direct3D::ID3DBlob; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl D3DBlob { + unsafe fn as_slice(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.GetBufferPointer().cast(), self.GetBufferSize()) } + } + + unsafe fn as_c_str(&self) -> Result<&ffi::CStr, ffi::FromBytesUntilNulError> { + ffi::CStr::from_bytes_until_nul(unsafe { self.as_slice() }) + } +} + +#[derive(Clone, Debug)] +pub struct Api; + +impl crate::Api for Api { + const VARIANT: wgt::Backend = wgt::Backend::Dx12; + + type Instance = Instance; + type Surface = Surface; + type Adapter = Adapter; + type Device = Device; + + type Queue = Queue; + type CommandEncoder = CommandEncoder; + type CommandBuffer = CommandBuffer; + + type Buffer = Buffer; + type Texture = Texture; + type SurfaceTexture = Texture; + type TextureView = TextureView; + type Sampler = Sampler; + type QuerySet = QuerySet; + type Fence = Fence; + + type BindGroupLayout = BindGroupLayout; + type BindGroup = BindGroup; + type PipelineLayout = PipelineLayout; + type ShaderModule = ShaderModule; + type RenderPipeline = RenderPipeline; + type ComputePipeline = ComputePipeline; + type PipelineCache = PipelineCache; + + type AccelerationStructure = AccelerationStructure; +} + +crate::impl_dyn_resource!( + Adapter, + AccelerationStructure, + BindGroup, + BindGroupLayout, + Buffer, + CommandBuffer, + CommandEncoder, + ComputePipeline, + Device, + Fence, + Instance, + PipelineCache, + PipelineLayout, + QuerySet, + Queue, + RenderPipeline, + Sampler, + ShaderModule, + Surface, + Texture, + TextureView +); + +// Limited by D3D12's root signature size of 64. Each element takes 1 or 2 entries. +const MAX_ROOT_ELEMENTS: usize = 64; +const ZERO_BUFFER_SIZE: wgt::BufferAddress = 256 << 10; + +pub struct Instance { + factory: DxgiFactory, + factory_media: Option, + // `device_factory` must be dropped before `library` because the COM + // object's Release call goes through the d3d12.dll vtable. If + // `library` (which unloads d3d12.dll) is dropped first the Release + // segfaults. + device_factory: Arc, + library: Arc, + dcomp_lib: Arc, + supports_allow_tearing: bool, + presentation_system: wgt::Dx12SwapchainKind, + _lib_dxgi: DxgiLib, + flags: wgt::InstanceFlags, + memory_budget_thresholds: wgt::MemoryBudgetThresholds, + compiler_container: Arc, + options: wgt::Dx12BackendOptions, + telemetry: Option, +} + +impl Instance { + /// Get the raw DXGI factory associated with this instance. + pub unsafe fn raw_factory4(&self) -> &Dxgi::IDXGIFactory4 { + self.factory.deref() + } + + pub unsafe fn create_surface_from_visual(&self, visual: *mut ffi::c_void) -> Surface { + let visual = unsafe { DirectComposition::IDCompositionVisual::from_raw_borrowed(&visual) } + .expect("COM pointer should not be NULL"); + Surface { + factory: self.factory.clone(), + factory_media: self.factory_media.clone(), + target: SurfaceTarget::Visual(visual.to_owned()), + supports_allow_tearing: self.supports_allow_tearing, + swap_chain: RwLock::new(None), + options: self.options.clone(), + } + } + + pub unsafe fn create_surface_from_surface_handle( + &self, + surface_handle: *mut ffi::c_void, + ) -> Surface { + // TODO: We're not given ownership, so we shouldn't call HANDLE::free(). This puts an extra burden on the caller to keep it alive. + // https://learn.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-duplicatehandle could help us, even though DirectComposition is not in the list? + // Or we make all these types owned, require an ownership transition, and replace SurfaceTargetUnsafe with SurfaceTarget. + let surface_handle = Foundation::HANDLE(surface_handle); + Surface { + factory: self.factory.clone(), + factory_media: self.factory_media.clone(), + target: SurfaceTarget::SurfaceHandle(surface_handle), + supports_allow_tearing: self.supports_allow_tearing, + swap_chain: RwLock::new(None), + options: self.options.clone(), + } + } + + pub unsafe fn create_surface_from_swap_chain_panel( + &self, + swap_chain_panel: *mut ffi::c_void, + ) -> Surface { + let swap_chain_panel = + unsafe { types::ISwapChainPanelNative::from_raw_borrowed(&swap_chain_panel) } + .expect("COM pointer should not be NULL"); + Surface { + factory: self.factory.clone(), + factory_media: self.factory_media.clone(), + target: SurfaceTarget::SwapChainPanel(swap_chain_panel.to_owned()), + supports_allow_tearing: self.supports_allow_tearing, + swap_chain: RwLock::new(None), + options: self.options.clone(), + } + } +} + +unsafe impl Send for Instance {} +unsafe impl Sync for Instance {} + +struct SwapChain { + // TODO: Drop order frees the SWC before the raw image pointers...? + raw: Dxgi::IDXGISwapChain3, + // need to associate raw image pointers with the swapchain so they can be properly released + // when the swapchain is destroyed + resources: Vec, + /// Handle is freed in [`Self::release_resources()`] + waitable: Option, + acquired_count: usize, + present_mode: wgt::PresentMode, + format: wgt::TextureFormat, + size: wgt::Extent3d, +} + +enum SurfaceTarget { + /// Borrowed, lifetime externally managed + WndHandle(Foundation::HWND), + /// `handle` is borrowed, lifetime externally managed + VisualFromWndHandle { + handle: Foundation::HWND, + dcomp_state: Mutex, + }, + Visual(DirectComposition::IDCompositionVisual), + /// Borrowed, lifetime externally managed + SurfaceHandle(Foundation::HANDLE), + SwapChainPanel(types::ISwapChainPanelNative), +} + +pub struct Surface { + factory: DxgiFactory, + factory_media: Option, + target: SurfaceTarget, + supports_allow_tearing: bool, + swap_chain: RwLock>, + options: wgt::Dx12BackendOptions, +} + +unsafe impl Send for Surface {} +unsafe impl Sync for Surface {} + +impl Surface { + pub fn swap_chain(&self) -> Option { + Some(self.swap_chain.read().as_ref()?.raw.clone()) + } + + /// Returns the waitable handle associated with this swap chain, if any. + /// Handle is only valid while the swap chain is alive. + pub unsafe fn waitable_handle(&self) -> Option { + self.swap_chain.read().as_ref()?.waitable + } +} + +#[derive(Debug, Clone, Copy)] +enum MemoryArchitecture { + Unified { + #[allow(unused)] + cache_coherent: bool, + }, + NonUnified, +} + +#[derive(Debug, Clone, Copy)] +struct PrivateCapabilities { + instance_flags: wgt::InstanceFlags, + workarounds: Workarounds, + #[allow(unused)] + heterogeneous_resource_heaps: bool, + memory_architecture: MemoryArchitecture, + heap_create_not_zeroed: bool, + casting_fully_typed_format_supported: bool, + suballocation_supported: bool, + shader_model: naga::back::hlsl::ShaderModel, + max_sampler_descriptor_heap_size: u32, + unrestricted_buffer_texture_copy_pitch_supported: bool, +} + +impl PrivateCapabilities { + fn texture_data_placement_alignment(&self) -> u64 { + if self.unrestricted_buffer_texture_copy_pitch_supported { + 4 + } else { + D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT.into() + } + } +} + +#[derive(Default, Debug, Copy, Clone)] +struct Workarounds { + // On WARP 1.0.13+, debug information in shaders in certain situations causes the device + // to hang. https://github.com/gfx-rs/wgpu/issues/8368 + avoid_shader_debug_info: bool, +} + +pub struct Adapter { + raw: DxgiAdapter, + device: Direct3D12::ID3D12Device, + library: Arc, + dcomp_lib: Arc, + private_caps: PrivateCapabilities, + presentation_timer: auxil::dxgi::time::PresentationTimer, + memory_budget_thresholds: wgt::MemoryBudgetThresholds, + compiler_container: Arc, + options: wgt::Dx12BackendOptions, +} + +unsafe impl Send for Adapter {} +unsafe impl Sync for Adapter {} + +impl Adapter { + pub fn as_raw(&self) -> &Dxgi::IDXGIAdapter3 { + &self.raw + } +} + +struct Event(pub Foundation::HANDLE); +impl Event { + pub fn create(manual_reset: bool, initial_state: bool) -> Result { + Ok(Self( + unsafe { Threading::CreateEventA(None, manual_reset, initial_state, None) } + .into_device_result("CreateEventA")?, + )) + } +} + +impl Drop for Event { + fn drop(&mut self) { + unsafe { Foundation::HANDLE::free(&mut self.0) } + } +} + +/// Helper structure for waiting for GPU. +struct Idler { + fence: Direct3D12::ID3D12Fence, +} + +#[derive(Debug, Clone)] +struct CommandSignatures { + draw: Direct3D12::ID3D12CommandSignature, + draw_indexed: Direct3D12::ID3D12CommandSignature, + draw_mesh: Option, + dispatch: Direct3D12::ID3D12CommandSignature, +} + +struct DeviceShared { + adapter: DxgiAdapter, + zero_buffer: Direct3D12::ID3D12Resource, + cmd_signatures: CommandSignatures, + heap_views: descriptor::GeneralHeap, + sampler_heap: sampler::SamplerHeap, + private_caps: PrivateCapabilities, +} + +unsafe impl Send for DeviceShared {} +unsafe impl Sync for DeviceShared {} + +pub struct Device { + raw: Direct3D12::ID3D12Device, + present_queue: Direct3D12::ID3D12CommandQueue, + idler: Idler, + features: wgt::Features, + shared: Arc, + options: wgt::Dx12BackendOptions, + // CPU only pools + rtv_pool: Arc>, + dsv_pool: Mutex, + srv_uav_pool: Mutex, + // library + library: Arc, + dcomp_lib: Arc, + #[cfg(feature = "renderdoc")] + render_doc: auxil::renderdoc::RenderDoc, + null_rtv_handle: descriptor::Handle, + mem_allocator: Allocator, + compiler_container: Arc, + shader_cache: Mutex, + counters: Arc, +} + +impl Drop for Device { + fn drop(&mut self) { + self.rtv_pool.lock().free_handle(self.null_rtv_handle); + if self + .shared + .private_caps + .instance_flags + .contains(wgt::InstanceFlags::VALIDATION) + { + auxil::dxgi::exception::unregister_exception_handler(); + } + } +} + +unsafe impl Send for Device {} +unsafe impl Sync for Device {} + +pub struct Queue { + raw: Direct3D12::ID3D12CommandQueue, + temp_lists: Mutex>>, +} + +impl Queue { + pub fn as_raw(&self) -> &Direct3D12::ID3D12CommandQueue { + &self.raw + } +} + +unsafe impl Send for Queue {} +unsafe impl Sync for Queue {} + +#[derive(Default)] +struct Temp { + marker: Vec, + barriers: Vec, +} + +impl Temp { + fn clear(&mut self) { + self.marker.clear(); + self.barriers.clear(); + } +} + +struct PassResolve { + src: (Direct3D12::ID3D12Resource, u32), + dst: (Direct3D12::ID3D12Resource, u32), + format: Dxgi::Common::DXGI_FORMAT, +} + +#[derive(Clone, Copy, Debug)] +enum RootElement { + Empty, + Constant, + SpecialConstantBuffer { + /// The first vertex in an indirect draw call, _or_ the `x` of a compute dispatch. + first_vertex: i32, + /// The first instance in an indirect draw call, _or_ the `y` of a compute dispatch. + first_instance: u32, + /// Unused in an indirect draw call, _or_ the `z` of a compute dispatch. + other: u32, + }, + /// Descriptor table. + Table(Direct3D12::D3D12_GPU_DESCRIPTOR_HANDLE), + /// Descriptor for an uniform buffer that has dynamic offset. + DynamicUniformBuffer { + address: Direct3D12::D3D12_GPU_DESCRIPTOR_HANDLE, + }, + /// Descriptor table referring to the entire sampler heap. + SamplerHeap, + /// Root constants for dynamic offsets. + /// + /// start..end is the range of values in [`PassState::dynamic_storage_buffer_offsets`] + /// that will be used to update the root constants. + DynamicOffsetsBuffer { + start: usize, + end: usize, + }, +} + +#[derive(Clone, Copy)] +enum PassKind { + Render, + Compute, + Transfer, +} + +struct PassState { + has_label: bool, + resolves: ArrayVec, + layout: PipelineLayoutShared, + root_elements: [RootElement; MAX_ROOT_ELEMENTS], + constant_data: [u32; MAX_ROOT_ELEMENTS], + dynamic_storage_buffer_offsets: Vec, + dirty_root_elements: u64, + vertex_buffers: [Direct3D12::D3D12_VERTEX_BUFFER_VIEW; crate::MAX_VERTEX_BUFFERS], + dirty_vertex_buffers: usize, + kind: PassKind, +} + +#[test] +fn test_dirty_mask() { + assert_eq!(MAX_ROOT_ELEMENTS, u64::BITS as usize); +} + +impl PassState { + fn new() -> Self { + PassState { + has_label: false, + resolves: ArrayVec::new(), + layout: PipelineLayoutShared { + signature: None, + total_root_elements: 0, + special_constants: None, + root_constant_info: None, + sampler_heap_root_index: None, + }, + root_elements: [RootElement::Empty; MAX_ROOT_ELEMENTS], + constant_data: [0; MAX_ROOT_ELEMENTS], + dynamic_storage_buffer_offsets: Vec::new(), + dirty_root_elements: 0, + vertex_buffers: [Default::default(); crate::MAX_VERTEX_BUFFERS], + dirty_vertex_buffers: 0, + kind: PassKind::Transfer, + } + } + + fn clear(&mut self) { + // careful about heap allocations! + *self = Self::new(); + } +} + +pub struct CommandEncoder { + allocator: Direct3D12::ID3D12CommandAllocator, + device: Direct3D12::ID3D12Device, + shared: Arc, + mem_allocator: Allocator, + + rtv_pool: Arc>, + temp_rtv_handles: Vec, + + intermediate_copy_bufs: Vec, + + null_rtv_handle: descriptor::Handle, + list: Option, + free_lists: Vec, + pass: PassState, + temp: Temp, + + /// If set, the end of the next render/compute pass will write a timestamp at + /// the given pool & location. + end_of_pass_timer_query: Option<(Direct3D12::ID3D12QueryHeap, u32)>, + + counters: Arc, +} + +unsafe impl Send for CommandEncoder {} +unsafe impl Sync for CommandEncoder {} + +impl fmt::Debug for CommandEncoder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CommandEncoder") + .field("allocator", &self.allocator) + .field("device", &self.allocator) + .finish() + } +} + +#[derive(Debug)] +pub struct CommandBuffer { + raw: Direct3D12::ID3D12GraphicsCommandList, +} + +impl crate::DynCommandBuffer for CommandBuffer {} + +unsafe impl Send for CommandBuffer {} +unsafe impl Sync for CommandBuffer {} + +#[derive(Debug)] +pub struct Buffer { + resource: Direct3D12::ID3D12Resource, + // While the allocation also has _a_ size, it may not + // be the same as the original size of the buffer, + // as the allocation size varies for assorted reasons. + size: wgt::BufferAddress, + allocation: suballocation::Allocation, +} + +unsafe impl Send for Buffer {} +unsafe impl Sync for Buffer {} + +impl crate::DynBuffer for Buffer {} + +impl crate::BufferBinding<'_, Buffer> { + fn resolve_size(&self) -> wgt::BufferAddress { + match self.size { + Some(size) => size.get(), + None => self.buffer.size - self.offset, + } + } + + // TODO: Return GPU handle directly? + fn resolve_address(&self) -> wgt::BufferAddress { + (unsafe { self.buffer.resource.GetGPUVirtualAddress() }) + self.offset + } +} + +#[derive(Debug)] +pub struct Texture { + resource: Direct3D12::ID3D12Resource, + format: wgt::TextureFormat, + dimension: wgt::TextureDimension, + size: wgt::Extent3d, + mip_level_count: u32, + sample_count: u32, + allocation: suballocation::Allocation, +} + +impl Texture { + pub unsafe fn raw_resource(&self) -> &Direct3D12::ID3D12Resource { + &self.resource + } +} + +impl crate::DynTexture for Texture {} +impl crate::DynSurfaceTexture for Texture {} + +impl core::borrow::Borrow for Texture { + fn borrow(&self) -> &dyn crate::DynTexture { + self + } +} + +unsafe impl Send for Texture {} +unsafe impl Sync for Texture {} + +impl Texture { + fn array_layer_count(&self) -> u32 { + match self.dimension { + wgt::TextureDimension::D1 | wgt::TextureDimension::D3 => 1, + wgt::TextureDimension::D2 => self.size.depth_or_array_layers, + } + } + + /// see + fn calc_subresource(&self, mip_level: u32, array_layer: u32, plane: u32) -> u32 { + mip_level + (array_layer + plane * self.array_layer_count()) * self.mip_level_count + } + + fn calc_subresource_for_copy(&self, base: &crate::TextureCopyBase) -> u32 { + let plane = match base.aspect { + crate::FormatAspects::COLOR + | crate::FormatAspects::DEPTH + | crate::FormatAspects::PLANE_0 => 0, + crate::FormatAspects::STENCIL | crate::FormatAspects::PLANE_1 => 1, + crate::FormatAspects::PLANE_2 => 2, + _ => unreachable!(), + }; + self.calc_subresource(base.mip_level, base.array_layer, plane) + } +} + +#[derive(Debug)] +pub struct TextureView { + raw_format: Dxgi::Common::DXGI_FORMAT, + aspects: crate::FormatAspects, + dimension: wgt::TextureViewDimension, + texture: Direct3D12::ID3D12Resource, + subresource_index: u32, + mip_slice: u32, + handle_srv: Option, + handle_uav: Option, + handle_rtv: Option, + handle_dsv_ro: Option, + handle_dsv_rw: Option, +} + +impl crate::DynTextureView for TextureView {} + +unsafe impl Send for TextureView {} +unsafe impl Sync for TextureView {} + +#[derive(Debug)] +pub struct Sampler { + index: sampler::SamplerIndex, + desc: Direct3D12::D3D12_SAMPLER_DESC, +} + +impl crate::DynSampler for Sampler {} + +unsafe impl Send for Sampler {} +unsafe impl Sync for Sampler {} + +#[derive(Debug)] +pub struct QuerySet { + raw: Direct3D12::ID3D12QueryHeap, + raw_ty: Direct3D12::D3D12_QUERY_TYPE, +} + +impl crate::DynQuerySet for QuerySet {} + +unsafe impl Send for QuerySet {} +unsafe impl Sync for QuerySet {} + +#[derive(Debug)] +pub struct Fence { + raw: Direct3D12::ID3D12Fence, +} + +impl crate::DynFence for Fence {} + +unsafe impl Send for Fence {} +unsafe impl Sync for Fence {} + +impl Fence { + pub fn raw_fence(&self) -> &Direct3D12::ID3D12Fence { + &self.raw + } +} + +#[derive(Debug)] +pub struct BindGroupLayout { + /// Sorted list of entries. + entries: Vec, + cpu_heap_views: Option, + copy_counts: Vec, // all 1's +} + +impl crate::DynBindGroupLayout for BindGroupLayout {} + +#[derive(Debug, Clone, Copy)] +enum DynamicBuffer { + Uniform(Direct3D12::D3D12_GPU_DESCRIPTOR_HANDLE), + Storage, +} + +#[derive(Debug)] +struct SamplerIndexBuffer { + buffer: Direct3D12::ID3D12Resource, + allocation: suballocation::Allocation, +} + +#[derive(Debug)] +pub struct BindGroup { + handle_views: Option, + sampler_index_buffer: Option, + dynamic_buffers: Vec, +} + +impl crate::DynBindGroup for BindGroup {} + +bitflags::bitflags! { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + struct TableTypes: u8 { + const SRV_CBV_UAV = 1 << 0; + const SAMPLERS = 1 << 1; + } +} + +// Element (also known as parameter) index into the root signature. +type RootIndex = u32; + +#[derive(Debug)] +struct BindGroupInfo { + base_root_index: RootIndex, + tables: TableTypes, + dynamic_storage_buffer_offsets: Option, +} + +#[derive(Debug, Clone)] +struct RootConstantInfo { + root_index: RootIndex, + range: core::ops::Range, +} + +#[derive(Debug, Clone)] +struct DynamicStorageBufferOffsets { + root_index: RootIndex, + range: core::ops::Range, +} + +#[derive(Debug, Clone)] +struct PipelineLayoutShared { + signature: Option, + total_root_elements: RootIndex, + special_constants: Option, + root_constant_info: Option, + sampler_heap_root_index: Option, +} + +unsafe impl Send for PipelineLayoutShared {} +unsafe impl Sync for PipelineLayoutShared {} + +#[derive(Debug, Clone)] +struct PipelineLayoutSpecialConstants { + root_index: RootIndex, + indirect_cmd_signatures: Option, +} + +unsafe impl Send for PipelineLayoutSpecialConstants {} +unsafe impl Sync for PipelineLayoutSpecialConstants {} + +#[derive(Debug)] +pub struct PipelineLayout { + shared: PipelineLayoutShared, + // Storing for each associated bind group, which tables we created + // in the root signature. This is required for binding descriptor sets. + bind_group_infos: [Option; crate::MAX_BIND_GROUPS], + naga_options: naga::back::hlsl::Options, +} + +impl crate::DynPipelineLayout for PipelineLayout {} + +#[derive(Debug)] +pub struct ShaderModule { + source: ShaderModuleSource, + raw_name: Option, + runtime_checks: wgt::ShaderRuntimeChecks, +} + +impl crate::DynShaderModule for ShaderModule {} + +#[derive(Default)] +pub struct ShaderCache { + nr_of_shaders_compiled: u32, + entries: HashMap, +} + +#[derive(PartialEq, Eq, Hash)] +pub(super) struct ShaderCacheKey { + source: String, + entry_point: String, + stage: naga::ShaderStage, + shader_model: naga::back::hlsl::ShaderModel, +} + +pub(super) struct ShaderCacheValue { + /// This is the value of [`ShaderCache::nr_of_shaders_compiled`] + /// at the time the cache entry was last used. + last_used: u32, + shader: CompiledShader, +} + +#[derive(Clone)] +pub(super) enum CompiledShader { + Dxc(Direct3D::Dxc::IDxcBlob), + Fxc(Direct3D::ID3DBlob), + Precompiled(Vec), +} + +impl CompiledShader { + fn create_native_shader(&self) -> Direct3D12::D3D12_SHADER_BYTECODE { + match self { + CompiledShader::Dxc(shader) => Direct3D12::D3D12_SHADER_BYTECODE { + pShaderBytecode: unsafe { shader.GetBufferPointer() }, + BytecodeLength: unsafe { shader.GetBufferSize() }, + }, + CompiledShader::Fxc(shader) => Direct3D12::D3D12_SHADER_BYTECODE { + pShaderBytecode: unsafe { shader.GetBufferPointer() }, + BytecodeLength: unsafe { shader.GetBufferSize() }, + }, + CompiledShader::Precompiled(shader) => Direct3D12::D3D12_SHADER_BYTECODE { + pShaderBytecode: shader.as_ptr().cast(), + BytecodeLength: shader.len(), + }, + } + } +} + +#[derive(Debug)] +pub struct RenderPipeline { + raw: Direct3D12::ID3D12PipelineState, + layout: PipelineLayoutShared, + topology: Direct3D::D3D_PRIMITIVE_TOPOLOGY, + vertex_strides: [Option; crate::MAX_VERTEX_BUFFERS], +} + +impl crate::DynRenderPipeline for RenderPipeline {} + +unsafe impl Send for RenderPipeline {} +unsafe impl Sync for RenderPipeline {} + +#[derive(Debug)] +pub struct ComputePipeline { + raw: Direct3D12::ID3D12PipelineState, + layout: PipelineLayoutShared, +} + +impl crate::DynComputePipeline for ComputePipeline {} + +unsafe impl Send for ComputePipeline {} +unsafe impl Sync for ComputePipeline {} + +#[derive(Debug)] +pub struct PipelineCache; + +impl crate::DynPipelineCache for PipelineCache {} + +#[derive(Debug)] +pub struct AccelerationStructure { + resource: Direct3D12::ID3D12Resource, + allocation: suballocation::Allocation, +} + +impl crate::DynAccelerationStructure for AccelerationStructure {} + +impl SwapChain { + unsafe fn release_resources(mut self) -> Dxgi::IDXGISwapChain3 { + if let Some(mut waitable) = self.waitable.take() { + unsafe { Foundation::HANDLE::free(&mut waitable) }; + } + self.raw + } + + unsafe fn wait( + &mut self, + timeout: Option, + ) -> Result { + let timeout_ms = match timeout { + Some(duration) => duration.as_millis() as u32, + None => Threading::INFINITE, + }; + + if let Some(waitable) = self.waitable { + match unsafe { Threading::WaitForSingleObject(waitable, timeout_ms) } { + Foundation::WAIT_ABANDONED | Foundation::WAIT_FAILED => { + Err(crate::SurfaceError::Lost) + } + Foundation::WAIT_OBJECT_0 => Ok(true), + Foundation::WAIT_TIMEOUT => Ok(false), + other => { + log::error!("Unexpected wait status: 0x{other:x?}"); + Err(crate::SurfaceError::Lost) + } + } + } else { + Ok(true) + } + } +} + +impl crate::Surface for Surface { + type A = Api; + + unsafe fn configure( + &self, + device: &Device, + config: &crate::SurfaceConfiguration, + ) -> Result<(), crate::SurfaceError> { + let mut flags = Dxgi::DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT; + // We always set ALLOW_TEARING on the swapchain no matter + // what kind of swapchain we want because ResizeBuffers + // cannot change the swapchain's ALLOW_TEARING flag. + // + // This does not change the behavior of the swapchain, just + // allow present calls to use tearing. + if self.supports_allow_tearing { + flags |= Dxgi::DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING; + } + + // While `configure`s contract ensures that no work on the GPU's main queues + // are in flight, we still need to wait for the present queue to be idle. + unsafe { device.wait_for_present_queue_idle() }?; + + let non_srgb_format = auxil::dxgi::conv::map_texture_format_nosrgb(config.format); + + // The range for `SetMaximumFrameLatency` is 1-16 so the maximum latency requested should be 15 because we add 1. + // https://learn.microsoft.com/en-us/windows/win32/api/dxgi/nf-dxgi-idxgidevice1-setmaximumframelatency + debug_assert!(config.maximum_frame_latency <= 15); + + // Nvidia recommends to use 1-2 more buffers than the maximum latency + // https://developer.nvidia.com/blog/advanced-api-performance-swap-chains/ + // For high latency extra buffers seems excessive, so go with a minimum of 3 and beyond that add 1. + let swap_chain_buffer = (config.maximum_frame_latency + 1).min(16); + + let swap_chain = match self.swap_chain.write().take() { + //Note: this path doesn't properly re-initialize all of the things + Some(sc) => { + let raw = unsafe { sc.release_resources() }; + let result = unsafe { + raw.ResizeBuffers( + swap_chain_buffer, + config.extent.width, + config.extent.height, + non_srgb_format, + flags, + ) + }; + if let Err(err) = result { + log::error!("ResizeBuffers failed: {err}"); + return Err(crate::SurfaceError::Other("window is in use")); + } + raw + } + None => { + let desc = Dxgi::DXGI_SWAP_CHAIN_DESC1 { + AlphaMode: auxil::dxgi::conv::map_acomposite_alpha_mode( + config.composite_alpha_mode, + ), + Width: config.extent.width, + Height: config.extent.height, + Format: non_srgb_format, + Stereo: false.into(), + SampleDesc: Dxgi::Common::DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + BufferUsage: Dxgi::DXGI_USAGE_RENDER_TARGET_OUTPUT, + BufferCount: swap_chain_buffer, + Scaling: Dxgi::DXGI_SCALING_STRETCH, + SwapEffect: Dxgi::DXGI_SWAP_EFFECT_FLIP_DISCARD, + Flags: flags.0 as u32, + }; + let swap_chain1 = match self.target { + SurfaceTarget::Visual(_) + | SurfaceTarget::VisualFromWndHandle { .. } + | SurfaceTarget::SwapChainPanel(_) => { + profiling::scope!("IDXGIFactory2::CreateSwapChainForComposition"); + unsafe { + self.factory.CreateSwapChainForComposition( + &device.present_queue, + &desc, + None, + ) + } + } + SurfaceTarget::SurfaceHandle(handle) => { + profiling::scope!( + "IDXGIFactoryMedia::CreateSwapChainForCompositionSurfaceHandle" + ); + unsafe { + self.factory_media + .as_ref() + .ok_or(crate::SurfaceError::Other("IDXGIFactoryMedia not found"))? + .CreateSwapChainForCompositionSurfaceHandle( + &device.present_queue, + Some(handle), + &desc, + None, + ) + } + } + SurfaceTarget::WndHandle(hwnd) => { + profiling::scope!("IDXGIFactory2::CreateSwapChainForHwnd"); + unsafe { + self.factory.CreateSwapChainForHwnd( + &device.present_queue, + hwnd, + &desc, + None, + None, + ) + } + } + }; + + let swap_chain1 = swap_chain1.map_err(|err| { + log::error!("SwapChain creation error: {err}"); + crate::SurfaceError::Other("swapchain creation") + })?; + + match &self.target { + SurfaceTarget::WndHandle(_) | SurfaceTarget::SurfaceHandle(_) => {} + SurfaceTarget::VisualFromWndHandle { + handle, + dcomp_state, + } => { + let mut dcomp_state = dcomp_state.lock(); + let dcomp_state = + unsafe { dcomp_state.get_or_init(&device.dcomp_lib, handle) }?; + // Set the new swap chain as the content for the backing visual + // and commit the changes to the composition visual tree. + { + profiling::scope!("IDCompositionVisual::SetContent"); + unsafe { dcomp_state.visual.SetContent(&swap_chain1) }.map_err( + |err| { + log::error!("IDCompositionVisual::SetContent failed: {err}"); + crate::SurfaceError::Other("IDCompositionVisual::SetContent") + }, + )?; + } + + // Commit the changes to the composition device. + { + profiling::scope!("IDCompositionDevice::Commit"); + unsafe { dcomp_state.device.Commit() }.map_err(|err| { + log::error!("IDCompositionDevice::Commit failed: {err}"); + crate::SurfaceError::Other("IDCompositionDevice::Commit") + })?; + } + } + SurfaceTarget::Visual(visual) => { + if let Err(err) = unsafe { visual.SetContent(&swap_chain1) } { + log::error!("Unable to SetContent: {err}"); + return Err(crate::SurfaceError::Other( + "IDCompositionVisual::SetContent", + )); + } + } + SurfaceTarget::SwapChainPanel(swap_chain_panel) => { + if let Err(err) = unsafe { swap_chain_panel.SetSwapChain(&swap_chain1) } { + log::error!("Unable to SetSwapChain: {err}"); + return Err(crate::SurfaceError::Other( + "ISwapChainPanelNative::SetSwapChain", + )); + } + } + } + + swap_chain1.cast::().map_err(|err| { + log::error!("Unable to cast swapchain: {err}"); + crate::SurfaceError::Other("swapchain cast to version 3") + })? + } + }; + + match self.target { + SurfaceTarget::WndHandle(wnd_handle) => { + // Disable automatic Alt+Enter handling by DXGI. + unsafe { + self.factory.MakeWindowAssociation( + wnd_handle, + Dxgi::DXGI_MWA_NO_WINDOW_CHANGES | Dxgi::DXGI_MWA_NO_ALT_ENTER, + ) + } + .into_device_result("MakeWindowAssociation")?; + } + SurfaceTarget::Visual(_) + | SurfaceTarget::VisualFromWndHandle { .. } + | SurfaceTarget::SurfaceHandle(_) + | SurfaceTarget::SwapChainPanel(_) => {} + } + + unsafe { swap_chain.SetMaximumFrameLatency(config.maximum_frame_latency) } + .into_device_result("SetMaximumFrameLatency")?; + + let waitable = match device.options.latency_waitable_object { + wgt::Dx12UseFrameLatencyWaitableObject::None => None, + wgt::Dx12UseFrameLatencyWaitableObject::Wait + | wgt::Dx12UseFrameLatencyWaitableObject::DontWait => { + Some(unsafe { swap_chain.GetFrameLatencyWaitableObject() }) + } + }; + + let mut resources = Vec::with_capacity(swap_chain_buffer as usize); + for i in 0..swap_chain_buffer { + let resource = unsafe { swap_chain.GetBuffer(i) } + .into_device_result("Failed to get swapchain buffer")?; + resources.push(resource); + } + + let mut swapchain = self.swap_chain.write(); + *swapchain = Some(SwapChain { + raw: swap_chain, + resources, + waitable, + acquired_count: 0, + present_mode: config.present_mode, + format: config.format, + size: config.extent, + }); + + Ok(()) + } + + unsafe fn unconfigure(&self, device: &Device) { + if let Some(sc) = self.swap_chain.write().take() { + unsafe { + // While `unconfigure`s contract ensures that no work on the GPU's main queues + // are in flight, we still need to wait for the present queue to be idle. + + // The major failure mode of this function is device loss, + // which if we have lost the device, we should just continue + // cleaning up, without error. + let _ = device.wait_for_present_queue_idle(); + + let _raw = sc.release_resources(); + } + } + } + + unsafe fn acquire_texture( + &self, + timeout: Option, + _fence: &Fence, + ) -> Result, crate::SurfaceError> { + let mut swapchain = self.swap_chain.write(); + let sc = swapchain.as_mut().unwrap(); + + match self.options.latency_waitable_object { + wgt::Dx12UseFrameLatencyWaitableObject::None + | wgt::Dx12UseFrameLatencyWaitableObject::DontWait => {} + wgt::Dx12UseFrameLatencyWaitableObject::Wait => { + unsafe { sc.wait(timeout) }?; + } + } + + let base_index = unsafe { sc.raw.GetCurrentBackBufferIndex() } as usize; + let index = (base_index + sc.acquired_count) % sc.resources.len(); + sc.acquired_count += 1; + + let texture = Texture { + resource: sc.resources[index].clone(), + format: sc.format, + dimension: wgt::TextureDimension::D2, + size: sc.size, + mip_level_count: 1, + sample_count: 1, + allocation: suballocation::Allocation::none( + suballocation::AllocationType::Texture, + sc.format.theoretical_memory_footprint(sc.size), + ), + }; + Ok(crate::AcquiredSurfaceTexture { + texture, + suboptimal: false, + }) + } + unsafe fn discard_texture(&self, _texture: Texture) { + let mut swapchain = self.swap_chain.write(); + let sc = swapchain.as_mut().unwrap(); + sc.acquired_count -= 1; + } +} + +impl crate::Queue for Queue { + type A = Api; + + unsafe fn submit( + &self, + command_buffers: &[&CommandBuffer], + _surface_textures: &[&Texture], + (signal_fence, signal_value): (&mut Fence, crate::FenceValue), + ) -> Result<(), crate::DeviceError> { + let mut temp_lists = self.temp_lists.lock(); + temp_lists.clear(); + for cmd_buf in command_buffers { + temp_lists.push(Some(cmd_buf.raw.clone().into())); + } + + { + profiling::scope!("ID3D12CommandQueue::ExecuteCommandLists"); + unsafe { self.raw.ExecuteCommandLists(&temp_lists) } + } + + unsafe { self.raw.Signal(&signal_fence.raw, signal_value) } + .into_device_result("Signal fence")?; + + // Note the lack of synchronization here between the main Direct queue + // and the dedicated presentation queue. This is automatically handled + // by the D3D runtime by detecting uses of resources derived from the + // swapchain. This automatic detection is why you cannot use a swapchain + // as an UAV in D3D12. + + Ok(()) + } + unsafe fn present( + &self, + surface: &Surface, + _texture: Texture, + ) -> Result<(), crate::SurfaceError> { + let mut swapchain = surface.swap_chain.write(); + let sc = swapchain.as_mut().unwrap(); + sc.acquired_count -= 1; + + let (interval, flags) = match sc.present_mode { + // We only allow immediate if ALLOW_TEARING is valid. + wgt::PresentMode::Immediate => (0, Dxgi::DXGI_PRESENT_ALLOW_TEARING), + wgt::PresentMode::Mailbox => (0, Dxgi::DXGI_PRESENT::default()), + wgt::PresentMode::Fifo => (1, Dxgi::DXGI_PRESENT::default()), + m => unreachable!("Cannot make surface with present mode {m:?}"), + }; + + profiling::scope!("IDXGISwapchain3::Present"); + unsafe { sc.raw.Present(interval, flags) } + .ok() + .into_device_result("Present")?; + + Ok(()) + } + + unsafe fn get_timestamp_period(&self) -> f32 { + let frequency = unsafe { self.raw.GetTimestampFrequency() }.expect("GetTimestampFrequency"); + (1_000_000_000.0 / frequency as f64) as f32 + } +} +#[derive(Debug)] +pub struct DxilPassthroughShader { + pub shader: Vec, + pub num_workgroups: (u32, u32, u32), +} + +#[derive(Debug)] +pub struct HlslPassthroughShader { + pub shader: String, + pub num_workgroups: (u32, u32, u32), +} + +#[derive(Debug)] +pub enum ShaderModuleSource { + Naga(crate::NagaShader), + DxilPassthrough(DxilPassthroughShader), + HlslPassthrough(HlslPassthroughShader), +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum FeatureLevel { + _11_0, + _11_1, + _12_0, + _12_1, + _12_2, +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum ShaderModel { + _5_1, + _6_0, + _6_1, + _6_2, + _6_3, + _6_4, + _6_5, + _6_6, + _6_7, + _6_8, + _6_9, +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/pipeline_desc.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/pipeline_desc.rs new file mode 100644 index 00000000..9fcf5661 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/pipeline_desc.rs @@ -0,0 +1,344 @@ +//! We try to use pipeline stream descriptors where possible, but this isn't allowed +//! on some older windows 10 versions. Therefore, we also must have some logic to +//! convert such descriptors to the "traditional" equivalent, +//! `D3D12_GRAPHICS_PIPELINE_STATE_DESC`. +//! +//! Stream descriptors allow extending the pipeline, enabling more advanced features, +//! including mesh shaders and multiview/view instancing. Using a stream descriptor +//! is like using a vulkan descriptor with a `pNext` chain. It doesn't have direct +//! benefits to all use cases, but allows new use cases. +//! +//! The code for pipeline stream descriptors is very complicated, and can have bad +//! consequences if it is written incorrectly. It has been isolated to this file for +//! that reason. + +use core::{ffi::c_void, mem::ManuallyDrop, ptr::NonNull}; + +use alloc::vec::Vec; +use windows::Win32::Graphics::Direct3D12::*; +use windows::Win32::Graphics::Dxgi::Common::*; +use windows_core::Interface; + +use crate::dx12::borrow_interface_temporarily; + +// Wrapper newtypes for various pipeline subobjects which +// use complicated or non-unique representations. + +#[repr(transparent)] +#[derive(Copy, Clone)] +// Option> is guaranteed to have the same representation as a raw pointer. +struct RootSignature(Option>); + +#[repr(transparent)] +#[derive(Copy, Clone)] +struct VertexShader(D3D12_SHADER_BYTECODE); +#[repr(transparent)] +#[derive(Copy, Clone)] +struct PixelShader(D3D12_SHADER_BYTECODE); + +#[repr(transparent)] +#[derive(Copy, Clone)] +struct MeshShader(D3D12_SHADER_BYTECODE); + +#[repr(transparent)] +#[derive(Copy, Clone)] +struct TaskShader(D3D12_SHADER_BYTECODE); + +#[repr(transparent)] +#[derive(Copy, Clone)] +struct SampleMask(u32); + +#[repr(transparent)] +#[derive(Copy, Clone)] +struct NodeMask(u32); + +/// Trait for types that can be used as subobjects in a pipeline state stream. +/// +/// Safety: +/// - The type must be the correct alignment and size for the subobject it represents. +/// - The type must map to exactly one `D3D12_PIPELINE_STATE_SUBOBJECT_TYPE` variant. +/// - The variant must correctly represent the type's role in the pipeline state stream. +/// - The type must be `Copy` to ensure safe duplication in the stream. +/// - The type must be valid to memcpy into the pipeline state stream. +unsafe trait RenderPipelineStreamObject: Copy { + const SUBOBJECT_TYPE: D3D12_PIPELINE_STATE_SUBOBJECT_TYPE; +} + +macro_rules! implement_stream_object { + (unsafe $ty:ty => $variant:expr) => { + unsafe impl RenderPipelineStreamObject for $ty { + const SUBOBJECT_TYPE: D3D12_PIPELINE_STATE_SUBOBJECT_TYPE = $variant; + } + }; +} + +implement_stream_object! { unsafe RootSignature => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_ROOT_SIGNATURE } +implement_stream_object! { unsafe VertexShader => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_VS } +implement_stream_object! { unsafe PixelShader => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_PS } +implement_stream_object! { unsafe MeshShader => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_MS } +implement_stream_object! { unsafe TaskShader => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_AS } +implement_stream_object! { unsafe D3D12_BLEND_DESC => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_BLEND } +implement_stream_object! { unsafe SampleMask => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_SAMPLE_MASK } +implement_stream_object! { unsafe D3D12_RASTERIZER_DESC => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_RASTERIZER } +implement_stream_object! { unsafe D3D12_DEPTH_STENCIL_DESC => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_DEPTH_STENCIL } +implement_stream_object! { unsafe D3D12_PRIMITIVE_TOPOLOGY_TYPE => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_PRIMITIVE_TOPOLOGY } +implement_stream_object! { unsafe D3D12_RT_FORMAT_ARRAY => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_RENDER_TARGET_FORMATS } +implement_stream_object! { unsafe DXGI_FORMAT => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_DEPTH_STENCIL_FORMAT } +implement_stream_object! { unsafe DXGI_SAMPLE_DESC => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_SAMPLE_DESC } +implement_stream_object! { unsafe NodeMask => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_NODE_MASK } +implement_stream_object! { unsafe D3D12_CACHED_PIPELINE_STATE => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_CACHED_PSO } +implement_stream_object! { unsafe D3D12_PIPELINE_STATE_FLAGS => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_FLAGS } +implement_stream_object! { unsafe D3D12_INPUT_LAYOUT_DESC => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_INPUT_LAYOUT } +implement_stream_object! { unsafe D3D12_INDEX_BUFFER_STRIP_CUT_VALUE => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_IB_STRIP_CUT_VALUE } +implement_stream_object! { unsafe D3D12_STREAM_OUTPUT_DESC => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_STREAM_OUTPUT } +implement_stream_object! { unsafe D3D12_VIEW_INSTANCING_DESC => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE_VIEW_INSTANCING } + +/// Implementaation of a pipeline state stream, which is a sequence of subobjects put into +/// a byte array according to some basic alignment rules. +/// +/// Each subobject must start on an 8 byte boundary. Each subobject contains a 32 bit +/// type identifier, followed by the actual subobject data, aligned as required by the +/// subobject's structure. +/// +/// See +/// for more information. +pub(super) struct RenderPipelineStateStream<'a> { + bytes: Vec, + _marker: core::marker::PhantomData<&'a ()>, +} + +impl<'a> RenderPipelineStateStream<'a> { + fn new() -> Self { + // Dynamic allocation is used here because the resulting stream can become very large. + // We pre-allocate the size based on an estimate of the size of the struct plus some extra space + // per member for tags and alignment padding. In practice this will always be too big, as not + // all members will be used. + let size_of_stream_desc = size_of::(); + let members = 20; // Approximate number of members we might push + let capacity = size_of_stream_desc + members * 8; // Extra space for tags and alignment + Self { + bytes: Vec::with_capacity(capacity), + _marker: core::marker::PhantomData, + } + } + + /// Align the internal byte buffer to the given alignment, + /// padding with zeros as necessary. + fn align_to(&mut self, alignment: usize) { + let aligned_length = self.bytes.len().next_multiple_of(alignment); + self.bytes.resize(aligned_length, 0); + } + + /// Adds a subobject to the pipeline state stream. + fn add_object(&mut self, object: T) { + // Ensure 8-byte alignment for the subobject start. + self.align_to(8); + + // Append the type tag (u32) + let tag: u32 = T::SUBOBJECT_TYPE.0 as u32; + self.bytes.extend_from_slice(&tag.to_ne_bytes()); + + // Align the data to its natural alignment. + self.align_to(align_of_val::(&object)); + + // Append the data itself, as raw bytes + let data_ptr: *const T = &object; + let data_u8_ptr: *const u8 = data_ptr.cast::(); + let data_size = size_of_val::(&object); + let slice = unsafe { core::slice::from_raw_parts::(data_u8_ptr, data_size) }; + self.bytes.extend_from_slice(slice); + } + + /// Creates a pipeline state object from the stream. + /// + /// Safety: + /// - All unsafety invariants required by [`ID3D12Device2::CreatePipelineState`] must be upheld by the caller. + pub unsafe fn create_pipeline_state( + &mut self, + device: &ID3D12Device2, + ) -> windows::core::Result { + let stream_desc = D3D12_PIPELINE_STATE_STREAM_DESC { + SizeInBytes: self.bytes.len(), + pPipelineStateSubobjectStream: self.bytes.as_mut_ptr().cast(), + }; + + // Safety: lifetime on Self preserved the contents + // of the stream. Other unsafety invariants are upheld by the caller. + unsafe { device.CreatePipelineState(&stream_desc) } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct RenderPipelineStateStreamDesc<'a> { + pub root_signature: Option<&'a ID3D12RootSignature>, + pub pixel_shader: D3D12_SHADER_BYTECODE, + pub blend_state: D3D12_BLEND_DESC, + pub sample_mask: u32, + pub rasterizer_state: D3D12_RASTERIZER_DESC, + pub depth_stencil_state: D3D12_DEPTH_STENCIL_DESC, + pub primitive_topology_type: D3D12_PRIMITIVE_TOPOLOGY_TYPE, + pub rtv_formats: D3D12_RT_FORMAT_ARRAY, + pub dsv_format: DXGI_FORMAT, + pub sample_desc: DXGI_SAMPLE_DESC, + pub node_mask: u32, + pub cached_pso: D3D12_CACHED_PIPELINE_STATE, + pub flags: D3D12_PIPELINE_STATE_FLAGS, + pub view_instancing: Option, + + // Vertex pipeline specific + pub vertex_shader: D3D12_SHADER_BYTECODE, + pub input_layout: D3D12_INPUT_LAYOUT_DESC, + pub index_buffer_strip_cut_value: D3D12_INDEX_BUFFER_STRIP_CUT_VALUE, + pub stream_output: D3D12_STREAM_OUTPUT_DESC, + + // Mesh pipeline specific + pub task_shader: D3D12_SHADER_BYTECODE, + pub mesh_shader: D3D12_SHADER_BYTECODE, +} + +impl RenderPipelineStateStreamDesc<'_> { + pub fn to_stream(&self) -> RenderPipelineStateStream<'_> { + let mut stream = RenderPipelineStateStream::new(); + + // Importantly here, the ID3D12RootSignature _itself_ is the pointer we're + // trying to serialize into the stream, not a pointer to the pointer. + // + // This is correct because as_raw() returns turns that smart object into the raw + // pointer that _is_ the com object handle. + let root_sig_pointer = self + .root_signature + .map(|a| NonNull::new(a.as_raw()).unwrap()); + // Because the stream object borrows from self for its entire lifetime, + // it is safe to store the pointer into it. + stream.add_object(RootSignature(root_sig_pointer)); + + stream.add_object(self.blend_state); + stream.add_object(SampleMask(self.sample_mask)); + stream.add_object(self.rasterizer_state); + stream.add_object(self.depth_stencil_state); + stream.add_object(self.primitive_topology_type); + if self.rtv_formats.NumRenderTargets != 0 { + stream.add_object(self.rtv_formats); + } + if self.dsv_format != DXGI_FORMAT_UNKNOWN { + stream.add_object(self.dsv_format); + } + stream.add_object(self.sample_desc); + if self.node_mask != 0 { + stream.add_object(NodeMask(self.node_mask)); + } + if !self.cached_pso.pCachedBlob.is_null() { + stream.add_object(self.cached_pso); + } + stream.add_object(self.flags); + if let Some(view_instancing) = self.view_instancing { + stream.add_object(view_instancing); + } + if !self.pixel_shader.pShaderBytecode.is_null() { + stream.add_object(PixelShader(self.pixel_shader)); + } + if !self.vertex_shader.pShaderBytecode.is_null() { + stream.add_object(VertexShader(self.vertex_shader)); + stream.add_object(self.input_layout); + stream.add_object(self.index_buffer_strip_cut_value); + stream.add_object(self.stream_output); + } + if !self.task_shader.pShaderBytecode.is_null() { + stream.add_object(TaskShader(self.task_shader)); + } + if !self.mesh_shader.pShaderBytecode.is_null() { + stream.add_object(MeshShader(self.mesh_shader)); + } + + stream + } + + /// Returns a traditional D3D12_GRAPHICS_PIPELINE_STATE_DESC. + /// + /// Safety: + /// - This returned struct must not outlive self. + pub unsafe fn to_graphics_pipeline_descriptor(&self) -> D3D12_GRAPHICS_PIPELINE_STATE_DESC { + D3D12_GRAPHICS_PIPELINE_STATE_DESC { + pRootSignature: if let Some(rsig) = self.root_signature { + unsafe { borrow_interface_temporarily(rsig) } + } else { + ManuallyDrop::new(None) + }, + VS: self.vertex_shader, + PS: self.pixel_shader, + DS: D3D12_SHADER_BYTECODE::default(), + HS: D3D12_SHADER_BYTECODE::default(), + GS: D3D12_SHADER_BYTECODE::default(), + StreamOutput: self.stream_output, + BlendState: self.blend_state, + SampleMask: self.sample_mask, + RasterizerState: self.rasterizer_state, + DepthStencilState: self.depth_stencil_state, + InputLayout: self.input_layout, + IBStripCutValue: self.index_buffer_strip_cut_value, + PrimitiveTopologyType: self.primitive_topology_type, + NumRenderTargets: self.rtv_formats.NumRenderTargets, + RTVFormats: self.rtv_formats.RTFormats, + DSVFormat: self.dsv_format, + SampleDesc: self.sample_desc, + NodeMask: self.node_mask, + CachedPSO: self.cached_pso, + Flags: self.flags, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wrappers() { + assert_eq!(size_of::(), size_of::()); + assert_eq!( + align_of::(), + align_of::() + ) + } + + implement_stream_object!(unsafe u16 => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE(1)); + implement_stream_object!(unsafe u32 => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE(2)); + implement_stream_object!(unsafe u64 => D3D12_PIPELINE_STATE_SUBOBJECT_TYPE(3)); + + #[test] + fn stream() { + let mut stream = RenderPipelineStateStream::new(); + + stream.add_object(42u16); + stream.add_object(84u32); + stream.add_object(168u64); + + assert_eq!(stream.bytes.len(), 32); + + // Object 1: u16 + + // Tag at the beginning + assert_eq!(&stream.bytes[0..4], &1u32.to_ne_bytes()); + // Data tucked in, aligned to the natural alignment of u16 + assert_eq!(&stream.bytes[4..6], &42u16.to_ne_bytes()); + // Padding to align the next subobject to an 8 byte boundary. + assert_eq!(&stream.bytes[6..8], &[0, 0]); + + // Object 2: u32 + + // Tag at the beginning + assert_eq!(&stream.bytes[8..12], &2u32.to_ne_bytes()); + // Data tucked in, aligned to the natural alignment of u32 + assert_eq!(&stream.bytes[12..16], &84u32.to_ne_bytes()); + + // Object 3: u64 + + // Tag at the beginning + assert_eq!(&stream.bytes[16..20], &3u32.to_ne_bytes()); + // Padding to align the u64 to an 8 byte boundary. + assert_eq!(&stream.bytes[20..24], &[0, 0, 0, 0]); + // Data tucked in, aligned to the natural alignment of u64 + assert_eq!(&stream.bytes[24..32], &168u64.to_ne_bytes()); + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/sampler.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/sampler.rs new file mode 100644 index 00000000..4b58a0b4 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/sampler.rs @@ -0,0 +1,252 @@ +//! Sampler management for DX12. +//! +//! Nearly identical to the Vulkan sampler cache, with added descriptor heap management. + +use alloc::vec::Vec; + +use hashbrown::{hash_map::Entry, HashMap}; + +use ordered_float::OrderedFloat; +use parking_lot::Mutex; +use windows::Win32::Graphics::Direct3D12::*; + +use crate::dx12::HResult; + +/// The index of a sampler in the global sampler heap. +/// +/// This is a type-safe, transparent wrapper around a u32. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(crate) struct SamplerIndex(u32); + +/// [`D3D12_SAMPLER_DESC`] is not hashable, so we wrap it in a newtype that is. +/// +/// We use [`OrderedFloat`] to allow for floating point values to be compared and +/// hashed in a defined way. +#[derive(Debug, Copy, Clone)] +struct HashableSamplerDesc(D3D12_SAMPLER_DESC); + +impl PartialEq for HashableSamplerDesc { + fn eq(&self, other: &Self) -> bool { + self.0.Filter == other.0.Filter + && self.0.AddressU == other.0.AddressU + && self.0.AddressV == other.0.AddressV + && self.0.AddressW == other.0.AddressW + && OrderedFloat(self.0.MipLODBias) == OrderedFloat(other.0.MipLODBias) + && self.0.MaxAnisotropy == other.0.MaxAnisotropy + && self.0.ComparisonFunc == other.0.ComparisonFunc + && self.0.BorderColor.map(OrderedFloat) == other.0.BorderColor.map(OrderedFloat) + && OrderedFloat(self.0.MinLOD) == OrderedFloat(other.0.MinLOD) + && OrderedFloat(self.0.MaxLOD) == OrderedFloat(other.0.MaxLOD) + } +} + +impl Eq for HashableSamplerDesc {} + +impl core::hash::Hash for HashableSamplerDesc { + fn hash(&self, state: &mut H) { + self.0.Filter.0.hash(state); + self.0.AddressU.0.hash(state); + self.0.AddressV.0.hash(state); + self.0.AddressW.0.hash(state); + OrderedFloat(self.0.MipLODBias).hash(state); + self.0.MaxAnisotropy.hash(state); + self.0.ComparisonFunc.0.hash(state); + self.0.BorderColor.map(OrderedFloat).hash(state); + OrderedFloat(self.0.MinLOD).hash(state); + OrderedFloat(self.0.MaxLOD).hash(state); + } +} + +/// Entry in the sampler cache. +struct CacheEntry { + index: SamplerIndex, + ref_count: u32, +} + +/// Container for the mutable management state of the sampler heap. +/// +/// We have this separated, using interior mutability, to allow for the outside world +/// to access the heap directly without needing to take the lock. +pub(crate) struct SamplerHeapState { + /// Mapping from the sampler description to the index within the heap and the refcount. + mapping: HashMap, + /// List of free sampler indices. + freelist: Vec, +} + +/// Global sampler heap for the device. +/// +/// As D3D12 only allows 2048 samplers to be in a single heap, we need to cache +/// samplers aggressively and refer to them in shaders by index. +pub(crate) struct SamplerHeap { + /// Mutable management state of the sampler heap. + state: Mutex, + + /// The heap itself. + heap: ID3D12DescriptorHeap, + /// The CPU-side handle to the first descriptor in the heap. + /// + /// Both the CPU and GPU handles point to the same descriptor, just in + /// different contexts. + heap_cpu_start_handle: D3D12_CPU_DESCRIPTOR_HANDLE, + /// The GPU-side handle to the first descriptor in the heap. + /// + /// Both the CPU and GPU handles point to the same descriptor, just in + /// different contexts. + heap_gpu_start_handle: D3D12_GPU_DESCRIPTOR_HANDLE, + + /// This is the device-specific size of sampler descriptors. + descriptor_stride: u32, +} + +impl SamplerHeap { + pub fn new( + device: &ID3D12Device, + private_caps: &super::PrivateCapabilities, + ) -> Result { + profiling::scope!("SamplerHeap::new"); + + // WARP can report this as 2M or more. We clamp it to 64k to be safe. + const SAMPLER_HEAP_SIZE_CLAMP: u32 = 64 * 1024; + + let max_unique_samplers = private_caps + .max_sampler_descriptor_heap_size + .min(SAMPLER_HEAP_SIZE_CLAMP); + + let desc = D3D12_DESCRIPTOR_HEAP_DESC { + Type: D3D12_DESCRIPTOR_HEAP_TYPE_SAMPLER, + NumDescriptors: max_unique_samplers, + Flags: D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE, + NodeMask: 0, + }; + let heap = unsafe { device.CreateDescriptorHeap::(&desc) } + .into_device_result("Failed to create global GPU-Visible Sampler Descriptor Heap")?; + + let heap_cpu_start_handle = unsafe { heap.GetCPUDescriptorHandleForHeapStart() }; + let heap_gpu_start_handle = unsafe { heap.GetGPUDescriptorHandleForHeapStart() }; + + let descriptor_stride = + unsafe { device.GetDescriptorHandleIncrementSize(D3D12_DESCRIPTOR_HEAP_TYPE_SAMPLER) }; + + Ok(Self { + state: Mutex::new(SamplerHeapState { + mapping: HashMap::new(), + // Reverse so that samplers get allocated starting from zero. + freelist: (0..max_unique_samplers).map(SamplerIndex).rev().collect(), + }), + heap, + heap_cpu_start_handle, + heap_gpu_start_handle, + descriptor_stride, + }) + } + + /// Returns a reference to the raw descriptor heap. + pub fn heap(&self) -> &ID3D12DescriptorHeap { + &self.heap + } + + /// Returns a reference the handle to be bound to the descriptor table. + pub fn gpu_descriptor_table(&self) -> D3D12_GPU_DESCRIPTOR_HANDLE { + self.heap_gpu_start_handle + } + + /// Add a sampler with the given description to the heap. + /// + /// If the sampler already exists, the refcount is incremented and the existing index is returned. + /// + /// If the sampler does not exist, a new sampler is created and the index is returned. + /// + /// If the heap is full, an error is returned. + pub fn create_sampler( + &self, + device: &ID3D12Device, + desc: D3D12_SAMPLER_DESC, + ) -> Result { + profiling::scope!("SamplerHeap::create_sampler"); + + let hashable_desc = HashableSamplerDesc(desc); + + // Eagarly dereference the lock to allow split borrows. + let state = &mut *self.state.lock(); + + // Lookup the sampler in the mapping. + match state.mapping.entry(hashable_desc) { + Entry::Occupied(occupied_entry) => { + // We have found a match, so increment the refcount and return the index. + let entry = occupied_entry.into_mut(); + entry.ref_count += 1; + Ok(entry.index) + } + Entry::Vacant(vacant_entry) => { + // We need to create a new sampler. + + // Try to get a new index from the freelist. + let Some(index) = state.freelist.pop() else { + // If the freelist is empty, we have hit the maximum number of samplers. + log::error!("There is no more room in the global sampler heap for more unique samplers. Your device supports a maximum of {} unique samplers.", state.mapping.len()); + return Err(crate::DeviceError::OutOfMemory); + }; + + // Compute the CPU side handle for the new sampler. + let handle = D3D12_CPU_DESCRIPTOR_HANDLE { + ptr: self.heap_cpu_start_handle.ptr + + self.descriptor_stride as usize * index.0 as usize, + }; + + unsafe { + device.CreateSampler(&desc, handle); + } + + // Insert the new sampler into the mapping. + vacant_entry.insert(CacheEntry { + index, + ref_count: 1, + }); + + Ok(index) + } + } + } + + /// Decrement the refcount of the sampler with the given description. + /// + /// If the refcount reaches zero, the sampler is destroyed and the index is returned to the freelist. + /// + /// The provided index is checked against the index of the sampler with the given description, ensuring + /// that there isn't a clerical error from the caller. + pub fn destroy_sampler(&self, desc: D3D12_SAMPLER_DESC, provided_index: SamplerIndex) { + profiling::scope!("SamplerHeap::destroy_sampler"); + + // Eagarly dereference the lock to allow split borrows. + let state = &mut *self.state.lock(); + + // Get the index of the sampler to destroy. + let Entry::Occupied(mut hash_map_entry) = state.mapping.entry(HashableSamplerDesc(desc)) + else { + log::error!( + "Tried to destroy a sampler that doesn't exist. Sampler description: {desc:#?}" + ); + return; + }; + let cache_entry = hash_map_entry.get_mut(); + + // Ensure that the provided index matches the index of the sampler to destroy. + assert_eq!( + cache_entry.index, provided_index, + "Mismatched sampler index, this is an implementation bug" + ); + + // Decrement the refcount of the sampler. + cache_entry.ref_count -= 1; + + // If we are the last reference, remove the sampler from the mapping and return the index to the freelist. + // + // As samplers only exist as descriptors in the heap, there is nothing needed to be done to destroy the sampler. + if cache_entry.ref_count == 0 { + state.freelist.push(cache_entry.index); + hash_map_entry.remove(); + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/shader_compilation.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/shader_compilation.rs new file mode 100644 index 00000000..4b199b43 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/shader_compilation.rs @@ -0,0 +1,447 @@ +use alloc::{string::String, vec::Vec}; +use core::ffi::CStr; +use std::path::PathBuf; + +use crate::auxil::dxgi::result::HResult; +use thiserror::Error; +use windows::{ + core::{Interface, PCSTR, PCWSTR}, + Win32::Graphics::Direct3D::{Dxc, Fxc, ID3DBlob, D3D_SHADER_MACRO}, +}; + +pub(super) enum CompilerContainer { + Fxc(CompilerFxc), + DynamicDxc(CompilerDynamicDxc), + #[cfg_attr(not(static_dxc), allow(unused))] + StaticDxc(CompilerStaticDxc), +} + +pub(super) struct CompilerFxc { + fxc: FxcLib, +} + +pub(super) struct CompilerDynamicDxc { + max_shader_model: wgt::DxcShaderModel, + compiler: Dxc::IDxcCompiler3, + // Has to be held onto for the lifetime of the device otherwise shaders will fail to compile. + // Only needed when using dynamic linking. + _dxc: DxcLib, +} + +pub(super) struct CompilerStaticDxc { + max_shader_model: wgt::DxcShaderModel, + compiler: Dxc::IDxcCompiler3, +} + +#[derive(Debug, Error)] +pub(super) enum GetContainerError { + #[error(transparent)] + Device(#[from] crate::DeviceError), + #[error("Failed to load {0}: {1}")] + FailedToLoad(&'static str, libloading::Error), +} + +impl CompilerContainer { + pub(super) fn new_fxc() -> Result { + FxcLib::new_dynamic().map(|fxc| Self::Fxc(CompilerFxc { fxc })) + } + + pub(super) fn new_dynamic_dxc(dxc_path: PathBuf) -> Result { + let dxc = DxcLib::new_dynamic(dxc_path) + .map_err(|e| GetContainerError::FailedToLoad("dxcompiler.dll", e))?; + + let compiler = dxc.create_instance::()?; + + let (mut major, mut minor) = (1, 0); + // DXC 1.0 didn't support this. If the cast fails assume it is DXC 1.0. + if let Ok(version_info) = compiler.cast::() { + unsafe { + version_info.GetVersion(&mut major, &mut minor).unwrap(); + } + } + + Ok(Self::DynamicDxc(CompilerDynamicDxc { + max_shader_model: wgt::DxcShaderModel::from_dxc_version(major, minor), + compiler, + _dxc: dxc, + })) + } + + /// Creates a [`CompilerContainer`] that delegates to the statically-linked version of DXC. + pub(super) fn new_static_dxc() -> Result { + #[cfg(static_dxc)] + { + unsafe { + let compiler = dxc_create_instance::(|clsid, iid, ppv| { + windows_core::HRESULT(mach_dxcompiler_rs::DxcCreateInstance( + clsid.cast(), + iid.cast(), + ppv, + )) + })?; + + Ok(CompilerContainer::StaticDxc(CompilerStaticDxc { + max_shader_model: wgt::DxcShaderModel::V6_7, + compiler, + })) + } + } + #[cfg(not(static_dxc))] + { + panic!("Attempted to create a static DXC shader compiler, but the static-dxc feature was not enabled") + } + } + + pub(super) fn max_shader_model(&self) -> Option { + match self { + CompilerContainer::Fxc(..) => None, + CompilerContainer::DynamicDxc(CompilerDynamicDxc { + max_shader_model, .. + }) + | CompilerContainer::StaticDxc(CompilerStaticDxc { + max_shader_model, .. + }) => Some(max_shader_model.clone()), + } + } + + pub(super) fn compile( + &self, + device: &super::Device, + source: &str, + source_name: Option<&CStr>, + raw_ep: &str, + stage_bit: wgt::ShaderStages, + full_stage: &str, + ) -> Result { + match self { + CompilerContainer::Fxc(CompilerFxc { fxc }) => compile_fxc( + device, + source, + source_name, + raw_ep, + stage_bit, + full_stage, + fxc, + ), + CompilerContainer::DynamicDxc(CompilerDynamicDxc { compiler, .. }) + | CompilerContainer::StaticDxc(CompilerStaticDxc { compiler, .. }) => compile_dxc( + device, + source, + source_name, + raw_ep, + stage_bit, + full_stage, + compiler, + ), + } + } +} + +type D3DCompileFn = unsafe extern "system" fn( + psrcdata: *const core::ffi::c_void, + srcdatasize: usize, + psourcename: PCSTR, + pdefines: *const D3D_SHADER_MACRO, + pinclude: *mut core::ffi::c_void, + pentrypoint: PCSTR, + ptarget: PCSTR, + flags1: u32, + flags2: u32, + ppcode: *mut *mut core::ffi::c_void, + pperrormsgs: *mut *mut core::ffi::c_void, +) -> windows_core::HRESULT; + +#[derive(Debug)] +struct FxcLib { + // `d3dcompile_fn` points into `_lib`, so `_lib` must be held for as long + // as we want to keep compiling shaders with FXC. + _lib: crate::dx12::DynLib, + d3dcompile_fn: D3DCompileFn, +} + +impl FxcLib { + const PATH: &str = "d3dcompiler_47.dll"; + + fn new_dynamic() -> Result { + unsafe { + let lib = crate::dx12::DynLib::new(Self::PATH) + .map_err(|e| GetContainerError::FailedToLoad(FxcLib::PATH, e))?; + let d3dcompile_fn: D3DCompileFn = *lib.get::(c"D3DCompile".to_bytes())?; + + Ok(Self { + _lib: lib, + d3dcompile_fn, + }) + } + } + + #[allow(clippy::too_many_arguments)] + fn compile( + &self, + source: &str, + source_name: Option<&CStr>, + raw_ep: &str, + full_stage: &str, + compile_flags: u32, + shader_data: &mut Option, + error: &mut Option, + ) -> Result, crate::DeviceError> { + unsafe { + let raw_ep = alloc::ffi::CString::new(raw_ep).unwrap(); + let full_stage = alloc::ffi::CString::new(full_stage).unwrap(); + + // If no name has been set, D3DCompile wants the null pointer. + let source_name = source_name + .map(|cstr| cstr.as_ptr().cast()) + .unwrap_or(core::ptr::null()); + + let shader_data: *mut Option = shader_data; + let error: *mut Option = error; + + { + profiling::scope!("Fxc::D3DCompile"); + Ok((self.d3dcompile_fn)( + source.as_ptr().cast(), + source.len(), + PCSTR(source_name), + core::ptr::null(), + core::ptr::null_mut(), + PCSTR(raw_ep.as_ptr().cast()), + PCSTR(full_stage.as_ptr().cast()), + compile_flags, + 0, + shader_data.cast(), + error.cast(), + ) + .ok()) + } + } + } +} + +fn compile_fxc( + device: &super::Device, + source: &str, + source_name: Option<&CStr>, + raw_ep: &str, + stage_bit: wgt::ShaderStages, + full_stage: &str, + fxc: &FxcLib, +) -> Result { + profiling::scope!("compile_fxc"); + let mut compile_flags = Fxc::D3DCOMPILE_ENABLE_STRICTNESS; + if device + .shared + .private_caps + .instance_flags + .contains(wgt::InstanceFlags::DEBUG) + { + compile_flags |= Fxc::D3DCOMPILE_DEBUG | Fxc::D3DCOMPILE_SKIP_OPTIMIZATION; + } + + let mut shader_data = None; + let mut error = None; + let hr = fxc.compile( + source, + source_name, + raw_ep, + full_stage, + compile_flags, + &mut shader_data, + &mut error, + )?; + + match hr { + Ok(()) => { + let shader_data = shader_data.unwrap(); + Ok(super::CompiledShader::Fxc(shader_data)) + } + Err(e) => { + let mut full_msg = format!("FXC D3DCompile error ({e})"); + if let Some(error) = error { + use core::fmt::Write as _; + let message = unsafe { + core::slice::from_raw_parts( + error.GetBufferPointer().cast(), + error.GetBufferSize(), + ) + }; + let _ = write!(full_msg, ": {}", String::from_utf8_lossy(message)); + } + Err(crate::PipelineError::Linkage(stage_bit, full_msg)) + } + } +} + +trait DxcObj: Interface { + const CLSID: windows::core::GUID; +} +impl DxcObj for Dxc::IDxcCompiler3 { + const CLSID: windows::core::GUID = Dxc::CLSID_DxcCompiler; +} +impl DxcObj for Dxc::IDxcUtils { + const CLSID: windows::core::GUID = Dxc::CLSID_DxcUtils; +} +impl DxcObj for Dxc::IDxcValidator { + const CLSID: windows::core::GUID = Dxc::CLSID_DxcValidator; +} + +#[derive(Debug)] +struct DxcLib { + lib: crate::dx12::DynLib, +} + +impl DxcLib { + fn new_dynamic(lib_path: PathBuf) -> Result { + unsafe { crate::dx12::DynLib::new(lib_path).map(|lib| Self { lib }) } + } + + pub fn create_instance(&self) -> Result { + unsafe { + type DxcCreateInstanceFn = unsafe extern "system" fn( + rclsid: *const windows_core::GUID, + riid: *const windows_core::GUID, + ppv: *mut *mut core::ffi::c_void, + ) + -> windows_core::HRESULT; + + let func: libloading::Symbol = + self.lib.get(c"DxcCreateInstance".to_bytes())?; + dxc_create_instance::(|clsid, iid, ppv| func(clsid, iid, ppv)) + } + } +} + +/// Invokes the provided library function to create a DXC object. +unsafe fn dxc_create_instance( + f: impl Fn( + *const windows_core::GUID, + *const windows_core::GUID, + *mut *mut core::ffi::c_void, + ) -> windows_core::HRESULT, +) -> Result { + let mut result__ = None; + f(&T::CLSID, &T::IID, <*mut _>::cast(&mut result__)) + .ok() + .into_device_result("DxcCreateInstance")?; + result__.ok_or(crate::DeviceError::Unexpected) +} + +/// Owned PCWSTR +#[allow(clippy::upper_case_acronyms)] +struct OPCWSTR { + inner: Vec, +} + +impl OPCWSTR { + fn new(s: &str) -> Self { + let mut inner: Vec<_> = s.encode_utf16().collect(); + inner.push(0); + Self { inner } + } + + fn ptr(&self) -> PCWSTR { + PCWSTR(self.inner.as_ptr()) + } +} + +fn get_output( + res: &Dxc::IDxcResult, + kind: Dxc::DXC_OUT_KIND, +) -> Result { + let mut result__: Option = None; + unsafe { res.GetOutput::(kind, &mut None, <*mut _>::cast(&mut result__)) } + .into_device_result("GetOutput")?; + result__.ok_or(crate::DeviceError::Unexpected) +} + +fn as_err_str(blob: &Dxc::IDxcBlobUtf8) -> Result<&str, crate::DeviceError> { + let ptr = unsafe { blob.GetStringPointer() }; + let len = unsafe { blob.GetStringLength() }; + core::str::from_utf8(unsafe { core::slice::from_raw_parts(ptr.0, len) }) + .map_err(|_| crate::DeviceError::Unexpected) +} + +fn compile_dxc( + device: &crate::dx12::Device, + source: &str, + source_name: Option<&CStr>, + raw_ep: &str, + stage_bit: wgt::ShaderStages, + full_stage: &str, + compiler: &Dxc::IDxcCompiler3, +) -> Result { + profiling::scope!("compile_dxc"); + + let source_name = source_name.and_then(|cstr| cstr.to_str().ok()); + + let source_name = source_name.map(OPCWSTR::new); + let raw_ep = OPCWSTR::new(raw_ep); + let full_stage = OPCWSTR::new(full_stage); + + let mut compile_args = arrayvec::ArrayVec::::new_const(); + + if let Some(source_name) = source_name.as_ref() { + compile_args.push(source_name.ptr()) + } + + compile_args.extend([ + windows::core::w!("-E"), + raw_ep.ptr(), + windows::core::w!("-T"), + full_stage.ptr(), + windows::core::w!("-HV"), + windows::core::w!("2018"), // Use HLSL 2018, Naga doesn't supported 2021 yet. + windows::core::w!("-no-warnings"), + Dxc::DXC_ARG_ENABLE_STRICTNESS, + ]); + + if device + .shared + .private_caps + .instance_flags + .contains(wgt::InstanceFlags::DEBUG) + && !device + .shared + .private_caps + .workarounds + .avoid_shader_debug_info + { + compile_args.push(Dxc::DXC_ARG_DEBUG); + compile_args.push(Dxc::DXC_ARG_SKIP_OPTIMIZATIONS); + } + + if device.features.contains(wgt::Features::SHADER_F16) { + compile_args.push(windows::core::w!("-enable-16bit-types")); + } + + let buffer = Dxc::DxcBuffer { + Ptr: source.as_ptr().cast(), + Size: source.len(), + Encoding: Dxc::DXC_CP_UTF8.0, + }; + + let compile_res: Dxc::IDxcResult = + unsafe { compiler.Compile(&buffer, Some(&compile_args), None) } + .into_device_result("Compile")?; + + drop(compile_args); + drop(source_name); + drop(raw_ep); + drop(full_stage); + + let err_blob = get_output::(&compile_res, Dxc::DXC_OUT_ERRORS)?; + + let len = unsafe { err_blob.GetStringLength() }; + if len != 0 { + let err = as_err_str(&err_blob)?; + return Err(crate::PipelineError::Linkage( + stage_bit, + format!("DXC compile error: {err}"), + )); + } + + let blob = get_output::(&compile_res, Dxc::DXC_OUT_OBJECT)?; + + Ok(crate::dx12::CompiledShader::Dxc(blob)) +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/suballocation.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/suballocation.rs new file mode 100644 index 00000000..89c7dc47 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/suballocation.rs @@ -0,0 +1,603 @@ +use alloc::sync::Arc; + +use gpu_allocator::{d3d12::AllocationCreateDesc, MemoryLocation}; +use parking_lot::Mutex; +use windows::Win32::Graphics::{Direct3D12, Dxgi}; + +use crate::{ + auxil::dxgi::{name::ObjectExt as _, result::HResult as _}, + dx12::conv, + AllocationSizes, +}; + +#[derive(Debug)] +pub(crate) enum AllocationType { + Buffer, + Texture, + AccelerationStructure, +} + +#[derive(Debug)] +enum AllocationInner { + /// This resource is suballocated from a heap. + Placed { + inner: gpu_allocator::d3d12::Allocation, + }, + /// This resource is a committed resource and does not belong to a + /// suballocated heap. We store an approximate size, so we can manage our counters + /// correctly. + /// + /// This is only used for Intel Xe drivers, which have a bug that + /// prevents suballocation from working correctly. + Committed { size: u64 }, +} + +#[derive(Debug)] +pub(crate) struct Allocation { + inner: AllocationInner, + ty: AllocationType, +} + +impl Allocation { + pub fn placed(inner: gpu_allocator::d3d12::Allocation, ty: AllocationType) -> Self { + Self { + inner: AllocationInner::Placed { inner }, + ty, + } + } + + pub fn none(ty: AllocationType, size: u64) -> Self { + Self { + inner: AllocationInner::Committed { size }, + ty, + } + } + + pub fn size(&self) -> u64 { + match self.inner { + AllocationInner::Placed { ref inner } => inner.size(), + AllocationInner::Committed { size } => size, + } + } +} + +#[derive(Clone)] +pub(crate) struct Allocator { + inner: Arc>, + device_memblock_size: u64, + host_memblock_size: u64, + pub memory_budget_thresholds: wgt::MemoryBudgetThresholds, +} + +impl Allocator { + pub(crate) fn new( + raw: &Direct3D12::ID3D12Device, + memory_hints: &wgt::MemoryHints, + memory_budget_thresholds: wgt::MemoryBudgetThresholds, + ) -> Result { + let allocation_sizes = AllocationSizes::from_memory_hints(memory_hints); + let device_memblock_size = allocation_sizes.min_device_memblock_size; + let host_memblock_size = allocation_sizes.min_host_memblock_size; + + let allocator_desc = gpu_allocator::d3d12::AllocatorCreateDesc { + device: gpu_allocator::d3d12::ID3D12DeviceVersion::Device(raw.clone()), + debug_settings: Default::default(), + allocation_sizes: allocation_sizes.into(), + }; + + let allocator = gpu_allocator::d3d12::Allocator::new(&allocator_desc).inspect_err(|e| { + log::error!("Failed to create d3d12 allocator, error: {e}"); + })?; + + Ok(Self { + inner: Arc::new(Mutex::new(allocator)), + device_memblock_size, + host_memblock_size, + memory_budget_thresholds, + }) + } + + pub(crate) fn generate_report(&self) -> wgt::AllocatorReport { + let mut upstream = self.inner.lock().generate_report(); + + let allocations = upstream + .allocations + .iter_mut() + .map(|alloc| wgt::AllocationReport { + name: core::mem::take(&mut alloc.name), + offset: alloc.offset, + size: alloc.size, + }) + .collect(); + + let blocks = upstream + .blocks + .iter() + .map(|block| wgt::MemoryBlockReport { + size: block.size, + allocations: block.allocations.clone(), + }) + .collect(); + + wgt::AllocatorReport { + allocations, + blocks, + total_allocated_bytes: upstream.total_allocated_bytes, + total_reserved_bytes: upstream.total_capacity_bytes, + } + } +} + +/// To allow us to construct buffers from both a `Device` and `CommandEncoder` +/// without needing each function to take a million arguments, we create a +/// borrowed context struct that contains the relevant members. +pub(crate) struct DeviceAllocationContext<'a> { + pub(crate) raw: &'a Direct3D12::ID3D12Device, + pub(crate) shared: &'a super::DeviceShared, + pub(crate) mem_allocator: &'a Allocator, + pub(crate) counters: &'a wgt::HalCounters, +} + +impl<'a> From<&'a super::Device> for DeviceAllocationContext<'a> { + fn from(device: &'a super::Device) -> Self { + Self { + raw: &device.raw, + shared: &device.shared, + mem_allocator: &device.mem_allocator, + counters: &device.counters, + } + } +} + +impl<'a> From<&'a super::CommandEncoder> for DeviceAllocationContext<'a> { + fn from(encoder: &'a super::CommandEncoder) -> Self { + Self { + raw: &encoder.device, + shared: &encoder.shared, + mem_allocator: &encoder.mem_allocator, + counters: &encoder.counters, + } + } +} + +impl<'a> DeviceAllocationContext<'a> { + /////////////////////// + // Resource Creation // + /////////////////////// + + pub(crate) fn create_buffer( + &self, + desc: &crate::BufferDescriptor, + ) -> Result<(Direct3D12::ID3D12Resource, Allocation), crate::DeviceError> { + let is_cpu_read = desc.usage.contains(wgt::BufferUses::MAP_READ); + let is_cpu_write = desc.usage.contains(wgt::BufferUses::MAP_WRITE); + + let location = match (is_cpu_read, is_cpu_write) { + (true, true) => MemoryLocation::CpuToGpu, + (true, false) => MemoryLocation::GpuToCpu, + (false, true) => MemoryLocation::CpuToGpu, + (false, false) => MemoryLocation::GpuOnly, + }; + + let raw_desc = conv::map_buffer_descriptor(desc); + let allocation_info = + self.error_if_would_oom_on_resource_allocation(&raw_desc, location)?; + + let (resource, allocation) = if self.shared.private_caps.suballocation_supported { + self.create_placed_buffer(desc, raw_desc, allocation_info, location)? + } else { + self.create_committed_buffer(raw_desc, location)? + }; + + if let Some(label) = desc.label { + resource.set_name(label)?; + } + + self.counters.buffer_memory.add(allocation.size() as isize); + + Ok((resource, allocation)) + } + + pub(crate) fn create_texture( + &self, + desc: &crate::TextureDescriptor, + raw_desc: Direct3D12::D3D12_RESOURCE_DESC, + ) -> Result<(Direct3D12::ID3D12Resource, Allocation), crate::DeviceError> { + let location = MemoryLocation::GpuOnly; + let allocation_info = + self.error_if_would_oom_on_resource_allocation(&raw_desc, location)?; + + let (resource, allocation) = if self.shared.private_caps.suballocation_supported { + self.create_placed_texture(desc, raw_desc, allocation_info, location)? + } else { + self.create_committed_texture(desc, raw_desc)? + }; + + if let Some(label) = desc.label { + resource.set_name(label)?; + } + + self.counters.texture_memory.add(allocation.size() as isize); + + Ok((resource, allocation)) + } + + pub(crate) fn create_acceleration_structure( + &self, + desc: &crate::AccelerationStructureDescriptor, + raw_desc: Direct3D12::D3D12_RESOURCE_DESC, + ) -> Result<(Direct3D12::ID3D12Resource, Allocation), crate::DeviceError> { + let location = MemoryLocation::GpuOnly; + let allocation_info = + self.error_if_would_oom_on_resource_allocation(&raw_desc, location)?; + + let (resource, allocation) = if self.shared.private_caps.suballocation_supported { + self.create_placed_acceleration_structure(desc, raw_desc, allocation_info, location)? + } else { + self.create_committed_acceleration_structure(desc, raw_desc)? + }; + + if let Some(label) = desc.label { + resource.set_name(label)?; + } + + self.counters + .acceleration_structure_memory + .add(allocation.size() as isize); + + Ok((resource, allocation)) + } + + ////////////////////////// + // Resource Destruction // + ////////////////////////// + + pub(crate) fn free_resource( + &self, + resource: Direct3D12::ID3D12Resource, + allocation: Allocation, + ) { + // Make sure the resource is released before we free the allocation. + drop(resource); + + let counter = match allocation.ty { + AllocationType::Buffer => &self.counters.buffer_memory, + AllocationType::Texture => &self.counters.texture_memory, + AllocationType::AccelerationStructure => &self.counters.acceleration_structure_memory, + }; + counter.sub(allocation.size() as isize); + + if let AllocationInner::Placed { inner } = allocation.inner { + match self.mem_allocator.inner.lock().free(inner) { + Ok(_) => (), + // TODO: Don't panic here + Err(e) => panic!("Failed to destroy dx12 {:?}, {e}", allocation.ty), + }; + } + } + + /////////////////////////////// + // Placed Resource Creation /// + /////////////////////////////// + + fn create_placed_buffer( + &self, + desc: &crate::BufferDescriptor<'_>, + raw_desc: Direct3D12::D3D12_RESOURCE_DESC, + allocation_info: Direct3D12::D3D12_RESOURCE_ALLOCATION_INFO, + location: MemoryLocation, + ) -> Result<(Direct3D12::ID3D12Resource, Allocation), crate::DeviceError> { + let name = desc.label.unwrap_or("Unlabeled buffer"); + + let mut allocator = self.mem_allocator.inner.lock(); + + let allocation_desc = AllocationCreateDesc { + name, + location, + size: allocation_info.SizeInBytes, + alignment: allocation_info.Alignment, + resource_category: gpu_allocator::d3d12::ResourceCategory::from(&raw_desc), + }; + + let allocation = allocator.allocate(&allocation_desc)?; + let mut resource = None; + unsafe { + self.raw.CreatePlacedResource( + allocation.heap(), + allocation.offset(), + &raw_desc, + Direct3D12::D3D12_RESOURCE_STATE_COMMON, + None, + &mut resource, + ) + } + .into_device_result("Placed buffer creation")?; + + let resource = resource.ok_or(crate::DeviceError::Unexpected)?; + let wrapped_allocation = Allocation::placed(allocation, AllocationType::Buffer); + + Ok((resource, wrapped_allocation)) + } + + fn create_placed_texture( + &self, + desc: &crate::TextureDescriptor<'_>, + raw_desc: Direct3D12::D3D12_RESOURCE_DESC, + allocation_info: Direct3D12::D3D12_RESOURCE_ALLOCATION_INFO, + location: MemoryLocation, + ) -> Result<(Direct3D12::ID3D12Resource, Allocation), crate::DeviceError> { + let name = desc.label.unwrap_or("Unlabeled texture"); + + let mut allocator = self.mem_allocator.inner.lock(); + + let allocation_desc = AllocationCreateDesc { + name, + location, + size: allocation_info.SizeInBytes, + alignment: allocation_info.Alignment, + resource_category: gpu_allocator::d3d12::ResourceCategory::from(&raw_desc), + }; + + let allocation = allocator.allocate(&allocation_desc)?; + let mut resource = None; + unsafe { + self.raw.CreatePlacedResource( + allocation.heap(), + allocation.offset(), + &raw_desc, + Direct3D12::D3D12_RESOURCE_STATE_COMMON, + None, // clear value + &mut resource, + ) + } + .into_device_result("Placed texture creation")?; + + let resource = resource.ok_or(crate::DeviceError::Unexpected)?; + let wrapped_allocation = Allocation::placed(allocation, AllocationType::Texture); + + Ok((resource, wrapped_allocation)) + } + + fn create_placed_acceleration_structure( + &self, + desc: &crate::AccelerationStructureDescriptor<'_>, + raw_desc: Direct3D12::D3D12_RESOURCE_DESC, + allocation_info: Direct3D12::D3D12_RESOURCE_ALLOCATION_INFO, + location: MemoryLocation, + ) -> Result<(Direct3D12::ID3D12Resource, Allocation), crate::DeviceError> { + let name = desc.label.unwrap_or("Unlabeled acceleration structure"); + + let mut allocator = self.mem_allocator.inner.lock(); + + let allocation_desc = AllocationCreateDesc { + name, + location, + size: allocation_info.SizeInBytes, + alignment: allocation_info.Alignment, + resource_category: gpu_allocator::d3d12::ResourceCategory::from(&raw_desc), + }; + + let allocation = allocator.allocate(&allocation_desc)?; + let mut resource = None; + unsafe { + self.raw.CreatePlacedResource( + allocation.heap(), + allocation.offset(), + &raw_desc, + Direct3D12::D3D12_RESOURCE_STATE_RAYTRACING_ACCELERATION_STRUCTURE, + None, + &mut resource, + ) + } + .into_device_result("Placed acceleration structure creation")?; + + let resource = resource.ok_or(crate::DeviceError::Unexpected)?; + let wrapped_allocation = + Allocation::placed(allocation, AllocationType::AccelerationStructure); + + Ok((resource, wrapped_allocation)) + } + + ///////////////////////////////// + // Committed Resource Creation // + ///////////////////////////////// + + fn create_committed_buffer( + &self, + raw_desc: Direct3D12::D3D12_RESOURCE_DESC, + location: MemoryLocation, + ) -> Result<(Direct3D12::ID3D12Resource, Allocation), crate::DeviceError> { + let is_uma = matches!( + self.shared.private_caps.memory_architecture, + crate::dx12::MemoryArchitecture::Unified { .. } + ); + + let heap_properties = Direct3D12::D3D12_HEAP_PROPERTIES { + Type: Direct3D12::D3D12_HEAP_TYPE_CUSTOM, + CPUPageProperty: match location { + MemoryLocation::GpuOnly => Direct3D12::D3D12_CPU_PAGE_PROPERTY_NOT_AVAILABLE, + MemoryLocation::CpuToGpu => Direct3D12::D3D12_CPU_PAGE_PROPERTY_WRITE_COMBINE, + MemoryLocation::GpuToCpu => Direct3D12::D3D12_CPU_PAGE_PROPERTY_WRITE_BACK, + _ => unreachable!(), + }, + MemoryPoolPreference: match (is_uma, location) { + // On dedicated GPUs, we only use L1 for GPU-only allocations. + (false, MemoryLocation::GpuOnly) => Direct3D12::D3D12_MEMORY_POOL_L1, + (_, _) => Direct3D12::D3D12_MEMORY_POOL_L0, + }, + CreationNodeMask: 0, + VisibleNodeMask: 0, + }; + + let mut resource = None; + + unsafe { + self.raw.CreateCommittedResource( + &heap_properties, + if self.shared.private_caps.heap_create_not_zeroed { + Direct3D12::D3D12_HEAP_FLAG_CREATE_NOT_ZEROED + } else { + Direct3D12::D3D12_HEAP_FLAG_NONE + }, + &raw_desc, + Direct3D12::D3D12_RESOURCE_STATE_COMMON, + None, + &mut resource, + ) + } + .into_device_result("Committed buffer creation")?; + + let resource = resource.ok_or(crate::DeviceError::Unexpected)?; + let wrapped_allocation = Allocation::none(AllocationType::Buffer, raw_desc.Width); + + Ok((resource, wrapped_allocation)) + } + + fn create_committed_texture( + &self, + desc: &crate::TextureDescriptor, + raw_desc: Direct3D12::D3D12_RESOURCE_DESC, + ) -> Result<(Direct3D12::ID3D12Resource, Allocation), crate::DeviceError> { + let heap_properties = Direct3D12::D3D12_HEAP_PROPERTIES { + Type: Direct3D12::D3D12_HEAP_TYPE_CUSTOM, + CPUPageProperty: Direct3D12::D3D12_CPU_PAGE_PROPERTY_NOT_AVAILABLE, + MemoryPoolPreference: match self.shared.private_caps.memory_architecture { + crate::dx12::MemoryArchitecture::NonUnified => Direct3D12::D3D12_MEMORY_POOL_L1, + crate::dx12::MemoryArchitecture::Unified { .. } => Direct3D12::D3D12_MEMORY_POOL_L0, + }, + CreationNodeMask: 0, + VisibleNodeMask: 0, + }; + + let mut resource = None; + + unsafe { + self.raw.CreateCommittedResource( + &heap_properties, + if self.shared.private_caps.heap_create_not_zeroed { + Direct3D12::D3D12_HEAP_FLAG_CREATE_NOT_ZEROED + } else { + Direct3D12::D3D12_HEAP_FLAG_NONE + }, + &raw_desc, + Direct3D12::D3D12_RESOURCE_STATE_COMMON, + None, // clear value + &mut resource, + ) + } + .into_device_result("Committed texture creation")?; + + let resource = resource.ok_or(crate::DeviceError::Unexpected)?; + let wrapped_allocation = Allocation::none( + AllocationType::Texture, + desc.format.theoretical_memory_footprint(desc.size), + ); + + Ok((resource, wrapped_allocation)) + } + + fn create_committed_acceleration_structure( + &self, + desc: &crate::AccelerationStructureDescriptor, + raw_desc: Direct3D12::D3D12_RESOURCE_DESC, + ) -> Result<(Direct3D12::ID3D12Resource, Allocation), crate::DeviceError> { + let heap_properties = Direct3D12::D3D12_HEAP_PROPERTIES { + Type: Direct3D12::D3D12_HEAP_TYPE_CUSTOM, + CPUPageProperty: Direct3D12::D3D12_CPU_PAGE_PROPERTY_NOT_AVAILABLE, + MemoryPoolPreference: match self.shared.private_caps.memory_architecture { + crate::dx12::MemoryArchitecture::NonUnified => Direct3D12::D3D12_MEMORY_POOL_L1, + crate::dx12::MemoryArchitecture::Unified { .. } => Direct3D12::D3D12_MEMORY_POOL_L0, + }, + CreationNodeMask: 0, + VisibleNodeMask: 0, + }; + + let mut resource = None; + + unsafe { + self.raw.CreateCommittedResource( + &heap_properties, + if self.shared.private_caps.heap_create_not_zeroed { + Direct3D12::D3D12_HEAP_FLAG_CREATE_NOT_ZEROED + } else { + Direct3D12::D3D12_HEAP_FLAG_NONE + }, + &raw_desc, + Direct3D12::D3D12_RESOURCE_STATE_RAYTRACING_ACCELERATION_STRUCTURE, + None, + &mut resource, + ) + } + .into_device_result("Committed acceleration structure creation")?; + + let resource = resource.ok_or(crate::DeviceError::Unexpected)?; + let wrapped_allocation = Allocation::none(AllocationType::AccelerationStructure, desc.size); + + Ok((resource, wrapped_allocation)) + } + + fn error_if_would_oom_on_resource_allocation( + &self, + desc: &Direct3D12::D3D12_RESOURCE_DESC, + location: MemoryLocation, + ) -> Result { + let allocation_info = unsafe { + self.raw + .GetResourceAllocationInfo(0, core::slice::from_ref(desc)) + }; + + // Some versions of WARP return SizeInBytes == 0 for very large + // allocations. Proceeding to attempt to allocate a zero-sized resource + // will result in a device lost error, so it seems preferable to return + // an out of memory error now. + if allocation_info.SizeInBytes == 0 { + return Err(crate::DeviceError::OutOfMemory); + } + + let Some(threshold) = self + .mem_allocator + .memory_budget_thresholds + .for_resource_creation + else { + return Ok(allocation_info); + }; + + let memory_segment_group = match location { + MemoryLocation::Unknown => unreachable!(), + MemoryLocation::GpuOnly => Dxgi::DXGI_MEMORY_SEGMENT_GROUP_LOCAL, + MemoryLocation::CpuToGpu | MemoryLocation::GpuToCpu => { + match self.shared.private_caps.memory_architecture { + super::MemoryArchitecture::Unified { .. } => { + Dxgi::DXGI_MEMORY_SEGMENT_GROUP_LOCAL + } + super::MemoryArchitecture::NonUnified => { + Dxgi::DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL + } + } + } + }; + + let info = self + .shared + .adapter + .query_video_memory_info(memory_segment_group)?; + + let memblock_size = match location { + MemoryLocation::Unknown => unreachable!(), + MemoryLocation::GpuOnly => self.mem_allocator.device_memblock_size, + MemoryLocation::CpuToGpu | MemoryLocation::GpuToCpu => { + self.mem_allocator.host_memblock_size + } + }; + + if info + .CurrentUsage + .checked_add(allocation_info.SizeInBytes.max(memblock_size)) + .is_none_or(|usage| usage >= info.Budget / 100 * threshold as u64) + { + return Err(crate::DeviceError::OutOfMemory); + } + + Ok(allocation_info) + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/types.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/types.rs new file mode 100644 index 00000000..5270c6ca --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/types.rs @@ -0,0 +1,39 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +use windows::Win32::Graphics::Dxgi; + +windows_core::imp::define_interface!( + ISwapChainPanelNative, + ISwapChainPanelNative_Vtbl, + 0x63aad0b8_7c24_40ff_85a8_640d944cc325 +); +impl core::ops::Deref for ISwapChainPanelNative { + type Target = windows_core::IUnknown; + fn deref(&self) -> &Self::Target { + unsafe { core::mem::transmute(self) } + } +} +windows_core::imp::interface_hierarchy!(ISwapChainPanelNative, windows_core::IUnknown); +impl ISwapChainPanelNative { + pub unsafe fn SetSwapChain(&self, swap_chain: P0) -> windows_core::Result<()> + where + P0: windows_core::Param, + { + unsafe { + (windows_core::Interface::vtable(self).SetSwapChain)( + windows_core::Interface::as_raw(self), + swap_chain.param().abi(), + ) + } + .ok() + } +} +#[repr(C)] +pub struct ISwapChainPanelNative_Vtbl { + pub base__: windows_core::IUnknown_Vtbl, + pub SetSwapChain: unsafe extern "system" fn( + swap_chain_panel_native: *mut core::ffi::c_void, + swap_chain: *mut core::ffi::c_void, + ) -> windows_core::HRESULT, +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dx12/view.rs b/third_party/wgpu-hal-29.0.4-patched/src/dx12/view.rs new file mode 100644 index 00000000..e541ac23 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dx12/view.rs @@ -0,0 +1,340 @@ +use windows::Win32::Graphics::{Direct3D12, Dxgi}; + +use crate::auxil; + +pub(super) struct ViewDescriptor { + dimension: wgt::TextureViewDimension, + pub aspects: crate::FormatAspects, + pub rtv_dsv_format: Dxgi::Common::DXGI_FORMAT, + srv_uav_format: Option, + multisampled: bool, + array_layer_base: u32, + array_layer_count: u32, + mip_level_base: u32, + mip_level_count: u32, +} + +impl crate::TextureViewDescriptor<'_> { + pub(super) fn to_internal(&self, texture: &super::Texture) -> ViewDescriptor { + let aspects = crate::FormatAspects::new(texture.format, self.range.aspect); + + ViewDescriptor { + dimension: self.dimension, + aspects, + rtv_dsv_format: auxil::dxgi::conv::map_texture_format(self.format), + srv_uav_format: auxil::dxgi::conv::map_texture_format_for_srv_uav(self.format, aspects), + multisampled: texture.sample_count > 1, + mip_level_base: self.range.base_mip_level, + mip_level_count: self.range.mip_level_count.unwrap_or(!0), + array_layer_base: self.range.base_array_layer, + array_layer_count: self.range.array_layer_count.unwrap_or(!0), + } + } +} + +fn aspects_to_plane(aspects: crate::FormatAspects) -> u32 { + match aspects { + crate::FormatAspects::STENCIL => 1, + crate::FormatAspects::PLANE_1 => 1, + crate::FormatAspects::PLANE_2 => 2, + _ => 0, + } +} + +impl ViewDescriptor { + pub(crate) unsafe fn to_srv(&self) -> Option { + let mut desc = Direct3D12::D3D12_SHADER_RESOURCE_VIEW_DESC { + Format: self.srv_uav_format?, + ViewDimension: Direct3D12::D3D12_SRV_DIMENSION_UNKNOWN, + Shader4ComponentMapping: Direct3D12::D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING, + Anonymous: Default::default(), + }; + + match self.dimension { + wgt::TextureViewDimension::D1 => { + desc.ViewDimension = Direct3D12::D3D12_SRV_DIMENSION_TEXTURE1D; + desc.Anonymous.Texture1D = Direct3D12::D3D12_TEX1D_SRV { + MostDetailedMip: self.mip_level_base, + MipLevels: self.mip_level_count, + ResourceMinLODClamp: 0.0, + } + } + /* + wgt::TextureViewDimension::D1Array => { + desc.ViewDimension = Direct3D12::D3D12_SRV_DIMENSION_TEXTURE1DARRAY; + desc.Anonymous.Texture1DArray = Direct3D12::D3D12_TEX1D_ARRAY_SRV { + MostDetailedMip: self.mip_level_base, + MipLevels: self.mip_level_count, + FirstArraySlice: self.array_layer_base, + ArraySize: self.array_layer_count, + ResourceMinLODClamp: 0.0, + } + } + */ + wgt::TextureViewDimension::D2 if self.multisampled && self.array_layer_base == 0 => { + desc.ViewDimension = Direct3D12::D3D12_SRV_DIMENSION_TEXTURE2DMS; + desc.Anonymous.Texture2DMS = Direct3D12::D3D12_TEX2DMS_SRV { + UnusedField_NothingToDefine: 0, + } + } + wgt::TextureViewDimension::D2 if self.array_layer_base == 0 => { + desc.ViewDimension = Direct3D12::D3D12_SRV_DIMENSION_TEXTURE2D; + desc.Anonymous.Texture2D = Direct3D12::D3D12_TEX2D_SRV { + MostDetailedMip: self.mip_level_base, + MipLevels: self.mip_level_count, + PlaneSlice: aspects_to_plane(self.aspects), + ResourceMinLODClamp: 0.0, + } + } + wgt::TextureViewDimension::D2 | wgt::TextureViewDimension::D2Array + if self.multisampled => + { + desc.ViewDimension = Direct3D12::D3D12_SRV_DIMENSION_TEXTURE2DMSARRAY; + desc.Anonymous.Texture2DMSArray = Direct3D12::D3D12_TEX2DMS_ARRAY_SRV { + FirstArraySlice: self.array_layer_base, + ArraySize: self.array_layer_count, + } + } + wgt::TextureViewDimension::D2 | wgt::TextureViewDimension::D2Array => { + desc.ViewDimension = Direct3D12::D3D12_SRV_DIMENSION_TEXTURE2DARRAY; + desc.Anonymous.Texture2DArray = Direct3D12::D3D12_TEX2D_ARRAY_SRV { + MostDetailedMip: self.mip_level_base, + MipLevels: self.mip_level_count, + FirstArraySlice: self.array_layer_base, + ArraySize: self.array_layer_count, + PlaneSlice: aspects_to_plane(self.aspects), + ResourceMinLODClamp: 0.0, + } + } + wgt::TextureViewDimension::D3 => { + desc.ViewDimension = Direct3D12::D3D12_SRV_DIMENSION_TEXTURE3D; + desc.Anonymous.Texture3D = Direct3D12::D3D12_TEX3D_SRV { + MostDetailedMip: self.mip_level_base, + MipLevels: self.mip_level_count, + ResourceMinLODClamp: 0.0, + } + } + wgt::TextureViewDimension::Cube if self.array_layer_base == 0 => { + desc.ViewDimension = Direct3D12::D3D12_SRV_DIMENSION_TEXTURECUBE; + desc.Anonymous.TextureCube = Direct3D12::D3D12_TEXCUBE_SRV { + MostDetailedMip: self.mip_level_base, + MipLevels: self.mip_level_count, + ResourceMinLODClamp: 0.0, + } + } + wgt::TextureViewDimension::Cube | wgt::TextureViewDimension::CubeArray => { + desc.ViewDimension = Direct3D12::D3D12_SRV_DIMENSION_TEXTURECUBEARRAY; + desc.Anonymous.TextureCubeArray = Direct3D12::D3D12_TEXCUBE_ARRAY_SRV { + MostDetailedMip: self.mip_level_base, + MipLevels: self.mip_level_count, + First2DArrayFace: self.array_layer_base, + NumCubes: if self.array_layer_count == !0 { + !0 + } else { + self.array_layer_count / 6 + }, + ResourceMinLODClamp: 0.0, + } + } + } + + Some(desc) + } + + pub(crate) unsafe fn to_uav(&self) -> Option { + let mut desc = Direct3D12::D3D12_UNORDERED_ACCESS_VIEW_DESC { + Format: self.srv_uav_format?, + ViewDimension: Direct3D12::D3D12_UAV_DIMENSION_UNKNOWN, + Anonymous: Default::default(), + }; + + match self.dimension { + wgt::TextureViewDimension::D1 => { + desc.ViewDimension = Direct3D12::D3D12_UAV_DIMENSION_TEXTURE1D; + desc.Anonymous.Texture1D = Direct3D12::D3D12_TEX1D_UAV { + MipSlice: self.mip_level_base, + } + } + /* + wgt::TextureViewDimension::D1Array => { + desc.ViewDimension = Direct3D12::D3D12_UAV_DIMENSION_TEXTURE1DARRAY; + desc.Anonymous.Texture1DArray = Direct3D12::D3D12_TEX1D_ARRAY_UAV { + MipSlice: self.mip_level_base, + FirstArraySlice: self.array_layer_base, + ArraySize, + } + }*/ + wgt::TextureViewDimension::D2 if self.array_layer_base == 0 => { + desc.ViewDimension = Direct3D12::D3D12_UAV_DIMENSION_TEXTURE2D; + desc.Anonymous.Texture2D = Direct3D12::D3D12_TEX2D_UAV { + MipSlice: self.mip_level_base, + PlaneSlice: aspects_to_plane(self.aspects), + } + } + wgt::TextureViewDimension::D2 | wgt::TextureViewDimension::D2Array => { + desc.ViewDimension = Direct3D12::D3D12_UAV_DIMENSION_TEXTURE2DARRAY; + desc.Anonymous.Texture2DArray = Direct3D12::D3D12_TEX2D_ARRAY_UAV { + MipSlice: self.mip_level_base, + FirstArraySlice: self.array_layer_base, + ArraySize: self.array_layer_count, + PlaneSlice: aspects_to_plane(self.aspects), + } + } + wgt::TextureViewDimension::D3 => { + desc.ViewDimension = Direct3D12::D3D12_UAV_DIMENSION_TEXTURE3D; + desc.Anonymous.Texture3D = Direct3D12::D3D12_TEX3D_UAV { + MipSlice: self.mip_level_base, + FirstWSlice: 0, + WSize: u32::MAX, + } + } + wgt::TextureViewDimension::Cube | wgt::TextureViewDimension::CubeArray => { + panic!("Unable to view texture as cube UAV") + } + } + + Some(desc) + } + + pub(crate) unsafe fn to_rtv(&self) -> Direct3D12::D3D12_RENDER_TARGET_VIEW_DESC { + let mut desc = Direct3D12::D3D12_RENDER_TARGET_VIEW_DESC { + Format: self.rtv_dsv_format, + ViewDimension: Direct3D12::D3D12_RTV_DIMENSION_UNKNOWN, + Anonymous: Default::default(), + }; + + match self.dimension { + wgt::TextureViewDimension::D1 => { + desc.ViewDimension = Direct3D12::D3D12_RTV_DIMENSION_TEXTURE1D; + desc.Anonymous.Texture1D = Direct3D12::D3D12_TEX1D_RTV { + MipSlice: self.mip_level_base, + } + } + /* + wgt::TextureViewDimension::D1Array => { + desc.ViewDimension = Direct3D12::D3D12_RTV_DIMENSION_TEXTURE1DARRAY; + desc.Anonymous.Texture1DArray = Direct3D12::D3D12_TEX1D_ARRAY_RTV { + MipSlice: self.mip_level_base, + FirstArraySlice: self.array_layer_base, + ArraySize, + } + }*/ + wgt::TextureViewDimension::D2 if self.multisampled && self.array_layer_base == 0 => { + desc.ViewDimension = Direct3D12::D3D12_RTV_DIMENSION_TEXTURE2DMS; + desc.Anonymous.Texture2DMS = Direct3D12::D3D12_TEX2DMS_RTV { + UnusedField_NothingToDefine: 0, + } + } + wgt::TextureViewDimension::D2 if self.array_layer_base == 0 => { + desc.ViewDimension = Direct3D12::D3D12_RTV_DIMENSION_TEXTURE2D; + desc.Anonymous.Texture2D = Direct3D12::D3D12_TEX2D_RTV { + MipSlice: self.mip_level_base, + PlaneSlice: aspects_to_plane(self.aspects), + } + } + wgt::TextureViewDimension::D2 | wgt::TextureViewDimension::D2Array + if self.multisampled => + { + desc.ViewDimension = Direct3D12::D3D12_RTV_DIMENSION_TEXTURE2DMSARRAY; + desc.Anonymous.Texture2DMSArray = Direct3D12::D3D12_TEX2DMS_ARRAY_RTV { + FirstArraySlice: self.array_layer_base, + ArraySize: self.array_layer_count, + } + } + wgt::TextureViewDimension::D2 | wgt::TextureViewDimension::D2Array => { + desc.ViewDimension = Direct3D12::D3D12_RTV_DIMENSION_TEXTURE2DARRAY; + desc.Anonymous.Texture2DArray = Direct3D12::D3D12_TEX2D_ARRAY_RTV { + MipSlice: self.mip_level_base, + FirstArraySlice: self.array_layer_base, + ArraySize: self.array_layer_count, + PlaneSlice: aspects_to_plane(self.aspects), + } + } + wgt::TextureViewDimension::D3 + | wgt::TextureViewDimension::Cube + | wgt::TextureViewDimension::CubeArray => { + panic!("Unable to view texture as cube or 3D RTV") + } + } + + desc + } + + pub(crate) unsafe fn to_dsv( + &self, + read_only: bool, + ) -> Direct3D12::D3D12_DEPTH_STENCIL_VIEW_DESC { + let mut desc = Direct3D12::D3D12_DEPTH_STENCIL_VIEW_DESC { + Format: self.rtv_dsv_format, + ViewDimension: Direct3D12::D3D12_DSV_DIMENSION_UNKNOWN, + Flags: { + let mut flags = Direct3D12::D3D12_DSV_FLAG_NONE; + if read_only { + if self.aspects.contains(crate::FormatAspects::DEPTH) { + flags |= Direct3D12::D3D12_DSV_FLAG_READ_ONLY_DEPTH; + } + if self.aspects.contains(crate::FormatAspects::STENCIL) { + flags |= Direct3D12::D3D12_DSV_FLAG_READ_ONLY_STENCIL; + } + } + flags + }, + Anonymous: Default::default(), + }; + + match self.dimension { + wgt::TextureViewDimension::D1 => { + desc.ViewDimension = Direct3D12::D3D12_DSV_DIMENSION_TEXTURE1D; + desc.Anonymous.Texture1D = Direct3D12::D3D12_TEX1D_DSV { + MipSlice: self.mip_level_base, + } + } + /* + wgt::TextureViewDimension::D1Array => { + desc.ViewDimension = Direct3D12::D3D12_DSV_DIMENSION_TEXTURE1DARRAY; + desc.Anonymous.Texture1DArray = Direct3D12::D3D12_TEX1D_ARRAY_DSV { + MipSlice: self.mip_level_base, + FirstArraySlice: self.array_layer_base, + ArraySize, + } + }*/ + wgt::TextureViewDimension::D2 if self.multisampled && self.array_layer_base == 0 => { + desc.ViewDimension = Direct3D12::D3D12_DSV_DIMENSION_TEXTURE2DMS; + desc.Anonymous.Texture2DMS = Direct3D12::D3D12_TEX2DMS_DSV { + UnusedField_NothingToDefine: 0, + } + } + wgt::TextureViewDimension::D2 if self.array_layer_base == 0 => { + desc.ViewDimension = Direct3D12::D3D12_DSV_DIMENSION_TEXTURE2D; + + desc.Anonymous.Texture2D = Direct3D12::D3D12_TEX2D_DSV { + MipSlice: self.mip_level_base, + } + } + wgt::TextureViewDimension::D2 | wgt::TextureViewDimension::D2Array + if self.multisampled => + { + desc.ViewDimension = Direct3D12::D3D12_DSV_DIMENSION_TEXTURE2DMSARRAY; + desc.Anonymous.Texture2DMSArray = Direct3D12::D3D12_TEX2DMS_ARRAY_DSV { + FirstArraySlice: self.array_layer_base, + ArraySize: self.array_layer_count, + } + } + wgt::TextureViewDimension::D2 | wgt::TextureViewDimension::D2Array => { + desc.ViewDimension = Direct3D12::D3D12_DSV_DIMENSION_TEXTURE2DARRAY; + desc.Anonymous.Texture2DArray = Direct3D12::D3D12_TEX2D_ARRAY_DSV { + MipSlice: self.mip_level_base, + FirstArraySlice: self.array_layer_base, + ArraySize: self.array_layer_count, + } + } + wgt::TextureViewDimension::D3 + | wgt::TextureViewDimension::Cube + | wgt::TextureViewDimension::CubeArray => { + panic!("Unable to view texture as cube or 3D DSV") + } + } + + desc + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dynamic/adapter.rs b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/adapter.rs new file mode 100644 index 00000000..6256a331 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/adapter.rs @@ -0,0 +1,81 @@ +use alloc::boxed::Box; + +use crate::{ + Adapter, Api, DeviceError, OpenDevice, SurfaceCapabilities, TextureFormatCapabilities, +}; + +use super::{DynDevice, DynQueue, DynResource, DynResourceExt, DynSurface}; + +pub struct DynOpenDevice { + pub device: Box, + pub queue: Box, +} + +impl From> for DynOpenDevice { + fn from(open_device: OpenDevice) -> Self { + Self { + device: Box::new(open_device.device), + queue: Box::new(open_device.queue), + } + } +} + +pub trait DynAdapter: DynResource { + unsafe fn open( + &self, + features: wgt::Features, + limits: &wgt::Limits, + memory_hints: &wgt::MemoryHints, + ) -> Result; + + unsafe fn texture_format_capabilities( + &self, + format: wgt::TextureFormat, + ) -> TextureFormatCapabilities; + + unsafe fn surface_capabilities(&self, surface: &dyn DynSurface) -> Option; + + unsafe fn get_presentation_timestamp(&self) -> wgt::PresentationTimestamp; + + fn get_ordered_buffer_usages(&self) -> wgt::BufferUses; + + fn get_ordered_texture_usages(&self) -> wgt::TextureUses; +} + +impl DynAdapter for A { + unsafe fn open( + &self, + features: wgt::Features, + limits: &wgt::Limits, + memory_hints: &wgt::MemoryHints, + ) -> Result { + unsafe { A::open(self, features, limits, memory_hints) }.map(|open_device| DynOpenDevice { + device: Box::new(open_device.device), + queue: Box::new(open_device.queue), + }) + } + + unsafe fn texture_format_capabilities( + &self, + format: wgt::TextureFormat, + ) -> TextureFormatCapabilities { + unsafe { A::texture_format_capabilities(self, format) } + } + + unsafe fn surface_capabilities(&self, surface: &dyn DynSurface) -> Option { + let surface = surface.expect_downcast_ref(); + unsafe { A::surface_capabilities(self, surface) } + } + + unsafe fn get_presentation_timestamp(&self) -> wgt::PresentationTimestamp { + unsafe { A::get_presentation_timestamp(self) } + } + + fn get_ordered_buffer_usages(&self) -> wgt::BufferUses { + A::get_ordered_buffer_usages(self) + } + + fn get_ordered_texture_usages(&self) -> wgt::TextureUses { + A::get_ordered_texture_usages(self) + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dynamic/command.rs b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/command.rs new file mode 100644 index 00000000..6c99970b --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/command.rs @@ -0,0 +1,761 @@ +use alloc::{boxed::Box, vec::Vec}; +use core::ops::Range; + +use crate::{ + AccelerationStructureBarrier, Api, Attachment, BufferBarrier, BufferBinding, BufferCopy, + BufferTextureCopy, BuildAccelerationStructureDescriptor, ColorAttachment, CommandEncoder, + ComputePassDescriptor, DepthStencilAttachment, DeviceError, Label, MemoryRange, + PassTimestampWrites, Rect, RenderPassDescriptor, TextureBarrier, TextureCopy, +}; + +use super::{ + DynAccelerationStructure, DynBindGroup, DynBuffer, DynCommandBuffer, DynComputePipeline, + DynPipelineLayout, DynQuerySet, DynRenderPipeline, DynResource, DynResourceExt as _, + DynTexture, DynTextureView, +}; + +pub trait DynCommandEncoder: DynResource + core::fmt::Debug { + unsafe fn begin_encoding(&mut self, label: Label) -> Result<(), DeviceError>; + + unsafe fn discard_encoding(&mut self); + + unsafe fn end_encoding(&mut self) -> Result, DeviceError>; + + unsafe fn reset_all(&mut self, command_buffers: Vec>); + + unsafe fn transition_buffers(&mut self, barriers: &[BufferBarrier<'_, dyn DynBuffer>]); + unsafe fn transition_textures(&mut self, barriers: &[TextureBarrier<'_, dyn DynTexture>]); + + unsafe fn clear_buffer(&mut self, buffer: &dyn DynBuffer, range: MemoryRange); + + unsafe fn copy_buffer_to_buffer( + &mut self, + src: &dyn DynBuffer, + dst: &dyn DynBuffer, + regions: &[BufferCopy], + ); + + unsafe fn copy_texture_to_texture( + &mut self, + src: &dyn DynTexture, + src_usage: wgt::TextureUses, + dst: &dyn DynTexture, + regions: &[TextureCopy], + ); + + unsafe fn copy_buffer_to_texture( + &mut self, + src: &dyn DynBuffer, + dst: &dyn DynTexture, + regions: &[BufferTextureCopy], + ); + + unsafe fn copy_texture_to_buffer( + &mut self, + src: &dyn DynTexture, + src_usage: wgt::TextureUses, + dst: &dyn DynBuffer, + regions: &[BufferTextureCopy], + ); + + unsafe fn set_bind_group( + &mut self, + layout: &dyn DynPipelineLayout, + index: u32, + group: &dyn DynBindGroup, + dynamic_offsets: &[wgt::DynamicOffset], + ); + + unsafe fn set_immediates( + &mut self, + layout: &dyn DynPipelineLayout, + offset_bytes: u32, + data: &[u32], + ); + + unsafe fn insert_debug_marker(&mut self, label: &str); + unsafe fn begin_debug_marker(&mut self, group_label: &str); + unsafe fn end_debug_marker(&mut self); + + unsafe fn begin_query(&mut self, set: &dyn DynQuerySet, index: u32); + unsafe fn end_query(&mut self, set: &dyn DynQuerySet, index: u32); + unsafe fn write_timestamp(&mut self, set: &dyn DynQuerySet, index: u32); + unsafe fn reset_queries(&mut self, set: &dyn DynQuerySet, range: Range); + unsafe fn copy_query_results( + &mut self, + set: &dyn DynQuerySet, + range: Range, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + stride: wgt::BufferSize, + ); + + unsafe fn begin_render_pass( + &mut self, + desc: &RenderPassDescriptor, + ) -> Result<(), DeviceError>; + unsafe fn end_render_pass(&mut self); + + unsafe fn set_render_pipeline(&mut self, pipeline: &dyn DynRenderPipeline); + + unsafe fn set_index_buffer<'a>( + &mut self, + binding: BufferBinding<'a, dyn DynBuffer>, + format: wgt::IndexFormat, + ); + + unsafe fn set_vertex_buffer<'a>( + &mut self, + index: u32, + binding: BufferBinding<'a, dyn DynBuffer>, + ); + unsafe fn set_viewport(&mut self, rect: &Rect, depth_range: Range); + unsafe fn set_scissor_rect(&mut self, rect: &Rect); + unsafe fn set_stencil_reference(&mut self, value: u32); + unsafe fn set_blend_constants(&mut self, color: &[f32; 4]); + + unsafe fn draw( + &mut self, + first_vertex: u32, + vertex_count: u32, + first_instance: u32, + instance_count: u32, + ); + unsafe fn draw_indexed( + &mut self, + first_index: u32, + index_count: u32, + base_vertex: i32, + first_instance: u32, + instance_count: u32, + ); + unsafe fn draw_mesh_tasks( + &mut self, + group_count_x: u32, + group_count_y: u32, + group_count_z: u32, + ); + unsafe fn draw_indirect( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + draw_count: u32, + ); + unsafe fn draw_indexed_indirect( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + draw_count: u32, + ); + unsafe fn draw_mesh_tasks_indirect( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + draw_count: u32, + ); + unsafe fn draw_indirect_count( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + count_buffer: &dyn DynBuffer, + count_offset: wgt::BufferAddress, + max_count: u32, + ); + unsafe fn draw_indexed_indirect_count( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + count_buffer: &dyn DynBuffer, + count_offset: wgt::BufferAddress, + max_count: u32, + ); + unsafe fn draw_mesh_tasks_indirect_count( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + count_buffer: &dyn DynBuffer, + count_offset: wgt::BufferAddress, + max_count: u32, + ); + + unsafe fn begin_compute_pass(&mut self, desc: &ComputePassDescriptor); + unsafe fn end_compute_pass(&mut self); + + unsafe fn set_compute_pipeline(&mut self, pipeline: &dyn DynComputePipeline); + + unsafe fn dispatch(&mut self, count: [u32; 3]); + unsafe fn dispatch_indirect(&mut self, buffer: &dyn DynBuffer, offset: wgt::BufferAddress); + + unsafe fn build_acceleration_structures<'a>( + &mut self, + descriptors: &'a [BuildAccelerationStructureDescriptor< + 'a, + dyn DynBuffer, + dyn DynAccelerationStructure, + >], + ); + unsafe fn place_acceleration_structure_barrier( + &mut self, + barrier: AccelerationStructureBarrier, + ); + unsafe fn copy_acceleration_structure_to_acceleration_structure( + &mut self, + src: &dyn DynAccelerationStructure, + dst: &dyn DynAccelerationStructure, + copy: wgt::AccelerationStructureCopy, + ); + unsafe fn read_acceleration_structure_compact_size( + &mut self, + acceleration_structure: &dyn DynAccelerationStructure, + buf: &dyn DynBuffer, + ); + unsafe fn set_acceleration_structure_dependencies( + &self, + command_buffers: &[Box], + dependencies: &[&dyn DynAccelerationStructure], + ); +} + +impl DynCommandEncoder for C { + unsafe fn begin_encoding(&mut self, label: Label) -> Result<(), DeviceError> { + unsafe { C::begin_encoding(self, label) } + } + + unsafe fn discard_encoding(&mut self) { + unsafe { C::discard_encoding(self) } + } + + unsafe fn end_encoding(&mut self) -> Result, DeviceError> { + unsafe { C::end_encoding(self) }.map(|cb| { + let boxed_command_buffer: Box<::CommandBuffer> = Box::new(cb); + let boxed_command_buffer: Box = boxed_command_buffer; + boxed_command_buffer + }) + } + + unsafe fn reset_all(&mut self, command_buffers: Vec>) { + unsafe { C::reset_all(self, command_buffers.into_iter().map(|cb| cb.unbox())) } + } + + unsafe fn transition_buffers(&mut self, barriers: &[BufferBarrier<'_, dyn DynBuffer>]) { + let barriers = barriers.iter().map(|barrier| BufferBarrier { + buffer: barrier.buffer.expect_downcast_ref(), + usage: barrier.usage.clone(), + }); + unsafe { self.transition_buffers(barriers) }; + } + + unsafe fn transition_textures(&mut self, barriers: &[TextureBarrier<'_, dyn DynTexture>]) { + let barriers = barriers.iter().map(|barrier| TextureBarrier { + texture: barrier.texture.expect_downcast_ref(), + usage: barrier.usage.clone(), + range: barrier.range, + }); + unsafe { self.transition_textures(barriers) }; + } + + unsafe fn clear_buffer(&mut self, buffer: &dyn DynBuffer, range: MemoryRange) { + let buffer = buffer.expect_downcast_ref(); + unsafe { C::clear_buffer(self, buffer, range) }; + } + + unsafe fn copy_buffer_to_buffer( + &mut self, + src: &dyn DynBuffer, + dst: &dyn DynBuffer, + regions: &[BufferCopy], + ) { + let src = src.expect_downcast_ref(); + let dst = dst.expect_downcast_ref(); + unsafe { + C::copy_buffer_to_buffer(self, src, dst, regions.iter().copied()); + } + } + + unsafe fn copy_texture_to_texture( + &mut self, + src: &dyn DynTexture, + src_usage: wgt::TextureUses, + dst: &dyn DynTexture, + regions: &[TextureCopy], + ) { + let src = src.expect_downcast_ref(); + let dst = dst.expect_downcast_ref(); + unsafe { + C::copy_texture_to_texture(self, src, src_usage, dst, regions.iter().cloned()); + } + } + + unsafe fn copy_buffer_to_texture( + &mut self, + src: &dyn DynBuffer, + dst: &dyn DynTexture, + regions: &[BufferTextureCopy], + ) { + let src = src.expect_downcast_ref(); + let dst = dst.expect_downcast_ref(); + unsafe { + C::copy_buffer_to_texture(self, src, dst, regions.iter().cloned()); + } + } + + unsafe fn copy_texture_to_buffer( + &mut self, + src: &dyn DynTexture, + src_usage: wgt::TextureUses, + dst: &dyn DynBuffer, + regions: &[BufferTextureCopy], + ) { + let src = src.expect_downcast_ref(); + let dst = dst.expect_downcast_ref(); + unsafe { + C::copy_texture_to_buffer(self, src, src_usage, dst, regions.iter().cloned()); + } + } + + unsafe fn set_bind_group( + &mut self, + layout: &dyn DynPipelineLayout, + index: u32, + group: &dyn DynBindGroup, + dynamic_offsets: &[wgt::DynamicOffset], + ) { + let layout = layout.expect_downcast_ref(); + let group = group.expect_downcast_ref(); + unsafe { C::set_bind_group(self, layout, index, group, dynamic_offsets) }; + } + + unsafe fn set_immediates( + &mut self, + layout: &dyn DynPipelineLayout, + offset_bytes: u32, + data: &[u32], + ) { + let layout = layout.expect_downcast_ref(); + unsafe { C::set_immediates(self, layout, offset_bytes, data) }; + } + + unsafe fn insert_debug_marker(&mut self, label: &str) { + unsafe { + C::insert_debug_marker(self, label); + } + } + + unsafe fn begin_debug_marker(&mut self, group_label: &str) { + unsafe { + C::begin_debug_marker(self, group_label); + } + } + + unsafe fn end_debug_marker(&mut self) { + unsafe { + C::end_debug_marker(self); + } + } + + unsafe fn begin_query(&mut self, set: &dyn DynQuerySet, index: u32) { + let set = set.expect_downcast_ref(); + unsafe { C::begin_query(self, set, index) }; + } + + unsafe fn end_query(&mut self, set: &dyn DynQuerySet, index: u32) { + let set = set.expect_downcast_ref(); + unsafe { C::end_query(self, set, index) }; + } + + unsafe fn write_timestamp(&mut self, set: &dyn DynQuerySet, index: u32) { + let set = set.expect_downcast_ref(); + unsafe { C::write_timestamp(self, set, index) }; + } + + unsafe fn reset_queries(&mut self, set: &dyn DynQuerySet, range: Range) { + let set = set.expect_downcast_ref(); + unsafe { C::reset_queries(self, set, range) }; + } + + unsafe fn copy_query_results( + &mut self, + set: &dyn DynQuerySet, + range: Range, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + stride: wgt::BufferSize, + ) { + let set = set.expect_downcast_ref(); + let buffer = buffer.expect_downcast_ref(); + unsafe { C::copy_query_results(self, set, range, buffer, offset, stride) }; + } + + unsafe fn begin_render_pass( + &mut self, + desc: &RenderPassDescriptor, + ) -> Result<(), DeviceError> { + let color_attachments = desc + .color_attachments + .iter() + .map(|attachment| { + attachment + .as_ref() + .map(|attachment| attachment.expect_downcast()) + }) + .collect::>(); + + let desc: RenderPassDescriptor<::QuerySet, ::TextureView> = + RenderPassDescriptor { + label: desc.label, + extent: desc.extent, + sample_count: desc.sample_count, + color_attachments: &color_attachments, + depth_stencil_attachment: desc + .depth_stencil_attachment + .as_ref() + .map(|ds| ds.expect_downcast()), + multiview_mask: desc.multiview_mask, + timestamp_writes: desc + .timestamp_writes + .as_ref() + .map(|writes| writes.expect_downcast()), + occlusion_query_set: desc + .occlusion_query_set + .map(|set| set.expect_downcast_ref()), + }; + unsafe { C::begin_render_pass(self, &desc) } + } + + unsafe fn end_render_pass(&mut self) { + unsafe { + C::end_render_pass(self); + } + } + + unsafe fn set_viewport(&mut self, rect: &Rect, depth_range: Range) { + unsafe { + C::set_viewport(self, rect, depth_range); + } + } + + unsafe fn set_scissor_rect(&mut self, rect: &Rect) { + unsafe { + C::set_scissor_rect(self, rect); + } + } + + unsafe fn set_stencil_reference(&mut self, value: u32) { + unsafe { + C::set_stencil_reference(self, value); + } + } + + unsafe fn set_blend_constants(&mut self, color: &[f32; 4]) { + unsafe { C::set_blend_constants(self, color) }; + } + + unsafe fn draw( + &mut self, + first_vertex: u32, + vertex_count: u32, + first_instance: u32, + instance_count: u32, + ) { + unsafe { + C::draw( + self, + first_vertex, + vertex_count, + first_instance, + instance_count, + ) + }; + } + + unsafe fn draw_indexed( + &mut self, + first_index: u32, + index_count: u32, + base_vertex: i32, + first_instance: u32, + instance_count: u32, + ) { + unsafe { + C::draw_indexed( + self, + first_index, + index_count, + base_vertex, + first_instance, + instance_count, + ) + }; + } + + unsafe fn draw_mesh_tasks( + &mut self, + group_count_x: u32, + group_count_y: u32, + group_count_z: u32, + ) { + unsafe { C::draw_mesh_tasks(self, group_count_x, group_count_y, group_count_z) }; + } + + unsafe fn draw_indirect( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + draw_count: u32, + ) { + let buffer = buffer.expect_downcast_ref(); + unsafe { C::draw_indirect(self, buffer, offset, draw_count) }; + } + + unsafe fn draw_indexed_indirect( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + draw_count: u32, + ) { + let buffer = buffer.expect_downcast_ref(); + unsafe { C::draw_indexed_indirect(self, buffer, offset, draw_count) }; + } + + unsafe fn draw_mesh_tasks_indirect( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + draw_count: u32, + ) { + let buffer = buffer.expect_downcast_ref(); + unsafe { C::draw_mesh_tasks_indirect(self, buffer, offset, draw_count) }; + } + + unsafe fn draw_indirect_count( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + count_buffer: &dyn DynBuffer, + count_offset: wgt::BufferAddress, + max_count: u32, + ) { + let buffer = buffer.expect_downcast_ref(); + let count_buffer = count_buffer.expect_downcast_ref(); + unsafe { + C::draw_indirect_count(self, buffer, offset, count_buffer, count_offset, max_count) + }; + } + + unsafe fn draw_indexed_indirect_count( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + count_buffer: &dyn DynBuffer, + count_offset: wgt::BufferAddress, + max_count: u32, + ) { + let buffer = buffer.expect_downcast_ref(); + let count_buffer = count_buffer.expect_downcast_ref(); + unsafe { + C::draw_indexed_indirect_count( + self, + buffer, + offset, + count_buffer, + count_offset, + max_count, + ) + }; + } + + unsafe fn draw_mesh_tasks_indirect_count( + &mut self, + buffer: &dyn DynBuffer, + offset: wgt::BufferAddress, + count_buffer: &dyn DynBuffer, + count_offset: wgt::BufferAddress, + max_count: u32, + ) { + let buffer = buffer.expect_downcast_ref(); + let count_buffer = count_buffer.expect_downcast_ref(); + unsafe { + C::draw_mesh_tasks_indirect_count( + self, + buffer, + offset, + count_buffer, + count_offset, + max_count, + ) + }; + } + + unsafe fn begin_compute_pass(&mut self, desc: &ComputePassDescriptor) { + let desc = ComputePassDescriptor { + label: desc.label, + timestamp_writes: desc + .timestamp_writes + .as_ref() + .map(|writes| writes.expect_downcast()), + }; + unsafe { C::begin_compute_pass(self, &desc) }; + } + + unsafe fn end_compute_pass(&mut self) { + unsafe { C::end_compute_pass(self) }; + } + + unsafe fn set_compute_pipeline(&mut self, pipeline: &dyn DynComputePipeline) { + let pipeline = pipeline.expect_downcast_ref(); + unsafe { C::set_compute_pipeline(self, pipeline) }; + } + + unsafe fn dispatch(&mut self, count: [u32; 3]) { + unsafe { C::dispatch(self, count) }; + } + + unsafe fn dispatch_indirect(&mut self, buffer: &dyn DynBuffer, offset: wgt::BufferAddress) { + let buffer = buffer.expect_downcast_ref(); + unsafe { C::dispatch_indirect(self, buffer, offset) }; + } + + unsafe fn set_render_pipeline(&mut self, pipeline: &dyn DynRenderPipeline) { + let pipeline = pipeline.expect_downcast_ref(); + unsafe { C::set_render_pipeline(self, pipeline) }; + } + + unsafe fn set_index_buffer<'a>( + &mut self, + binding: BufferBinding<'a, dyn DynBuffer>, + format: wgt::IndexFormat, + ) { + let binding = binding.expect_downcast(); + unsafe { self.set_index_buffer(binding, format) }; + } + + unsafe fn set_vertex_buffer<'a>( + &mut self, + index: u32, + binding: BufferBinding<'a, dyn DynBuffer>, + ) { + let binding = binding.expect_downcast(); + unsafe { self.set_vertex_buffer(index, binding) }; + } + + unsafe fn build_acceleration_structures<'a>( + &mut self, + descriptors: &'a [BuildAccelerationStructureDescriptor< + 'a, + dyn DynBuffer, + dyn DynAccelerationStructure, + >], + ) { + // Need to collect entries here so we can reference them in the descriptor. + // TODO: API should be redesigned to avoid this and other descriptor copies that happen due to the dyn api. + let descriptor_entries = descriptors + .iter() + .map(|d| d.entries.expect_downcast()) + .collect::>(); + let descriptors = descriptors + .iter() + .zip(descriptor_entries.iter()) + .map(|(d, entries)| BuildAccelerationStructureDescriptor::< + ::Buffer, + ::AccelerationStructure, + > { + entries, + mode: d.mode, + flags: d.flags, + source_acceleration_structure: d + .source_acceleration_structure + .map(|a| a.expect_downcast_ref()), + destination_acceleration_structure: d + .destination_acceleration_structure + .expect_downcast_ref(), + scratch_buffer: d.scratch_buffer.expect_downcast_ref(), + scratch_buffer_offset: d.scratch_buffer_offset, + }); + unsafe { C::build_acceleration_structures(self, descriptors.len() as _, descriptors) }; + } + + unsafe fn place_acceleration_structure_barrier( + &mut self, + barrier: AccelerationStructureBarrier, + ) { + unsafe { C::place_acceleration_structure_barrier(self, barrier) }; + } + + unsafe fn copy_acceleration_structure_to_acceleration_structure( + &mut self, + src: &dyn DynAccelerationStructure, + dst: &dyn DynAccelerationStructure, + copy: wgt::AccelerationStructureCopy, + ) { + let src = src.expect_downcast_ref(); + let dst = dst.expect_downcast_ref(); + unsafe { C::copy_acceleration_structure_to_acceleration_structure(self, src, dst, copy) }; + } + unsafe fn read_acceleration_structure_compact_size( + &mut self, + acceleration_structure: &dyn DynAccelerationStructure, + buf: &dyn DynBuffer, + ) { + let acceleration_structure = acceleration_structure.expect_downcast_ref(); + let buf = buf.expect_downcast_ref(); + unsafe { C::read_acceleration_structure_compact_size(self, acceleration_structure, buf) } + } + + unsafe fn set_acceleration_structure_dependencies( + &self, + command_buffers: &[Box], + dependencies: &[&dyn DynAccelerationStructure], + ) { + let command_buffers: Vec<&::CommandBuffer> = command_buffers + .iter() + .map(|command_buffer| command_buffer.expect_downcast_ref()) + .collect(); + let dependencies: Vec<&::AccelerationStructure> = dependencies + .iter() + .map(|dependency| dependency.expect_downcast_ref()) + .collect(); + unsafe { C::set_acceleration_structure_dependencies(&command_buffers, &dependencies) } + } +} + +impl<'a> PassTimestampWrites<'a, dyn DynQuerySet> { + pub fn expect_downcast(&self) -> PassTimestampWrites<'a, B> { + PassTimestampWrites { + query_set: self.query_set.expect_downcast_ref(), + beginning_of_pass_write_index: self.beginning_of_pass_write_index, + end_of_pass_write_index: self.end_of_pass_write_index, + } + } +} + +impl<'a> Attachment<'a, dyn DynTextureView> { + pub fn expect_downcast(&self) -> Attachment<'a, B> { + Attachment { + view: self.view.expect_downcast_ref(), + usage: self.usage, + } + } +} + +impl<'a> ColorAttachment<'a, dyn DynTextureView> { + pub fn expect_downcast(&self) -> ColorAttachment<'a, B> { + ColorAttachment { + target: self.target.expect_downcast(), + depth_slice: self.depth_slice, + resolve_target: self.resolve_target.as_ref().map(|rt| rt.expect_downcast()), + ops: self.ops, + clear_value: self.clear_value, + } + } +} + +impl<'a> DepthStencilAttachment<'a, dyn DynTextureView> { + pub fn expect_downcast(&self) -> DepthStencilAttachment<'a, B> { + DepthStencilAttachment { + target: self.target.expect_downcast(), + depth_ops: self.depth_ops, + stencil_ops: self.stencil_ops, + clear_value: self.clear_value, + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dynamic/device.rs b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/device.rs new file mode 100644 index 00000000..b5ff5904 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/device.rs @@ -0,0 +1,558 @@ +use alloc::{borrow::ToOwned as _, boxed::Box, vec::Vec}; + +use crate::{ + AccelerationStructureBuildSizes, AccelerationStructureDescriptor, Api, BindGroupDescriptor, + BindGroupLayoutDescriptor, BufferDescriptor, BufferMapping, CommandEncoderDescriptor, + ComputePipelineDescriptor, Device, DeviceError, FenceValue, + GetAccelerationStructureBuildSizesDescriptor, Label, MemoryRange, PipelineCacheDescriptor, + PipelineCacheError, PipelineError, PipelineLayoutDescriptor, RenderPipelineDescriptor, + SamplerDescriptor, ShaderError, ShaderInput, ShaderModuleDescriptor, TextureDescriptor, + TextureViewDescriptor, TlasInstance, +}; + +use super::{ + DynAccelerationStructure, DynBindGroup, DynBindGroupLayout, DynBuffer, DynCommandEncoder, + DynComputePipeline, DynFence, DynPipelineCache, DynPipelineLayout, DynQuerySet, DynQueue, + DynRenderPipeline, DynResource, DynResourceExt as _, DynSampler, DynShaderModule, DynTexture, + DynTextureView, +}; + +pub trait DynDevice: DynResource { + unsafe fn create_buffer( + &self, + desc: &BufferDescriptor, + ) -> Result, DeviceError>; + + unsafe fn destroy_buffer(&self, buffer: Box); + unsafe fn add_raw_buffer(&self, buffer: &dyn DynBuffer); + + unsafe fn map_buffer( + &self, + buffer: &dyn DynBuffer, + range: MemoryRange, + ) -> Result; + + unsafe fn unmap_buffer(&self, buffer: &dyn DynBuffer); + + unsafe fn flush_mapped_ranges(&self, buffer: &dyn DynBuffer, ranges: &[MemoryRange]); + unsafe fn invalidate_mapped_ranges(&self, buffer: &dyn DynBuffer, ranges: &[MemoryRange]); + + unsafe fn create_texture( + &self, + desc: &TextureDescriptor, + ) -> Result, DeviceError>; + unsafe fn destroy_texture(&self, texture: Box); + unsafe fn add_raw_texture(&self, texture: &dyn DynTexture); + + unsafe fn create_texture_view( + &self, + texture: &dyn DynTexture, + desc: &TextureViewDescriptor, + ) -> Result, DeviceError>; + unsafe fn destroy_texture_view(&self, view: Box); + unsafe fn create_sampler( + &self, + desc: &SamplerDescriptor, + ) -> Result, DeviceError>; + unsafe fn destroy_sampler(&self, sampler: Box); + + unsafe fn create_command_encoder( + &self, + desc: &CommandEncoderDescriptor, + ) -> Result, DeviceError>; + + unsafe fn create_bind_group_layout( + &self, + desc: &BindGroupLayoutDescriptor, + ) -> Result, DeviceError>; + unsafe fn destroy_bind_group_layout(&self, bg_layout: Box); + + unsafe fn create_pipeline_layout( + &self, + desc: &PipelineLayoutDescriptor, + ) -> Result, DeviceError>; + unsafe fn destroy_pipeline_layout(&self, pipeline_layout: Box); + + unsafe fn create_bind_group( + &self, + desc: &BindGroupDescriptor< + dyn DynBindGroupLayout, + dyn DynBuffer, + dyn DynSampler, + dyn DynTextureView, + dyn DynAccelerationStructure, + >, + ) -> Result, DeviceError>; + unsafe fn destroy_bind_group(&self, group: Box); + + unsafe fn create_shader_module( + &self, + desc: &ShaderModuleDescriptor, + shader: ShaderInput, + ) -> Result, ShaderError>; + unsafe fn destroy_shader_module(&self, module: Box); + + unsafe fn create_render_pipeline( + &self, + desc: &RenderPipelineDescriptor< + dyn DynPipelineLayout, + dyn DynShaderModule, + dyn DynPipelineCache, + >, + ) -> Result, PipelineError>; + unsafe fn destroy_render_pipeline(&self, pipeline: Box); + + unsafe fn create_compute_pipeline( + &self, + desc: &ComputePipelineDescriptor< + dyn DynPipelineLayout, + dyn DynShaderModule, + dyn DynPipelineCache, + >, + ) -> Result, PipelineError>; + unsafe fn destroy_compute_pipeline(&self, pipeline: Box); + + unsafe fn create_pipeline_cache( + &self, + desc: &PipelineCacheDescriptor<'_>, + ) -> Result, PipelineCacheError>; + fn pipeline_cache_validation_key(&self) -> Option<[u8; 16]> { + None + } + unsafe fn destroy_pipeline_cache(&self, cache: Box); + + unsafe fn create_query_set( + &self, + desc: &wgt::QuerySetDescriptor) -> Self { + Self { + adapter: Box::new(exposed_adapter.adapter), + info: exposed_adapter.info, + features: exposed_adapter.features, + capabilities: exposed_adapter.capabilities, + } + } +} + +pub trait DynInstance: DynResource { + unsafe fn create_surface( + &self, + display_handle: raw_window_handle::RawDisplayHandle, + window_handle: raw_window_handle::RawWindowHandle, + ) -> Result, InstanceError>; + + unsafe fn enumerate_adapters( + &self, + surface_hint: Option<&dyn DynSurface>, + ) -> Vec; +} + +impl DynInstance for I { + unsafe fn create_surface( + &self, + display_handle: raw_window_handle::RawDisplayHandle, + window_handle: raw_window_handle::RawWindowHandle, + ) -> Result, InstanceError> { + unsafe { I::create_surface(self, display_handle, window_handle) } + .map(|surface| -> Box { Box::new(surface) }) + } + + unsafe fn enumerate_adapters( + &self, + surface_hint: Option<&dyn DynSurface>, + ) -> Vec { + let surface_hint = surface_hint.map(|s| s.expect_downcast_ref()); + unsafe { I::enumerate_adapters(self, surface_hint) } + .into_iter() + .map(|exposed| DynExposedAdapter { + adapter: Box::new(exposed.adapter), + info: exposed.info, + features: exposed.features, + capabilities: exposed.capabilities, + }) + .collect() + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dynamic/mod.rs b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/mod.rs new file mode 100644 index 00000000..85d8ca00 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/mod.rs @@ -0,0 +1,221 @@ +mod adapter; +mod command; +mod device; +mod instance; +mod queue; +mod surface; + +pub use adapter::{DynAdapter, DynOpenDevice}; +pub use command::DynCommandEncoder; +pub use device::DynDevice; +pub use instance::{DynExposedAdapter, DynInstance}; +pub use queue::DynQueue; +pub use surface::{DynAcquiredSurfaceTexture, DynSurface}; + +use alloc::boxed::Box; +use core::{ + any::{Any, TypeId}, + fmt, +}; + +use wgt::WasmNotSendSync; + +use crate::{ + AccelerationStructureAABBs, AccelerationStructureEntries, AccelerationStructureInstances, + AccelerationStructureTriangleIndices, AccelerationStructureTriangleTransform, + AccelerationStructureTriangles, BufferBinding, ExternalTextureBinding, ProgrammableStage, + TextureBinding, +}; + +/// Base trait for all resources, allows downcasting via [`Any`]. +pub trait DynResource: Any + WasmNotSendSync + 'static { + fn as_any(&self) -> &dyn Any; + fn as_any_mut(&mut self) -> &mut dyn Any; +} + +/// Utility macro for implementing `DynResource` for a list of types. +macro_rules! impl_dyn_resource { + ($($type:ty),*) => { + $( + impl crate::DynResource for $type { + fn as_any(&self) -> &dyn ::core::any::Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn ::core::any::Any { + self + } + } + )* + }; +} +pub(crate) use impl_dyn_resource; + +/// Extension trait for `DynResource` used by implementations of various dynamic resource traits. +trait DynResourceExt { + /// # Panics + /// + /// - Panics if `self` is not downcastable to `T`. + fn expect_downcast_ref(&self) -> &T; + /// # Panics + /// + /// - Panics if `self` is not downcastable to `T`. + fn expect_downcast_mut(&mut self) -> &mut T; + + /// Unboxes a `Box` to a concrete type. + /// + /// # Safety + /// + /// - `self` must be the correct concrete type. + unsafe fn unbox(self: Box) -> T; +} + +impl DynResourceExt for R { + fn expect_downcast_ref<'a, T: DynResource>(&'a self) -> &'a T { + self.as_any() + .downcast_ref() + .expect("Resource doesn't have the expected backend type.") + } + + fn expect_downcast_mut<'a, T: DynResource>(&'a mut self) -> &'a mut T { + self.as_any_mut() + .downcast_mut() + .expect("Resource doesn't have the expected backend type.") + } + + unsafe fn unbox(self: Box) -> T { + debug_assert!( + ::type_id(self.as_ref()) == TypeId::of::(), + "Resource doesn't have the expected type, expected {:?}, got {:?}", + TypeId::of::(), + ::type_id(self.as_ref()) + ); + + let casted_ptr = Box::into_raw(self).cast::(); + // SAFETY: This is adheres to the safety contract of `Box::from_raw` because: + // + // - We are casting the value of a previously `Box`ed value, which guarantees: + // - `casted_ptr` is not null. + // - `casted_ptr` is valid for reads and writes, though by itself this does not mean + // valid reads and writes for `T` (read on for that). + // - We don't change the allocator. + // - The contract of `Box::from_raw` requires that an initialized and aligned `T` is stored + // within `casted_ptr`. + *unsafe { Box::from_raw(casted_ptr) } + } +} + +pub trait DynAccelerationStructure: DynResource + fmt::Debug {} +pub trait DynBindGroup: DynResource + fmt::Debug {} +pub trait DynBindGroupLayout: DynResource + fmt::Debug {} +pub trait DynBuffer: DynResource + fmt::Debug {} +pub trait DynCommandBuffer: DynResource + fmt::Debug {} +pub trait DynComputePipeline: DynResource + fmt::Debug {} +pub trait DynFence: DynResource + fmt::Debug {} +pub trait DynPipelineCache: DynResource + fmt::Debug {} +pub trait DynPipelineLayout: DynResource + fmt::Debug {} +pub trait DynQuerySet: DynResource + fmt::Debug {} +pub trait DynRenderPipeline: DynResource + fmt::Debug {} +pub trait DynSampler: DynResource + fmt::Debug {} +pub trait DynShaderModule: DynResource + fmt::Debug {} +pub trait DynSurfaceTexture: + DynResource + core::borrow::Borrow + fmt::Debug +{ +} +pub trait DynTexture: DynResource + fmt::Debug {} +pub trait DynTextureView: DynResource + fmt::Debug {} + +impl<'a> BufferBinding<'a, dyn DynBuffer> { + pub fn expect_downcast(self) -> BufferBinding<'a, B> { + BufferBinding { + buffer: self.buffer.expect_downcast_ref(), + offset: self.offset, + size: self.size, + } + } +} + +impl<'a> TextureBinding<'a, dyn DynTextureView> { + pub fn expect_downcast(self) -> TextureBinding<'a, T> { + TextureBinding { + view: self.view.expect_downcast_ref(), + usage: self.usage, + } + } +} + +impl<'a> ExternalTextureBinding<'a, dyn DynBuffer, dyn DynTextureView> { + pub fn expect_downcast( + self, + ) -> ExternalTextureBinding<'a, B, T> { + let planes = self.planes.map(|plane| plane.expect_downcast()); + let params = self.params.expect_downcast(); + ExternalTextureBinding { planes, params } + } +} + +impl<'a> ProgrammableStage<'a, dyn DynShaderModule> { + fn expect_downcast(self) -> ProgrammableStage<'a, T> { + ProgrammableStage { + module: self.module.expect_downcast_ref(), + entry_point: self.entry_point, + constants: self.constants, + zero_initialize_workgroup_memory: self.zero_initialize_workgroup_memory, + } + } +} + +impl<'a> AccelerationStructureEntries<'a, dyn DynBuffer> { + fn expect_downcast(&self) -> AccelerationStructureEntries<'a, B> { + match self { + AccelerationStructureEntries::Instances(instances) => { + AccelerationStructureEntries::Instances(AccelerationStructureInstances { + buffer: instances.buffer.map(|b| b.expect_downcast_ref()), + offset: instances.offset, + count: instances.count, + }) + } + AccelerationStructureEntries::Triangles(triangles) => { + AccelerationStructureEntries::Triangles( + triangles + .iter() + .map(|t| AccelerationStructureTriangles { + vertex_buffer: t.vertex_buffer.map(|b| b.expect_downcast_ref()), + vertex_format: t.vertex_format, + first_vertex: t.first_vertex, + vertex_count: t.vertex_count, + vertex_stride: t.vertex_stride, + indices: t.indices.as_ref().map(|i| { + AccelerationStructureTriangleIndices { + buffer: i.buffer.map(|b| b.expect_downcast_ref()), + format: i.format, + offset: i.offset, + count: i.count, + } + }), + transform: t.transform.as_ref().map(|t| { + AccelerationStructureTriangleTransform { + buffer: t.buffer.expect_downcast_ref(), + offset: t.offset, + } + }), + flags: t.flags, + }) + .collect(), + ) + } + AccelerationStructureEntries::AABBs(entries) => AccelerationStructureEntries::AABBs( + entries + .iter() + .map(|e| AccelerationStructureAABBs { + buffer: e.buffer.map(|b| b.expect_downcast_ref()), + offset: e.offset, + count: e.count, + stride: e.stride, + flags: e.flags, + }) + .collect(), + ), + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dynamic/queue.rs b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/queue.rs new file mode 100644 index 00000000..2af6d1f9 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/queue.rs @@ -0,0 +1,56 @@ +use alloc::{boxed::Box, vec::Vec}; + +use crate::{ + DeviceError, DynCommandBuffer, DynFence, DynResource, DynSurface, DynSurfaceTexture, + FenceValue, Queue, SurfaceError, +}; + +use super::DynResourceExt as _; + +pub trait DynQueue: DynResource { + unsafe fn submit( + &self, + command_buffers: &[&dyn DynCommandBuffer], + surface_textures: &[&dyn DynSurfaceTexture], + signal_fence: (&mut dyn DynFence, FenceValue), + ) -> Result<(), DeviceError>; + unsafe fn present( + &self, + surface: &dyn DynSurface, + texture: Box, + ) -> Result<(), SurfaceError>; + unsafe fn get_timestamp_period(&self) -> f32; +} + +impl DynQueue for Q { + unsafe fn submit( + &self, + command_buffers: &[&dyn DynCommandBuffer], + surface_textures: &[&dyn DynSurfaceTexture], + signal_fence: (&mut dyn DynFence, FenceValue), + ) -> Result<(), DeviceError> { + let command_buffers = command_buffers + .iter() + .map(|cb| (*cb).expect_downcast_ref()) + .collect::>(); + let surface_textures = surface_textures + .iter() + .map(|surface| (*surface).expect_downcast_ref()) + .collect::>(); + let signal_fence = (signal_fence.0.expect_downcast_mut(), signal_fence.1); + unsafe { Q::submit(self, &command_buffers, &surface_textures, signal_fence) } + } + + unsafe fn present( + &self, + surface: &dyn DynSurface, + texture: Box, + ) -> Result<(), SurfaceError> { + let surface = surface.expect_downcast_ref(); + unsafe { Q::present(self, surface, texture.unbox()) } + } + + unsafe fn get_timestamp_period(&self) -> f32 { + unsafe { Q::get_timestamp_period(self) } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/dynamic/surface.rs b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/surface.rs new file mode 100644 index 00000000..17fa8ee8 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/dynamic/surface.rs @@ -0,0 +1,72 @@ +use alloc::boxed::Box; +use core::time::Duration; + +use crate::{ + DynDevice, DynFence, DynResource, DynSurfaceTexture, Surface, SurfaceConfiguration, + SurfaceError, +}; + +use super::DynResourceExt as _; + +#[derive(Debug)] +pub struct DynAcquiredSurfaceTexture { + pub texture: Box, + /// The presentation configuration no longer matches + /// the surface properties exactly, but can still be used to present + /// to the surface successfully. + pub suboptimal: bool, +} + +pub trait DynSurface: DynResource { + unsafe fn configure( + &self, + device: &dyn DynDevice, + config: &SurfaceConfiguration, + ) -> Result<(), SurfaceError>; + + unsafe fn unconfigure(&self, device: &dyn DynDevice); + + unsafe fn acquire_texture( + &self, + timeout: Option, + fence: &dyn DynFence, + ) -> Result; + + unsafe fn discard_texture(&self, texture: Box); +} + +impl DynSurface for S { + unsafe fn configure( + &self, + device: &dyn DynDevice, + config: &SurfaceConfiguration, + ) -> Result<(), SurfaceError> { + let device = device.expect_downcast_ref(); + unsafe { S::configure(self, device, config) } + } + + unsafe fn unconfigure(&self, device: &dyn DynDevice) { + let device = device.expect_downcast_ref(); + unsafe { S::unconfigure(self, device) } + } + + unsafe fn acquire_texture( + &self, + timeout: Option, + fence: &dyn DynFence, + ) -> Result { + let fence = fence.expect_downcast_ref(); + unsafe { S::acquire_texture(self, timeout, fence) }.map(|ast| { + let texture = Box::new(ast.texture); + let suboptimal = ast.suboptimal; + DynAcquiredSurfaceTexture { + texture, + suboptimal, + } + }) + } + + unsafe fn discard_texture(&self, texture: Box) { + unsafe { S::discard_texture(self, texture.unbox()) } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/adapter.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/adapter.rs new file mode 100644 index 00000000..c682145f --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/adapter.rs @@ -0,0 +1,1366 @@ +use alloc::{borrow::ToOwned as _, format, string::String, sync::Arc, vec, vec::Vec}; +use core::sync::atomic::AtomicU8; + +use glow::HasContext; +use parking_lot::Mutex; +use wgt::AstcChannel; + +use crate::auxil::db; +use crate::gles::ShaderClearProgram; + +// https://webgl2fundamentals.org/webgl/lessons/webgl-data-textures.html + +const GL_UNMASKED_VENDOR_WEBGL: u32 = 0x9245; +const GL_UNMASKED_RENDERER_WEBGL: u32 = 0x9246; + +impl super::Adapter { + /// Note that this function is intentionally lenient in regards to parsing, + /// and will try to recover at least the first two version numbers without + /// resulting in an `Err`. + /// # Notes + /// `WebGL 2` version returned as `OpenGL ES 3.0` + fn parse_version(mut src: &str) -> Result<(u8, u8), crate::InstanceError> { + let webgl_sig = "WebGL "; + // According to the WebGL specification + // VERSION WebGL1.0 + // SHADING_LANGUAGE_VERSION WebGLGLSLES1.0 + let is_webgl = src.starts_with(webgl_sig); + if is_webgl { + let pos = src.rfind(webgl_sig).unwrap_or(0); + src = &src[pos + webgl_sig.len()..]; + } else { + let es_sig = " ES "; + match src.rfind(es_sig) { + Some(pos) => { + src = &src[pos + es_sig.len()..]; + } + None => { + return Err(crate::InstanceError::new(format!( + "OpenGL version {src:?} does not contain 'ES'" + ))); + } + } + }; + + let glsl_es_sig = "GLSL ES "; + let is_glsl = match src.find(glsl_es_sig) { + Some(pos) => { + src = &src[pos + glsl_es_sig.len()..]; + true + } + None => false, + }; + + Self::parse_full_version(src).map(|(major, minor)| { + ( + // Return WebGL 2.0 version as OpenGL ES 3.0 + if is_webgl && !is_glsl { + major + 1 + } else { + major + }, + minor, + ) + }) + } + + /// According to the OpenGL specification, the version information is + /// expected to follow the following syntax: + /// + /// ~~~bnf + /// ::= + /// ::= + /// ::= + /// ::= + /// ::= "." ["." ] + /// ::= [" " ] + /// ~~~ + /// + /// Note that this function is intentionally lenient in regards to parsing, + /// and will try to recover at least the first two version numbers without + /// resulting in an `Err`. + pub(super) fn parse_full_version(src: &str) -> Result<(u8, u8), crate::InstanceError> { + let (version, _vendor_info) = match src.find(' ') { + Some(i) => (&src[..i], src[i + 1..].to_owned()), + None => (src, String::new()), + }; + + // TODO: make this even more lenient so that we can also accept + // ` "." []` + let mut it = version.split('.'); + let major = it.next().and_then(|s| s.parse().ok()); + let minor = it.next().and_then(|s| { + let trimmed = if s.starts_with('0') { + "0" + } else { + s.trim_end_matches('0') + }; + trimmed.parse().ok() + }); + + match (major, minor) { + (Some(major), Some(minor)) => Ok((major, minor)), + _ => Err(crate::InstanceError::new(format!( + "unable to extract OpenGL version from {version:?}" + ))), + } + } + + fn make_info(vendor_orig: String, renderer_orig: String, version: String) -> wgt::AdapterInfo { + let vendor = vendor_orig.to_lowercase(); + let renderer = renderer_orig.to_lowercase(); + + // opengl has no way to discern device_type, so we can try to infer it from the renderer string + let strings_that_imply_integrated = [ + " xpress", // space here is on purpose so we don't match express + "amd renoir", + "radeon hd 4200", + "radeon hd 4250", + "radeon hd 4290", + "radeon hd 4270", + "radeon hd 4225", + "radeon hd 3100", + "radeon hd 3200", + "radeon hd 3000", + "radeon hd 3300", + "radeon(tm) r4 graphics", + "radeon(tm) r5 graphics", + "radeon(tm) r6 graphics", + "radeon(tm) r7 graphics", + "radeon r7 graphics", + "nforce", // all nvidia nforce are integrated + "tegra", // all nvidia tegra are integrated + "shield", // all nvidia shield are integrated + "igp", + "mali", + "intel", + "v3d", + "apple m", // all apple m are integrated + ]; + let strings_that_imply_cpu = ["mesa offscreen", "swiftshader", "llvmpipe"]; + + //TODO: handle Intel Iris XE as discreet + let inferred_device_type = if vendor.contains("qualcomm") + || vendor.contains("intel") + || strings_that_imply_integrated + .iter() + .any(|&s| renderer.contains(s)) + { + wgt::DeviceType::IntegratedGpu + } else if strings_that_imply_cpu.iter().any(|&s| renderer.contains(s)) { + wgt::DeviceType::Cpu + } else { + // At this point the Device type is Unknown. + // It's most likely DiscreteGpu, but we do not know for sure. + // Use "Other" to avoid possibly making incorrect assumptions. + // Note that if this same device is available under some other API (ex: Vulkan), + // It will mostly likely get a different device type (probably DiscreteGpu). + wgt::DeviceType::Other + }; + + // source: Sascha Willems at Vulkan + let vendor_id = if vendor.contains("amd") { + db::amd::VENDOR + } else if vendor.contains("imgtec") { + db::imgtec::VENDOR + } else if vendor.contains("nvidia") { + db::nvidia::VENDOR + } else if vendor.contains("arm") { + db::arm::VENDOR + } else if vendor.contains("qualcomm") { + db::qualcomm::VENDOR + } else if vendor.contains("intel") { + db::intel::VENDOR + } else if vendor.contains("broadcom") { + db::broadcom::VENDOR + } else if vendor.contains("mesa") { + db::mesa::VENDOR + } else if vendor.contains("apple") { + db::apple::VENDOR + } else { + 0 + }; + + wgt::AdapterInfo { + name: renderer_orig, + vendor: vendor_id, + device: 0, + device_type: inferred_device_type, + driver: "".to_owned(), + device_pci_bus_id: String::new(), + driver_info: version, + backend: wgt::Backend::Gl, + subgroup_min_size: wgt::MINIMUM_SUBGROUP_MIN_SIZE, + subgroup_max_size: wgt::MAXIMUM_SUBGROUP_MAX_SIZE, + transient_saves_memory: false, + } + } + + pub(super) unsafe fn expose( + context: super::AdapterContext, + backend_options: wgt::GlBackendOptions, + ) -> Option> { + let gl = context.lock(); + let extensions = gl.supported_extensions(); + + let (vendor_const, renderer_const) = if extensions.contains("WEBGL_debug_renderer_info") { + // emscripten doesn't enable "WEBGL_debug_renderer_info" extension by default. so, we do it manually. + // See https://github.com/gfx-rs/wgpu/issues/3245 for context + #[cfg(Emscripten)] + if unsafe { + super::emscripten::enable_extension(c"WEBGL_debug_renderer_info".to_str().unwrap()) + } { + (GL_UNMASKED_VENDOR_WEBGL, GL_UNMASKED_RENDERER_WEBGL) + } else { + (glow::VENDOR, glow::RENDERER) + } + // glow already enables WEBGL_debug_renderer_info on wasm32-unknown-unknown target by default. + #[cfg(not(Emscripten))] + (GL_UNMASKED_VENDOR_WEBGL, GL_UNMASKED_RENDERER_WEBGL) + } else { + (glow::VENDOR, glow::RENDERER) + }; + + let vendor = unsafe { gl.get_parameter_string(vendor_const) }; + let renderer = unsafe { gl.get_parameter_string(renderer_const) }; + let version = unsafe { gl.get_parameter_string(glow::VERSION) }; + log::debug!("Vendor: {vendor}"); + log::debug!("Renderer: {renderer}"); + log::debug!("Version: {version}"); + + let full_ver = Self::parse_full_version(&version).ok(); + let es_ver = full_ver.map_or_else(|| Self::parse_version(&version).ok(), |_| None); + + if let Some(full_ver) = full_ver { + let core_profile = (full_ver >= (3, 2)).then(|| unsafe { + gl.get_parameter_i32(glow::CONTEXT_PROFILE_MASK) + & glow::CONTEXT_CORE_PROFILE_BIT as i32 + != 0 + }); + log::trace!( + "Profile: {}", + core_profile + .map(|core_profile| if core_profile { + "Core" + } else { + "Compatibility" + }) + .unwrap_or("Legacy") + ); + } + + if es_ver.is_none() && full_ver.is_none() { + log::warn!("Unable to parse OpenGL version"); + return None; + } + + if let Some(es_ver) = es_ver { + if es_ver < (3, 0) { + log::warn!( + "Returned GLES context is {}.{}, when 3.0+ was requested", + es_ver.0, + es_ver.1 + ); + return None; + } + } + + if let Some(full_ver) = full_ver { + if full_ver < (3, 3) { + log::warn!( + "Returned GL context is {}.{}, when 3.3+ is needed", + full_ver.0, + full_ver.1 + ); + return None; + } + } + + let shading_language_version = { + let sl_version = unsafe { gl.get_parameter_string(glow::SHADING_LANGUAGE_VERSION) }; + log::debug!("SL version: {}", &sl_version); + if full_ver.is_some() { + let (sl_major, sl_minor) = Self::parse_full_version(&sl_version).ok()?; + let mut value = sl_major as u16 * 100 + sl_minor as u16 * 10; + // Naga doesn't think it supports GL 460+, so we cap it at 450 + if value > 450 { + value = 450; + } + naga::back::glsl::Version::Desktop(value) + } else { + let (sl_major, sl_minor) = Self::parse_version(&sl_version).ok()?; + let value = sl_major as u16 * 100 + sl_minor as u16 * 10; + naga::back::glsl::Version::Embedded { + version: value, + is_webgl: cfg!(any(webgl, Emscripten)), + } + } + }; + + log::debug!("Supported GL Extensions: {extensions:#?}"); + + let supported = |(req_es_major, req_es_minor), (req_full_major, req_full_minor)| { + let es_supported = es_ver + .map(|es_ver| es_ver >= (req_es_major, req_es_minor)) + .unwrap_or_default(); + + let full_supported = full_ver + .map(|full_ver| full_ver >= (req_full_major, req_full_minor)) + .unwrap_or_default(); + + es_supported || full_supported + }; + + let supports_storage = + supported((3, 1), (4, 3)) || extensions.contains("GL_ARB_shader_storage_buffer_object"); + let supports_compute = + supported((3, 1), (4, 3)) || extensions.contains("GL_ARB_compute_shader"); + let supports_work_group_params = supports_compute; + + // ANGLE provides renderer strings like: "ANGLE (Apple, Apple M1 Pro, OpenGL 4.1)" + let is_angle = renderer.contains("ANGLE"); + + let vertex_shader_storage_blocks = if supports_storage { + let value = + (unsafe { gl.get_parameter_i32(glow::MAX_VERTEX_SHADER_STORAGE_BLOCKS) } as u32); + + if value == 0 && extensions.contains("GL_ARB_shader_storage_buffer_object") { + // The driver for AMD Radeon HD 5870 returns zero here, so assume the value matches the compute shader storage block count. + // Windows doesn't recognize `GL_MAX_VERTEX_ATTRIB_STRIDE`. + let new = (unsafe { gl.get_parameter_i32(glow::MAX_COMPUTE_SHADER_STORAGE_BLOCKS) } + as u32); + log::debug!("Max vertex shader storage blocks is zero, but GL_ARB_shader_storage_buffer_object is specified. Assuming the compute value {new}"); + new + } else { + value + } + } else { + 0 + }; + let fragment_shader_storage_blocks = if supports_storage { + (unsafe { gl.get_parameter_i32(glow::MAX_FRAGMENT_SHADER_STORAGE_BLOCKS) } as u32) + } else { + 0 + }; + let vertex_shader_storage_textures = if supports_storage { + (unsafe { gl.get_parameter_i32(glow::MAX_VERTEX_IMAGE_UNIFORMS) } as u32) + } else { + 0 + }; + let fragment_shader_storage_textures = if supports_storage { + (unsafe { gl.get_parameter_i32(glow::MAX_FRAGMENT_IMAGE_UNIFORMS) } as u32) + } else { + 0 + }; + let max_storage_block_size = if supports_storage { + (unsafe { gl.get_parameter_i32(glow::MAX_SHADER_STORAGE_BLOCK_SIZE) } as u32) + } else { + 0 + }; + let max_element_index = unsafe { gl.get_parameter_i32(glow::MAX_ELEMENT_INDEX) } as u32; + + // WORKAROUND: In order to work around an issue with GL on RPI4 and similar, we ignore a + // zero vertex ssbo count if there are vertex sstos. (more info: + // https://github.com/gfx-rs/wgpu/pull/1607#issuecomment-874938961) The hardware does not + // want us to write to these SSBOs, but GLES cannot express that. We detect this case and + // disable writing to SSBOs. + let vertex_ssbo_false_zero = + vertex_shader_storage_blocks == 0 && vertex_shader_storage_textures != 0; + if vertex_ssbo_false_zero { + // We only care about fragment here as the 0 is a lie. + log::debug!("Max vertex shader SSBO == 0 and SSTO != 0. Interpreting as false zero."); + } + + let max_storage_buffers_per_shader_stage = if vertex_shader_storage_blocks == 0 { + fragment_shader_storage_blocks + } else { + vertex_shader_storage_blocks.min(fragment_shader_storage_blocks) + }; + let max_storage_textures_per_shader_stage = if vertex_shader_storage_textures == 0 { + fragment_shader_storage_textures + } else { + vertex_shader_storage_textures.min(fragment_shader_storage_textures) + }; + // NOTE: GL_ARB_compute_shader adds support for indirect dispatch + let indirect_execution = supported((3, 1), (4, 3)) + || (extensions.contains("GL_ARB_draw_indirect") && supports_compute); + let supports_cube_array = supported((3, 2), (4, 0)) + || (supported((3, 1), (4, 0)) && extensions.contains("GL_EXT_texture_cube_map_array")); + + let mut downlevel_flags = wgt::DownlevelFlags::empty() + | wgt::DownlevelFlags::NON_POWER_OF_TWO_MIPMAPPED_TEXTURES + | wgt::DownlevelFlags::COMPARISON_SAMPLERS + | wgt::DownlevelFlags::SHADER_F16_IN_F32; + downlevel_flags.set( + wgt::DownlevelFlags::CUBE_ARRAY_TEXTURES, + supports_cube_array, + ); + downlevel_flags.set(wgt::DownlevelFlags::COMPUTE_SHADERS, supports_compute); + downlevel_flags.set( + wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE, + max_storage_block_size != 0, + ); + downlevel_flags.set(wgt::DownlevelFlags::INDIRECT_EXECUTION, indirect_execution); + downlevel_flags.set(wgt::DownlevelFlags::BASE_VERTEX, supported((3, 2), (3, 2))); + downlevel_flags.set( + wgt::DownlevelFlags::INDEPENDENT_BLEND, + supported((3, 2), (4, 0)) || extensions.contains("GL_EXT_draw_buffers_indexed"), + ); + downlevel_flags.set( + wgt::DownlevelFlags::VERTEX_STORAGE, + max_storage_block_size != 0 + && max_storage_buffers_per_shader_stage != 0 + && (vertex_shader_storage_blocks != 0 || vertex_ssbo_false_zero), + ); + downlevel_flags.set(wgt::DownlevelFlags::FRAGMENT_STORAGE, supports_storage); + if extensions.contains("EXT_texture_filter_anisotropic") + || extensions.contains("GL_EXT_texture_filter_anisotropic") + { + let max_aniso = + unsafe { gl.get_parameter_i32(glow::MAX_TEXTURE_MAX_ANISOTROPY_EXT) } as u32; + downlevel_flags.set(wgt::DownlevelFlags::ANISOTROPIC_FILTERING, max_aniso >= 16); + } + downlevel_flags.set( + wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED, + !(cfg!(any(webgl, Emscripten)) || is_angle), + ); + // see https://registry.khronos.org/webgl/specs/latest/2.0/#BUFFER_OBJECT_BINDING + downlevel_flags.set( + wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER, + !cfg!(any(webgl, Emscripten)), + ); + downlevel_flags.set( + wgt::DownlevelFlags::UNRESTRICTED_EXTERNAL_TEXTURE_COPIES, + !cfg!(any(webgl, Emscripten)), + ); + downlevel_flags.set( + wgt::DownlevelFlags::FULL_DRAW_INDEX_UINT32, + max_element_index == u32::MAX, + ); + downlevel_flags.set( + wgt::DownlevelFlags::MULTISAMPLED_SHADING, + supported((3, 2), (4, 0)) || extensions.contains("OES_sample_variables"), + ); + let query_buffers = extensions.contains("GL_ARB_query_buffer_object") + || extensions.contains("GL_AMD_query_buffer_object"); + if query_buffers { + downlevel_flags.set(wgt::DownlevelFlags::NONBLOCKING_QUERY_RESOLVE, true); + } + + let mut features = wgt::Features::empty() + | wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES + | wgt::Features::CLEAR_TEXTURE + | wgt::Features::IMMEDIATES + | wgt::Features::DEPTH32FLOAT_STENCIL8; + features.set( + wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER | wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO, + extensions.contains("GL_EXT_texture_border_clamp") + || extensions.contains("GL_ARB_texture_border_clamp"), + ); + features.set( + wgt::Features::DEPTH_CLIP_CONTROL, + extensions.contains("GL_EXT_depth_clamp") || extensions.contains("GL_ARB_depth_clamp"), + ); + features.set( + wgt::Features::VERTEX_WRITABLE_STORAGE, + downlevel_flags.contains(wgt::DownlevelFlags::VERTEX_STORAGE) + && vertex_shader_storage_textures != 0, + ); + features.set( + wgt::Features::MULTIVIEW, + extensions.contains("OVR_multiview2") || extensions.contains("GL_OVR_multiview2"), + ); + features.set( + wgt::Features::DUAL_SOURCE_BLENDING, + extensions.contains("GL_EXT_blend_func_extended") + || extensions.contains("GL_ARB_blend_func_extended"), + ); + features.set( + wgt::Features::CLIP_DISTANCES, + full_ver.is_some() || extensions.contains("GL_EXT_clip_cull_distance"), + ); + features.set( + wgt::Features::PRIMITIVE_INDEX, + supported((3, 2), (3, 2)) + || extensions.contains("OES_geometry_shader") + || extensions.contains("GL_ARB_geometry_shader4"), + ); + features.set( + wgt::Features::SHADER_EARLY_DEPTH_TEST, + supported((3, 1), (4, 2)) || extensions.contains("GL_ARB_shader_image_load_store"), + ); + if extensions.contains("GL_ARB_timer_query") { + features.set(wgt::Features::TIMESTAMP_QUERY, true); + features.set(wgt::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS, true); + features.set(wgt::Features::TIMESTAMP_QUERY_INSIDE_PASSES, true); + } + let gl_bcn_exts = [ + "GL_EXT_texture_compression_s3tc", + "GL_EXT_texture_compression_rgtc", + "GL_ARB_texture_compression_bptc", + ]; + let gles_bcn_exts = [ + "GL_EXT_texture_compression_s3tc_srgb", + "GL_EXT_texture_compression_rgtc", + "GL_EXT_texture_compression_bptc", + ]; + let webgl_bcn_exts = [ + "WEBGL_compressed_texture_s3tc", + "WEBGL_compressed_texture_s3tc_srgb", + "EXT_texture_compression_rgtc", + "EXT_texture_compression_bptc", + ]; + let bcn_exts = if cfg!(any(webgl, Emscripten)) { + &webgl_bcn_exts[..] + } else if es_ver.is_some() { + &gles_bcn_exts[..] + } else { + &gl_bcn_exts[..] + }; + features.set( + wgt::Features::TEXTURE_COMPRESSION_BC, + bcn_exts.iter().all(|&ext| extensions.contains(ext)), + ); + features.set( + wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D, + bcn_exts.iter().all(|&ext| extensions.contains(ext)), // BC guaranteed Sliced 3D + ); + let has_etc = if cfg!(any(webgl, Emscripten)) { + extensions.contains("WEBGL_compressed_texture_etc") + } else { + es_ver.is_some() || extensions.contains("GL_ARB_ES3_compatibility") + }; + features.set(wgt::Features::TEXTURE_COMPRESSION_ETC2, has_etc); + + // `OES_texture_compression_astc` provides 2D + 3D, LDR + HDR support + if extensions.contains("WEBGL_compressed_texture_astc") + || extensions.contains("GL_OES_texture_compression_astc") + { + #[cfg(webgl)] + { + if context + .glow_context + .compressed_texture_astc_supports_ldr_profile() + { + features.insert(wgt::Features::TEXTURE_COMPRESSION_ASTC); + features.insert(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D); + } + if context + .glow_context + .compressed_texture_astc_supports_hdr_profile() + { + features.insert(wgt::Features::TEXTURE_COMPRESSION_ASTC_HDR); + } + } + + #[cfg(any(native, Emscripten))] + { + features.insert(wgt::Features::TEXTURE_COMPRESSION_ASTC); + features.insert(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D); + features.insert(wgt::Features::TEXTURE_COMPRESSION_ASTC_HDR); + } + } else { + features.set( + wgt::Features::TEXTURE_COMPRESSION_ASTC, + extensions.contains("GL_KHR_texture_compression_astc_ldr"), + ); + features.set( + wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D, + extensions.contains("GL_KHR_texture_compression_astc_ldr") + && extensions.contains("GL_KHR_texture_compression_astc_sliced_3d"), + ); + features.set( + wgt::Features::TEXTURE_COMPRESSION_ASTC_HDR, + extensions.contains("GL_KHR_texture_compression_astc_hdr"), + ); + } + + features.set( + wgt::Features::FLOAT32_FILTERABLE, + extensions.contains("GL_ARB_color_buffer_float") + || extensions.contains("GL_EXT_color_buffer_float") + || extensions.contains("OES_texture_float_linear"), + ); + + if es_ver.is_none() { + features |= wgt::Features::POLYGON_MODE_LINE | wgt::Features::POLYGON_MODE_POINT; + } + + // We *might* be able to emulate bgra8unorm-storage but currently don't attempt to. + + let mut private_caps = super::PrivateCapabilities::empty(); + private_caps.set( + super::PrivateCapabilities::BUFFER_ALLOCATION, + extensions.contains("GL_EXT_buffer_storage") + || extensions.contains("GL_ARB_buffer_storage"), + ); + private_caps.set( + super::PrivateCapabilities::SHADER_BINDING_LAYOUT, + supports_compute, + ); + private_caps.set( + super::PrivateCapabilities::SHADER_TEXTURE_SHADOW_LOD, + extensions.contains("GL_EXT_texture_shadow_lod"), + ); + private_caps.set( + super::PrivateCapabilities::MEMORY_BARRIERS, + supported((3, 1), (4, 2)), + ); + private_caps.set( + super::PrivateCapabilities::VERTEX_BUFFER_LAYOUT, + supported((3, 1), (4, 3)) || extensions.contains("GL_ARB_vertex_attrib_binding"), + ); + private_caps.set( + super::PrivateCapabilities::INDEX_BUFFER_ROLE_CHANGE, + !cfg!(any(webgl, Emscripten)), + ); + private_caps.set( + super::PrivateCapabilities::GET_BUFFER_SUB_DATA, + cfg!(any(webgl, Emscripten)) || full_ver.is_some(), + ); + let color_buffer_float = extensions.contains("GL_EXT_color_buffer_float") + || extensions.contains("GL_ARB_color_buffer_float") + || extensions.contains("EXT_color_buffer_float"); + let color_buffer_half_float = extensions.contains("GL_EXT_color_buffer_half_float") + || extensions.contains("GL_ARB_half_float_pixel"); + private_caps.set( + super::PrivateCapabilities::COLOR_BUFFER_HALF_FLOAT, + color_buffer_half_float || color_buffer_float, + ); + private_caps.set( + super::PrivateCapabilities::COLOR_BUFFER_FLOAT, + color_buffer_float, + ); + private_caps.set(super::PrivateCapabilities::QUERY_BUFFERS, query_buffers); + private_caps.set(super::PrivateCapabilities::QUERY_64BIT, full_ver.is_some()); + private_caps.set( + super::PrivateCapabilities::TEXTURE_STORAGE, + supported((3, 0), (4, 2)), + ); + let is_mali = renderer.to_lowercase().contains("mali"); + let debug_fns_enabled = match backend_options.debug_fns { + wgt::GlDebugFns::Auto => gl.supports_debug() && !is_mali, + wgt::GlDebugFns::ForceEnabled => gl.supports_debug(), + wgt::GlDebugFns::Disabled => false, + }; + private_caps.set(super::PrivateCapabilities::DEBUG_FNS, debug_fns_enabled); + private_caps.set( + super::PrivateCapabilities::INVALIDATE_FRAMEBUFFER, + supported((3, 0), (4, 3)), + ); + if let Some(full_ver) = full_ver { + let supported = + full_ver >= (4, 2) && extensions.contains("GL_ARB_shader_draw_parameters"); + private_caps.set( + super::PrivateCapabilities::FULLY_FEATURED_INSTANCING, + supported, + ); + // Desktop 4.2 and greater specify the first instance parameter. + // + // For all other versions, the behavior is undefined. + // + // We only support indirect first instance when we also have ARB_shader_draw_parameters as + // that's the only way to get gl_InstanceID to work correctly. + features.set(wgt::Features::INDIRECT_FIRST_INSTANCE, supported); + } + private_caps.set( + super::PrivateCapabilities::MULTISAMPLED_RENDER_TO_TEXTURE, + extensions.contains("GL_EXT_multisampled_render_to_texture"), + ); + + // GLSL ES 3.10+ / GLSL 4.30+ natively support coherent/volatile qualifiers + // on storage buffers. These were introduced alongside storage buffer support. + if supports_storage { + features |= wgt::Features::MEMORY_DECORATION_COHERENT + | wgt::Features::MEMORY_DECORATION_VOLATILE; + } + + let max_texture_size = unsafe { gl.get_parameter_i32(glow::MAX_TEXTURE_SIZE) } as u32; + let max_texture_3d_size = unsafe { gl.get_parameter_i32(glow::MAX_3D_TEXTURE_SIZE) } as u32; + + let min_uniform_buffer_offset_alignment = + (unsafe { gl.get_parameter_i32(glow::UNIFORM_BUFFER_OFFSET_ALIGNMENT) } as u32); + let min_storage_buffer_offset_alignment = if supports_storage { + (unsafe { gl.get_parameter_i32(glow::SHADER_STORAGE_BUFFER_OFFSET_ALIGNMENT) } as u32) + } else { + 256 + }; + let max_uniform_buffers_per_shader_stage = + unsafe { gl.get_parameter_i32(glow::MAX_VERTEX_UNIFORM_BLOCKS) } + .min(unsafe { gl.get_parameter_i32(glow::MAX_FRAGMENT_UNIFORM_BLOCKS) }) + as u32; + + let max_compute_workgroups_per_dimension = if supports_work_group_params { + unsafe { gl.get_parameter_indexed_i32(glow::MAX_COMPUTE_WORK_GROUP_COUNT, 0) } + .min(unsafe { gl.get_parameter_indexed_i32(glow::MAX_COMPUTE_WORK_GROUP_COUNT, 1) }) + .min(unsafe { gl.get_parameter_indexed_i32(glow::MAX_COMPUTE_WORK_GROUP_COUNT, 2) }) + as u32 + } else { + 0 + }; + + let max_color_attachments = unsafe { + gl.get_parameter_i32(glow::MAX_COLOR_ATTACHMENTS) + .min(gl.get_parameter_i32(glow::MAX_DRAW_BUFFERS)) as u32 + }; + + // 16 bytes per sample is the maximum size of a color attachment. + let max_color_attachment_bytes_per_sample = + max_color_attachments * wgt::TextureFormat::MAX_TARGET_PIXEL_BYTE_COST; + + let limits = crate::auxil::adjust_raw_limits(wgt::Limits { + max_texture_dimension_1d: max_texture_size, + max_texture_dimension_2d: max_texture_size, + max_texture_dimension_3d: max_texture_3d_size, + max_texture_array_layers: unsafe { + gl.get_parameter_i32(glow::MAX_ARRAY_TEXTURE_LAYERS) + } as u32, + max_bind_groups: crate::MAX_BIND_GROUPS as u32, + // No real limit. + max_bindings_per_bind_group: u32::MAX, + max_dynamic_uniform_buffers_per_pipeline_layout: max_uniform_buffers_per_shader_stage, + max_dynamic_storage_buffers_per_pipeline_layout: max_storage_buffers_per_shader_stage, + max_sampled_textures_per_shader_stage: super::MAX_TEXTURE_SLOTS as u32, + max_samplers_per_shader_stage: super::MAX_SAMPLERS as u32, + max_storage_buffers_per_shader_stage, + max_storage_textures_per_shader_stage, + max_uniform_buffers_per_shader_stage, + max_binding_array_elements_per_shader_stage: 0, + max_binding_array_sampler_elements_per_shader_stage: 0, + max_binding_array_acceleration_structure_elements_per_shader_stage: 0, + max_uniform_buffer_binding_size: unsafe { + gl.get_parameter_i32(glow::MAX_UNIFORM_BLOCK_SIZE) + } as u64, + max_storage_buffer_binding_size: if supports_storage { + unsafe { gl.get_parameter_i32(glow::MAX_SHADER_STORAGE_BLOCK_SIZE) } + } else { + 0 + } as u64, + max_vertex_buffers: if private_caps + .contains(super::PrivateCapabilities::VERTEX_BUFFER_LAYOUT) + { + (unsafe { gl.get_parameter_i32(glow::MAX_VERTEX_ATTRIB_BINDINGS) } as u32) + } else { + 16 // should this be different? + }, + max_vertex_attributes: (unsafe { gl.get_parameter_i32(glow::MAX_VERTEX_ATTRIBS) } + as u32) + .min(super::MAX_VERTEX_ATTRIBUTES as u32), + max_vertex_buffer_array_stride: if private_caps + .contains(super::PrivateCapabilities::VERTEX_BUFFER_LAYOUT) + { + if let Some(full_ver) = full_ver { + if full_ver >= (4, 4) { + // We can query `GL_MAX_VERTEX_ATTRIB_STRIDE` in OpenGL 4.4+ + let value = + (unsafe { gl.get_parameter_i32(glow::MAX_VERTEX_ATTRIB_STRIDE) }) + as u32; + + if value == 0 { + // This should be at least 2048, but the driver for AMD Radeon HD 5870 on + // Windows doesn't recognize `GL_MAX_VERTEX_ATTRIB_STRIDE`. + + log::debug!("Max vertex attribute stride is 0. Assuming it is the OpenGL minimum spec 2048"); + 2048 + } else { + value + } + } else { + log::debug!("Max vertex attribute stride unknown. Assuming it is the OpenGL minimum spec 2048"); + 2048 + } + } else { + (unsafe { gl.get_parameter_i32(glow::MAX_VERTEX_ATTRIB_STRIDE) }) as u32 + } + } else { + !0 + }, + max_immediate_size: super::MAX_IMMEDIATES as u32 * 4, + min_uniform_buffer_offset_alignment, + min_storage_buffer_offset_alignment, + max_inter_stage_shader_variables: { + // MAX_VARYING_COMPONENTS may return 0, because it is deprecated since OpenGL 3.2 core, + // and an OpenGL Context with the core profile and with forward-compatibility=true, + // will make deprecated constants unavailable. + let max_varying_components = + unsafe { gl.get_parameter_i32(glow::MAX_VARYING_COMPONENTS) } as u32; + if max_varying_components == 0 { + // default value for max_inter_stage_shader_variables + 15 + } else { + max_varying_components / 4 + } + }, + max_color_attachments, + max_color_attachment_bytes_per_sample, + max_compute_workgroup_storage_size: if supports_work_group_params { + (unsafe { gl.get_parameter_i32(glow::MAX_COMPUTE_SHARED_MEMORY_SIZE) } as u32) + } else { + 0 + }, + max_compute_invocations_per_workgroup: if supports_work_group_params { + (unsafe { gl.get_parameter_i32(glow::MAX_COMPUTE_WORK_GROUP_INVOCATIONS) } as u32) + } else { + 0 + }, + max_compute_workgroup_size_x: if supports_work_group_params { + (unsafe { gl.get_parameter_indexed_i32(glow::MAX_COMPUTE_WORK_GROUP_SIZE, 0) } + as u32) + } else { + 0 + }, + max_compute_workgroup_size_y: if supports_work_group_params { + (unsafe { gl.get_parameter_indexed_i32(glow::MAX_COMPUTE_WORK_GROUP_SIZE, 1) } + as u32) + } else { + 0 + }, + max_compute_workgroup_size_z: if supports_work_group_params { + (unsafe { gl.get_parameter_indexed_i32(glow::MAX_COMPUTE_WORK_GROUP_SIZE, 2) } + as u32) + } else { + 0 + }, + max_compute_workgroups_per_dimension, + max_buffer_size: i32::MAX as u64, + max_non_sampler_bindings: u32::MAX, + + max_task_mesh_workgroup_total_count: 0, + max_task_mesh_workgroups_per_dimension: 0, + max_task_invocations_per_workgroup: 0, + max_task_invocations_per_dimension: 0, + max_mesh_invocations_per_workgroup: 0, + max_mesh_invocations_per_dimension: 0, + max_task_payload_size: 0, + max_mesh_output_vertices: 0, + max_mesh_output_primitives: 0, + max_mesh_output_layers: 0, + max_mesh_multiview_view_count: 0, + + max_blas_primitive_count: 0, + max_blas_geometry_count: 0, + max_tlas_instance_count: 0, + max_acceleration_structures_per_shader_stage: 0, + + max_multiview_view_count: 0, + }); + + let mut workarounds = super::Workarounds::empty(); + + workarounds.set( + super::Workarounds::EMULATE_BUFFER_MAP, + cfg!(any(webgl, Emscripten)), + ); + + let r = renderer.to_lowercase(); + // Check for Mesa sRGB clear bug. See + // [`super::PrivateCapabilities::MESA_I915_SRGB_SHADER_CLEAR`]. + if context.is_owned() + && r.contains("mesa") + && r.contains("intel") + && r.split(&[' ', '(', ')'][..]) + .any(|substr| substr.len() == 3 && substr.chars().nth(2) == Some('l')) + { + log::debug!( + "Detected skylake derivative running on mesa i915. Clears to srgb textures will \ + use manual shader clears." + ); + workarounds.set(super::Workarounds::MESA_I915_SRGB_SHADER_CLEAR, true); + } + + let downlevel_defaults = wgt::DownlevelLimits {}; + let max_samples = unsafe { gl.get_parameter_i32(glow::MAX_SAMPLES) }; + + // Drop the GL guard so we can move the context into AdapterShared + // ( on Wasm the gl handle is just a ref so we tell clippy to allow + // dropping the ref ) + #[cfg_attr(target_arch = "wasm32", allow(dropping_references))] + drop(gl); + + Some(crate::ExposedAdapter { + adapter: super::Adapter { + shared: Arc::new(super::AdapterShared { + context, + private_caps, + workarounds, + features, + limits: limits.clone(), + options: backend_options, + shading_language_version, + next_shader_id: Default::default(), + program_cache: Default::default(), + es: es_ver.is_some(), + max_msaa_samples: max_samples, + }), + }, + info: Self::make_info(vendor, renderer, version), + features, + capabilities: crate::Capabilities { + limits, + downlevel: wgt::DownlevelCapabilities { + flags: downlevel_flags, + limits: downlevel_defaults, + shader_model: wgt::ShaderModel::Sm5, + }, + alignments: crate::Alignments { + buffer_copy_offset: wgt::BufferSize::new(4).unwrap(), + buffer_copy_pitch: wgt::BufferSize::new(4).unwrap(), + // #6151: `wgpu_hal::gles` doesn't ask Naga to inject bounds + // checks in GLSL, and it doesn't request extensions like + // `KHR_robust_buffer_access_behavior` that would provide + // them, so we can't really implement the checks promised by + // [`crate::BufferBinding`]. + // + // Since this is a pre-existing condition, for the time + // being, provide 1 as the value here, to cause as little + // trouble as possible. + uniform_bounds_check_alignment: wgt::BufferSize::new(1).unwrap(), + raw_tlas_instance_size: 0, + ray_tracing_scratch_buffer_alignment: 0, + }, + cooperative_matrix_properties: Vec::new(), + }, + }) + } + + unsafe fn compile_shader( + source: &str, + gl: &glow::Context, + shader_type: u32, + es: bool, + ) -> Option { + let source = if es { + format!("#version 300 es\nprecision lowp float;\n{source}") + } else { + let version = gl.version(); + if version.major == 3 && version.minor == 0 { + // OpenGL 3.0 only supports this format + format!("#version 130\n{source}") + } else { + // OpenGL 3.1+ support this format + format!("#version 140\n{source}") + } + }; + let shader = unsafe { gl.create_shader(shader_type) }.expect("Could not create shader"); + unsafe { gl.shader_source(shader, &source) }; + unsafe { gl.compile_shader(shader) }; + + if !unsafe { gl.get_shader_compile_status(shader) } { + let msg = unsafe { gl.get_shader_info_log(shader) }; + if !msg.is_empty() { + log::error!("\tShader compile error: {msg}"); + } + unsafe { gl.delete_shader(shader) }; + None + } else { + Some(shader) + } + } + + unsafe fn create_shader_clear_program( + gl: &glow::Context, + es: bool, + ) -> Option { + let program = unsafe { gl.create_program() }.expect("Could not create shader program"); + let vertex = unsafe { + Self::compile_shader( + include_str!("./shaders/clear.vert"), + gl, + glow::VERTEX_SHADER, + es, + )? + }; + let fragment = unsafe { + Self::compile_shader( + include_str!("./shaders/clear.frag"), + gl, + glow::FRAGMENT_SHADER, + es, + )? + }; + unsafe { gl.attach_shader(program, vertex) }; + unsafe { gl.attach_shader(program, fragment) }; + unsafe { gl.link_program(program) }; + + let linked_ok = unsafe { gl.get_program_link_status(program) }; + let msg = unsafe { gl.get_program_info_log(program) }; + if !msg.is_empty() { + log::error!("Shader link error: {msg}"); + } + if !linked_ok { + return None; + } + + let color_uniform_location = unsafe { gl.get_uniform_location(program, "color") } + .expect("Could not find color uniform in shader clear shader"); + unsafe { gl.delete_shader(vertex) }; + unsafe { gl.delete_shader(fragment) }; + + Some(ShaderClearProgram { + program, + color_uniform_location, + }) + } +} + +impl crate::Adapter for super::Adapter { + type A = super::Api; + + unsafe fn open( + &self, + features: wgt::Features, + _limits: &wgt::Limits, + _memory_hints: &wgt::MemoryHints, + ) -> Result, crate::DeviceError> { + let gl = &self.shared.context.lock(); + unsafe { gl.pixel_store_i32(glow::UNPACK_ALIGNMENT, 1) }; + unsafe { gl.pixel_store_i32(glow::PACK_ALIGNMENT, 1) }; + let main_vao = + unsafe { gl.create_vertex_array() }.map_err(|_| crate::DeviceError::OutOfMemory)?; + unsafe { gl.bind_vertex_array(Some(main_vao)) }; + + let zero_buffer = + unsafe { gl.create_buffer() }.map_err(|_| crate::DeviceError::OutOfMemory)?; + unsafe { gl.bind_buffer(glow::COPY_READ_BUFFER, Some(zero_buffer)) }; + let zeroes = vec![0u8; super::ZERO_BUFFER_SIZE]; + unsafe { gl.buffer_data_u8_slice(glow::COPY_READ_BUFFER, &zeroes, glow::STATIC_DRAW) }; + + // Compile the shader program we use for doing manual clears to work around Mesa fastclear + // bug. + + let shader_clear_program = if self + .shared + .workarounds + .contains(super::Workarounds::MESA_I915_SRGB_SHADER_CLEAR) + { + Some(unsafe { + Self::create_shader_clear_program(gl, self.shared.es) + .ok_or(crate::DeviceError::Lost)? + }) + } else { + // If we don't need the workaround, don't waste time and resources compiling the clear program + None + }; + + Ok(crate::OpenDevice { + device: super::Device { + shared: Arc::clone(&self.shared), + main_vao, + #[cfg(all(native, feature = "renderdoc"))] + render_doc: Default::default(), + counters: Default::default(), + }, + queue: super::Queue { + shared: Arc::clone(&self.shared), + features, + draw_fbo: unsafe { gl.create_framebuffer() } + .map_err(|_| crate::DeviceError::OutOfMemory)?, + copy_fbo: unsafe { gl.create_framebuffer() } + .map_err(|_| crate::DeviceError::OutOfMemory)?, + shader_clear_program, + zero_buffer, + temp_query_results: Mutex::new(Vec::new()), + draw_buffer_count: AtomicU8::new(1), + current_index_buffer: Mutex::new(None), + }, + }) + } + + unsafe fn texture_format_capabilities( + &self, + format: wgt::TextureFormat, + ) -> crate::TextureFormatCapabilities { + use crate::TextureFormatCapabilities as Tfc; + use wgt::TextureFormat as Tf; + + let sample_count = { + let max_samples = self.shared.max_msaa_samples; + if max_samples >= 16 { + Tfc::MULTISAMPLE_X2 + | Tfc::MULTISAMPLE_X4 + | Tfc::MULTISAMPLE_X8 + | Tfc::MULTISAMPLE_X16 + } else if max_samples >= 8 { + Tfc::MULTISAMPLE_X2 | Tfc::MULTISAMPLE_X4 | Tfc::MULTISAMPLE_X8 + } else { + // The lowest supported level in GLE3.0/WebGL2 is 4X + // (see GL_MAX_SAMPLES in https://registry.khronos.org/OpenGL-Refpages/es3.0/html/glGet.xhtml). + // On some platforms, like iOS Safari, `get_parameter_i32(MAX_SAMPLES)` returns 0, + // so we always fall back to supporting 4x here. + Tfc::MULTISAMPLE_X2 | Tfc::MULTISAMPLE_X4 + } + }; + + // Base types are pulled from the table in the OpenGLES 3.0 spec in section 3.8. + // + // The storage types are based on table 8.26, in section + // "TEXTURE IMAGE LOADS AND STORES" of OpenGLES-3.2 spec. + let empty = Tfc::empty(); + let base = Tfc::COPY_SRC | Tfc::COPY_DST; + let unfilterable = base | Tfc::SAMPLED; + let depth = base | Tfc::SAMPLED | sample_count | Tfc::DEPTH_STENCIL_ATTACHMENT; + let filterable = unfilterable | Tfc::SAMPLED_LINEAR; + let renderable = + unfilterable | Tfc::COLOR_ATTACHMENT | sample_count | Tfc::MULTISAMPLE_RESOLVE; + let filterable_renderable = filterable | renderable | Tfc::COLOR_ATTACHMENT_BLEND; + let storage = + base | Tfc::STORAGE_READ_WRITE | Tfc::STORAGE_READ_ONLY | Tfc::STORAGE_WRITE_ONLY; + + let feature_fn = |f, caps| { + if self.shared.features.contains(f) { + caps + } else { + empty + } + }; + + let bcn_features = feature_fn(wgt::Features::TEXTURE_COMPRESSION_BC, filterable); + let etc2_features = feature_fn(wgt::Features::TEXTURE_COMPRESSION_ETC2, filterable); + let astc_features = feature_fn(wgt::Features::TEXTURE_COMPRESSION_ASTC, filterable); + let astc_hdr_features = feature_fn(wgt::Features::TEXTURE_COMPRESSION_ASTC_HDR, filterable); + + let private_caps_fn = |f, caps| { + if self.shared.private_caps.contains(f) { + caps + } else { + empty + } + }; + + let half_float_renderable = private_caps_fn( + super::PrivateCapabilities::COLOR_BUFFER_HALF_FLOAT, + Tfc::COLOR_ATTACHMENT + | Tfc::COLOR_ATTACHMENT_BLEND + | sample_count + | Tfc::MULTISAMPLE_RESOLVE, + ); + + let float_renderable = private_caps_fn( + super::PrivateCapabilities::COLOR_BUFFER_FLOAT, + Tfc::COLOR_ATTACHMENT + | Tfc::COLOR_ATTACHMENT_BLEND + | sample_count + | Tfc::MULTISAMPLE_RESOLVE, + ); + + let texture_float_linear = feature_fn(wgt::Features::FLOAT32_FILTERABLE, filterable); + + let image_atomic = feature_fn(wgt::Features::TEXTURE_ATOMIC, Tfc::STORAGE_ATOMIC); + let image_64_atomic = feature_fn(wgt::Features::TEXTURE_INT64_ATOMIC, Tfc::STORAGE_ATOMIC); + + match format { + Tf::R8Unorm => filterable_renderable, + Tf::R8Snorm => filterable, + Tf::R8Uint => renderable, + Tf::R8Sint => renderable, + Tf::R16Uint => renderable, + Tf::R16Sint => renderable, + Tf::R16Unorm => empty, + Tf::R16Snorm => empty, + Tf::R16Float => filterable | half_float_renderable, + Tf::Rg8Unorm => filterable_renderable, + Tf::Rg8Snorm => filterable, + Tf::Rg8Uint => renderable, + Tf::Rg8Sint => renderable, + Tf::R32Uint => renderable | storage | image_atomic, + Tf::R32Sint => renderable | storage | image_atomic, + Tf::R32Float => unfilterable | storage | float_renderable | texture_float_linear, + Tf::Rg16Uint => renderable, + Tf::Rg16Sint => renderable, + Tf::Rg16Unorm => empty, + Tf::Rg16Snorm => empty, + Tf::Rg16Float => filterable | half_float_renderable, + Tf::Rgba8Unorm => filterable_renderable | storage, + Tf::Rgba8UnormSrgb => filterable_renderable, + Tf::Bgra8Unorm | Tf::Bgra8UnormSrgb => filterable_renderable, + Tf::Rgba8Snorm => filterable | storage, + Tf::Rgba8Uint => renderable | storage, + Tf::Rgba8Sint => renderable | storage, + Tf::Rgb10a2Uint => renderable, + Tf::Rgb10a2Unorm => filterable_renderable, + Tf::Rg11b10Ufloat => filterable | float_renderable, + Tf::R64Uint => image_64_atomic, + Tf::Rg32Uint => renderable, + Tf::Rg32Sint => renderable, + Tf::Rg32Float => unfilterable | float_renderable | texture_float_linear, + Tf::Rgba16Uint => renderable | storage, + Tf::Rgba16Sint => renderable | storage, + Tf::Rgba16Unorm => empty, + Tf::Rgba16Snorm => empty, + Tf::Rgba16Float => filterable | storage | half_float_renderable, + Tf::Rgba32Uint => renderable | storage, + Tf::Rgba32Sint => renderable | storage, + Tf::Rgba32Float => unfilterable | storage | float_renderable | texture_float_linear, + Tf::Stencil8 + | Tf::Depth16Unorm + | Tf::Depth32Float + | Tf::Depth32FloatStencil8 + | Tf::Depth24Plus + | Tf::Depth24PlusStencil8 => depth, + Tf::NV12 => empty, + Tf::P010 => empty, + Tf::Rgb9e5Ufloat => filterable, + Tf::Bc1RgbaUnorm + | Tf::Bc1RgbaUnormSrgb + | Tf::Bc2RgbaUnorm + | Tf::Bc2RgbaUnormSrgb + | Tf::Bc3RgbaUnorm + | Tf::Bc3RgbaUnormSrgb + | Tf::Bc4RUnorm + | Tf::Bc4RSnorm + | Tf::Bc5RgUnorm + | Tf::Bc5RgSnorm + | Tf::Bc6hRgbFloat + | Tf::Bc6hRgbUfloat + | Tf::Bc7RgbaUnorm + | Tf::Bc7RgbaUnormSrgb => bcn_features, + Tf::Etc2Rgb8Unorm + | Tf::Etc2Rgb8UnormSrgb + | Tf::Etc2Rgb8A1Unorm + | Tf::Etc2Rgb8A1UnormSrgb + | Tf::Etc2Rgba8Unorm + | Tf::Etc2Rgba8UnormSrgb + | Tf::EacR11Unorm + | Tf::EacR11Snorm + | Tf::EacRg11Unorm + | Tf::EacRg11Snorm => etc2_features, + Tf::Astc { + block: _, + channel: AstcChannel::Unorm | AstcChannel::UnormSrgb, + } => astc_features, + Tf::Astc { + block: _, + channel: AstcChannel::Hdr, + } => astc_hdr_features, + } + } + + unsafe fn surface_capabilities( + &self, + surface: &super::Surface, + ) -> Option { + #[cfg(webgl)] + if self.shared.context.webgl2_context != surface.webgl2_context { + return None; + } + + if surface.presentable { + let mut formats = vec![ + wgt::TextureFormat::Rgba8Unorm, + #[cfg(native)] + wgt::TextureFormat::Bgra8Unorm, + ]; + if surface.supports_srgb() { + formats.extend([ + wgt::TextureFormat::Rgba8UnormSrgb, + #[cfg(native)] + wgt::TextureFormat::Bgra8UnormSrgb, + ]) + } + if self + .shared + .private_caps + .contains(super::PrivateCapabilities::COLOR_BUFFER_HALF_FLOAT) + { + formats.push(wgt::TextureFormat::Rgba16Float) + } + + Some(crate::SurfaceCapabilities { + formats, + present_modes: if cfg!(windows) { + vec![wgt::PresentMode::Fifo, wgt::PresentMode::Immediate] + } else { + vec![wgt::PresentMode::Fifo] //TODO + }, + composite_alpha_modes: vec![wgt::CompositeAlphaMode::Opaque], //TODO + maximum_frame_latency: 2..=2, //TODO, unused currently + current_extent: None, + usage: wgt::TextureUses::COLOR_TARGET, + }) + } else { + None + } + } + + unsafe fn get_presentation_timestamp(&self) -> wgt::PresentationTimestamp { + wgt::PresentationTimestamp::INVALID_TIMESTAMP + } + + fn get_ordered_buffer_usages(&self) -> wgt::BufferUses { + wgt::BufferUses::INCLUSIVE | wgt::BufferUses::MAP_WRITE + } + + // Don't put barriers between inclusive uses + fn get_ordered_texture_usages(&self) -> wgt::TextureUses { + wgt::TextureUses::INCLUSIVE + | wgt::TextureUses::COLOR_TARGET + | wgt::TextureUses::DEPTH_STENCIL_WRITE + } +} + +impl super::AdapterShared { + pub(super) unsafe fn get_buffer_sub_data( + &self, + gl: &glow::Context, + target: u32, + offset: i32, + dst_data: &mut [u8], + ) { + if self + .private_caps + .contains(super::PrivateCapabilities::GET_BUFFER_SUB_DATA) + { + unsafe { gl.get_buffer_sub_data(target, offset, dst_data) }; + } else { + log::error!("Fake map"); + let length = dst_data.len(); + let buffer_mapping = + unsafe { gl.map_buffer_range(target, offset, length as _, glow::MAP_READ_BIT) }; + + unsafe { + core::ptr::copy_nonoverlapping(buffer_mapping, dst_data.as_mut_ptr(), length) + }; + + unsafe { gl.unmap_buffer(target) }; + } + } +} + +#[cfg(send_sync)] +unsafe impl Sync for super::Adapter {} +#[cfg(send_sync)] +unsafe impl Send for super::Adapter {} + +#[cfg(test)] +mod tests { + use super::super::Adapter; + + #[test] + fn test_version_parse() { + Adapter::parse_version("1").unwrap_err(); + Adapter::parse_version("1.").unwrap_err(); + Adapter::parse_version("1 h3l1o. W0rld").unwrap_err(); + Adapter::parse_version("1. h3l1o. W0rld").unwrap_err(); + Adapter::parse_version("1.2.3").unwrap_err(); + + assert_eq!(Adapter::parse_version("OpenGL ES 3.1").unwrap(), (3, 1)); + assert_eq!( + Adapter::parse_version("OpenGL ES 2.0 Google Nexus").unwrap(), + (2, 0) + ); + assert_eq!(Adapter::parse_version("GLSL ES 1.1").unwrap(), (1, 1)); + assert_eq!( + Adapter::parse_version("OpenGL ES GLSL ES 3.20").unwrap(), + (3, 2) + ); + assert_eq!( + // WebGL 2.0 should parse as OpenGL ES 3.0 + Adapter::parse_version("WebGL 2.0 (OpenGL ES 3.0 Chromium)").unwrap(), + (3, 0) + ); + assert_eq!( + Adapter::parse_version("WebGL GLSL ES 3.00 (OpenGL ES GLSL ES 3.0 Chromium)").unwrap(), + (3, 0) + ); + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/command.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/command.rs new file mode 100644 index 00000000..b03a5604 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/command.rs @@ -0,0 +1,1297 @@ +use alloc::string::String; +use core::{mem, ops::Range}; + +use arrayvec::ArrayVec; + +use super::{conv, Command as C}; + +#[derive(Clone, Copy, Debug, Default)] +struct TextureSlotDesc { + tex_target: super::BindTarget, + sampler_index: Option, +} + +pub(super) struct State { + topology: u32, + primitive: super::PrimitiveState, + index_format: wgt::IndexFormat, + index_offset: wgt::BufferAddress, + vertex_buffers: + [(super::VertexBufferDesc, Option); crate::MAX_VERTEX_BUFFERS], + vertex_attributes: ArrayVec, + color_targets: ArrayVec, + stencil: super::StencilState, + depth_bias: wgt::DepthBiasState, + alpha_to_coverage_enabled: bool, + samplers: [Option; super::MAX_SAMPLERS], + texture_slots: [TextureSlotDesc; super::MAX_TEXTURE_SLOTS], + render_size: wgt::Extent3d, + resolve_attachments: ArrayVec<(u32, super::TextureView), { crate::MAX_COLOR_ATTACHMENTS }>, + invalidate_attachments: ArrayVec, + has_pass_label: bool, + instance_vbuf_mask: usize, + dirty_vbuf_mask: usize, + active_first_instance: u32, + first_instance_location: Option, + immediates_descs: ArrayVec, + // The current state of the immediate data block. + current_immediates_data: [u32; super::MAX_IMMEDIATES], + end_of_pass_timestamp: Option, + clip_distance_count: u32, +} + +impl Default for State { + fn default() -> Self { + Self { + topology: Default::default(), + primitive: Default::default(), + index_format: Default::default(), + index_offset: Default::default(), + vertex_buffers: Default::default(), + vertex_attributes: Default::default(), + color_targets: Default::default(), + stencil: Default::default(), + depth_bias: Default::default(), + alpha_to_coverage_enabled: Default::default(), + samplers: Default::default(), + texture_slots: Default::default(), + render_size: Default::default(), + resolve_attachments: Default::default(), + invalidate_attachments: Default::default(), + has_pass_label: Default::default(), + instance_vbuf_mask: Default::default(), + dirty_vbuf_mask: Default::default(), + active_first_instance: Default::default(), + first_instance_location: Default::default(), + immediates_descs: Default::default(), + current_immediates_data: [0; super::MAX_IMMEDIATES], + end_of_pass_timestamp: Default::default(), + clip_distance_count: Default::default(), + } + } +} + +impl super::CommandBuffer { + fn clear(&mut self) { + self.label = None; + self.commands.clear(); + self.data_bytes.clear(); + self.queries.clear(); + } + + fn add_marker(&mut self, marker: &str) -> Range { + let start = self.data_bytes.len() as u32; + self.data_bytes.extend(marker.as_bytes()); + start..self.data_bytes.len() as u32 + } + + fn add_immediates_data(&mut self, data: &[u32]) -> Range { + let data_raw = bytemuck::cast_slice(data); + let start = self.data_bytes.len(); + assert!(start < u32::MAX as usize); + self.data_bytes.extend_from_slice(data_raw); + let end = self.data_bytes.len(); + assert!(end < u32::MAX as usize); + (start as u32)..(end as u32) + } +} + +impl Drop for super::CommandEncoder { + fn drop(&mut self) { + use crate::CommandEncoder; + unsafe { self.discard_encoding() } + self.counters.command_encoders.sub(1); + } +} + +impl super::CommandEncoder { + fn rebind_stencil_func(&mut self) { + fn make(s: &super::StencilSide, face: u32) -> C { + C::SetStencilFunc { + face, + function: s.function, + reference: s.reference, + read_mask: s.mask_read, + } + } + + let s = &self.state.stencil; + if s.front.function == s.back.function + && s.front.mask_read == s.back.mask_read + && s.front.reference == s.back.reference + { + self.cmd_buffer + .commands + .push(make(&s.front, glow::FRONT_AND_BACK)); + } else { + self.cmd_buffer.commands.push(make(&s.front, glow::FRONT)); + self.cmd_buffer.commands.push(make(&s.back, glow::BACK)); + } + } + + fn rebind_vertex_data(&mut self, first_instance: u32) { + if self + .private_caps + .contains(super::PrivateCapabilities::VERTEX_BUFFER_LAYOUT) + { + for (index, pair) in self.state.vertex_buffers.iter().enumerate() { + if self.state.dirty_vbuf_mask & (1 << index) == 0 { + continue; + } + let (buffer_desc, vb) = match *pair { + // Not all dirty bindings are necessarily filled. Some may be unused. + (_, None) => continue, + (ref vb_desc, Some(ref vb)) => (vb_desc.clone(), vb), + }; + let instance_offset = match buffer_desc.step { + wgt::VertexStepMode::Vertex => 0, + wgt::VertexStepMode::Instance => first_instance * buffer_desc.stride, + }; + + self.cmd_buffer.commands.push(C::SetVertexBuffer { + index: index as u32, + buffer: super::BufferBinding { + raw: vb.raw, + offset: vb.offset + instance_offset as wgt::BufferAddress, + }, + buffer_desc, + }); + self.state.dirty_vbuf_mask ^= 1 << index; + } + } else { + let mut vbuf_mask = 0; + for attribute in self.state.vertex_attributes.iter() { + if self.state.dirty_vbuf_mask & (1 << attribute.buffer_index) == 0 { + continue; + } + let (buffer_desc, vb) = + match self.state.vertex_buffers[attribute.buffer_index as usize] { + // Not all dirty bindings are necessarily filled. Some may be unused. + (_, None) => continue, + (ref vb_desc, Some(ref vb)) => (vb_desc.clone(), vb), + }; + + let mut attribute_desc = attribute.clone(); + attribute_desc.offset += vb.offset as u32; + if buffer_desc.step == wgt::VertexStepMode::Instance { + attribute_desc.offset += buffer_desc.stride * first_instance; + } + + self.cmd_buffer.commands.push(C::SetVertexAttribute { + buffer: Some(vb.raw), + buffer_desc, + attribute_desc, + }); + vbuf_mask |= 1 << attribute.buffer_index; + } + self.state.dirty_vbuf_mask ^= vbuf_mask; + } + } + + fn rebind_sampler_states(&mut self, dirty_textures: u32, dirty_samplers: u32) { + for (texture_index, slot) in self.state.texture_slots.iter().enumerate() { + if dirty_textures & (1 << texture_index) != 0 + || slot + .sampler_index + .is_some_and(|si| dirty_samplers & (1 << si) != 0) + { + let sampler = slot + .sampler_index + .and_then(|si| self.state.samplers[si as usize]); + self.cmd_buffer + .commands + .push(C::BindSampler(texture_index as u32, sampler)); + } + } + } + + fn prepare_draw(&mut self, first_instance: u32) { + // If we support fully featured instancing, we want to bind everything as normal + // and let the draw call sort it out. + let emulated_first_instance_value = if self + .private_caps + .contains(super::PrivateCapabilities::FULLY_FEATURED_INSTANCING) + { + 0 + } else { + first_instance + }; + + if emulated_first_instance_value != self.state.active_first_instance { + // rebind all per-instance buffers on first-instance change + self.state.dirty_vbuf_mask |= self.state.instance_vbuf_mask; + self.state.active_first_instance = emulated_first_instance_value; + } + if self.state.dirty_vbuf_mask != 0 { + self.rebind_vertex_data(emulated_first_instance_value); + } + } + + fn set_pipeline_inner(&mut self, inner: &super::PipelineInner) { + self.cmd_buffer.commands.push(C::SetProgram(inner.program)); + + self.state + .first_instance_location + .clone_from(&inner.first_instance_location); + self.state + .immediates_descs + .clone_from(&inner.immediates_descs); + + // rebind textures, if needed + let mut dirty_textures = 0u32; + for (texture_index, (slot, &sampler_index)) in self + .state + .texture_slots + .iter_mut() + .zip(inner.sampler_map.iter()) + .enumerate() + { + if slot.sampler_index != sampler_index { + slot.sampler_index = sampler_index; + dirty_textures |= 1 << texture_index; + } + } + if dirty_textures != 0 { + self.rebind_sampler_states(dirty_textures, 0); + } + } +} + +impl crate::CommandEncoder for super::CommandEncoder { + type A = super::Api; + + unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> { + self.state = State::default(); + self.cmd_buffer.label = label.map(String::from); + Ok(()) + } + unsafe fn discard_encoding(&mut self) { + self.cmd_buffer.clear(); + } + unsafe fn end_encoding(&mut self) -> Result { + Ok(mem::take(&mut self.cmd_buffer)) + } + unsafe fn reset_all(&mut self, _command_buffers: I) { + //TODO: could re-use the allocations in all these command buffers + } + + unsafe fn transition_buffers<'a, T>(&mut self, barriers: T) + where + T: Iterator>, + { + if !self + .private_caps + .contains(super::PrivateCapabilities::MEMORY_BARRIERS) + { + return; + } + for bar in barriers { + // GLES only synchronizes storage -> anything explicitly + if !bar.usage.from.contains(wgt::BufferUses::STORAGE_READ_WRITE) { + continue; + } + self.cmd_buffer + .commands + .push(C::BufferBarrier(bar.buffer.raw.unwrap(), bar.usage.to)); + } + } + + unsafe fn transition_textures<'a, T>(&mut self, barriers: T) + where + T: Iterator>, + { + if !self + .private_caps + .contains(super::PrivateCapabilities::MEMORY_BARRIERS) + { + return; + } + + let mut combined_usage = wgt::TextureUses::empty(); + for bar in barriers { + // GLES only synchronizes storage -> anything explicitly + // if shader writes to a texture then barriers should be placed + if !bar.usage.from.intersects( + wgt::TextureUses::STORAGE_READ_WRITE | wgt::TextureUses::STORAGE_WRITE_ONLY, + ) { + continue; + } + // unlike buffers, there is no need for a concrete texture + // object to be bound anywhere for a barrier + combined_usage |= bar.usage.to; + } + + if !combined_usage.is_empty() { + self.cmd_buffer + .commands + .push(C::TextureBarrier(combined_usage)); + } + } + + unsafe fn clear_buffer(&mut self, buffer: &super::Buffer, range: crate::MemoryRange) { + self.cmd_buffer.commands.push(C::ClearBuffer { + dst: buffer.clone(), + dst_target: buffer.target, + range, + }); + } + + unsafe fn copy_buffer_to_buffer( + &mut self, + src: &super::Buffer, + dst: &super::Buffer, + regions: T, + ) where + T: Iterator, + { + let (src_target, dst_target) = if src.target == dst.target { + (glow::COPY_READ_BUFFER, glow::COPY_WRITE_BUFFER) + } else { + (src.target, dst.target) + }; + for copy in regions { + self.cmd_buffer.commands.push(C::CopyBufferToBuffer { + src: src.clone(), + src_target, + dst: dst.clone(), + dst_target, + copy, + }) + } + } + + #[cfg(webgl)] + unsafe fn copy_external_image_to_texture( + &mut self, + src: &wgt::CopyExternalImageSourceInfo, + dst: &super::Texture, + dst_premultiplication: bool, + regions: T, + ) where + T: Iterator, + { + let (dst_raw, dst_target) = dst.inner.as_native(); + for copy in regions { + self.cmd_buffer + .commands + .push(C::CopyExternalImageToTexture { + src: src.clone(), + dst: dst_raw, + dst_target, + dst_format: dst.format, + dst_premultiplication, + copy, + }) + } + } + + unsafe fn copy_texture_to_texture( + &mut self, + src: &super::Texture, + _src_usage: wgt::TextureUses, + dst: &super::Texture, + regions: T, + ) where + T: Iterator, + { + let (src_raw, src_target) = src.inner.as_native(); + let (dst_raw, dst_target) = dst.inner.as_native(); + for mut copy in regions { + copy.clamp_size_to_virtual(&src.copy_size, &dst.copy_size); + self.cmd_buffer.commands.push(C::CopyTextureToTexture { + src: src_raw, + src_target, + dst: dst_raw, + dst_target, + copy, + }) + } + } + + unsafe fn copy_buffer_to_texture( + &mut self, + src: &super::Buffer, + dst: &super::Texture, + regions: T, + ) where + T: Iterator, + { + let (dst_raw, dst_target) = dst.inner.as_native(); + + for mut copy in regions { + copy.clamp_size_to_virtual(&dst.copy_size); + self.cmd_buffer.commands.push(C::CopyBufferToTexture { + src: src.clone(), + src_target: src.target, + dst: dst_raw, + dst_target, + dst_format: dst.format, + copy, + }) + } + } + + unsafe fn copy_texture_to_buffer( + &mut self, + src: &super::Texture, + _src_usage: wgt::TextureUses, + dst: &super::Buffer, + regions: T, + ) where + T: Iterator, + { + let (src_raw, src_target) = src.inner.as_native(); + for mut copy in regions { + copy.clamp_size_to_virtual(&src.copy_size); + self.cmd_buffer.commands.push(C::CopyTextureToBuffer { + src: src_raw, + src_target, + src_format: src.format, + dst: dst.clone(), + dst_target: dst.target, + copy, + }) + } + } + + unsafe fn begin_query(&mut self, set: &super::QuerySet, index: u32) { + let query = set.queries[index as usize]; + self.cmd_buffer + .commands + .push(C::BeginQuery(query, set.target)); + } + unsafe fn end_query(&mut self, set: &super::QuerySet, _index: u32) { + self.cmd_buffer.commands.push(C::EndQuery(set.target)); + } + unsafe fn write_timestamp(&mut self, set: &super::QuerySet, index: u32) { + let query = set.queries[index as usize]; + self.cmd_buffer.commands.push(C::TimestampQuery(query)); + } + unsafe fn reset_queries(&mut self, _set: &super::QuerySet, _range: Range) { + //TODO: what do we do here? + } + unsafe fn copy_query_results( + &mut self, + set: &super::QuerySet, + range: Range, + buffer: &super::Buffer, + offset: wgt::BufferAddress, + _stride: wgt::BufferSize, + ) { + let start = self.cmd_buffer.queries.len(); + self.cmd_buffer + .queries + .extend_from_slice(&set.queries[range.start as usize..range.end as usize]); + let query_range = start as u32..self.cmd_buffer.queries.len() as u32; + self.cmd_buffer.commands.push(C::CopyQueryResults { + query_range, + dst: buffer.clone(), + dst_target: buffer.target, + dst_offset: offset, + }); + } + + // render + + unsafe fn begin_render_pass( + &mut self, + desc: &crate::RenderPassDescriptor, + ) -> Result<(), crate::DeviceError> { + debug_assert!(self.state.end_of_pass_timestamp.is_none()); + if let Some(ref t) = desc.timestamp_writes { + if let Some(index) = t.beginning_of_pass_write_index { + unsafe { self.write_timestamp(t.query_set, index) } + } + self.state.end_of_pass_timestamp = t + .end_of_pass_write_index + .map(|index| t.query_set.queries[index as usize]); + } + + self.state.render_size = desc.extent; + self.state.resolve_attachments.clear(); + self.state.invalidate_attachments.clear(); + if let Some(label) = desc.label { + let range = self.cmd_buffer.add_marker(label); + self.cmd_buffer.commands.push(C::PushDebugGroup(range)); + self.state.has_pass_label = true; + } + + let rendering_to_external_framebuffer = desc + .color_attachments + .iter() + .filter_map(|at| at.as_ref()) + .any(|at| match at.target.view.inner { + #[cfg(webgl)] + super::TextureInner::ExternalFramebuffer { .. } => true, + #[cfg(native)] + super::TextureInner::ExternalNativeFramebuffer { .. } => true, + _ => false, + }); + + if rendering_to_external_framebuffer && desc.color_attachments.len() != 1 { + panic!("Multiple render attachments with external framebuffers are not supported."); + } + + // `COLOR_ATTACHMENT0` to `COLOR_ATTACHMENT31` gives 32 possible color attachments. + assert!(desc.color_attachments.len() <= 32); + + match desc + .color_attachments + .first() + .filter(|at| at.is_some()) + .and_then(|at| at.as_ref().map(|at| &at.target.view.inner)) + { + // default framebuffer (provided externally) + Some(&super::TextureInner::DefaultRenderbuffer) => { + self.cmd_buffer + .commands + .push(C::ResetFramebuffer { is_default: true }); + } + _ => { + // set the framebuffer + self.cmd_buffer + .commands + .push(C::ResetFramebuffer { is_default: false }); + + for (i, cat) in desc.color_attachments.iter().enumerate() { + if let Some(cat) = cat.as_ref() { + let attachment = glow::COLOR_ATTACHMENT0 + i as u32; + // Try to use the multisampled render-to-texture extension to avoid resolving + if let Some(ref rat) = cat.resolve_target { + if matches!(rat.view.inner, super::TextureInner::Texture { .. }) + && self.private_caps.contains( + super::PrivateCapabilities::MULTISAMPLED_RENDER_TO_TEXTURE, + ) + && !cat.ops.contains(crate::AttachmentOps::STORE) + // Extension specifies that only COLOR_ATTACHMENT0 is valid + && i == 0 + { + self.cmd_buffer.commands.push(C::BindAttachment { + attachment, + view: rat.view.clone(), + depth_slice: None, + sample_count: desc.sample_count, + }); + continue; + } + } + self.cmd_buffer.commands.push(C::BindAttachment { + attachment, + view: cat.target.view.clone(), + depth_slice: cat.depth_slice, + sample_count: 1, + }); + if let Some(ref rat) = cat.resolve_target { + self.state + .resolve_attachments + .push((attachment, rat.view.clone())); + } + if cat.ops.contains(crate::AttachmentOps::STORE_DISCARD) { + self.state.invalidate_attachments.push(attachment); + } + } + } + if let Some(ref dsat) = desc.depth_stencil_attachment { + let aspects = dsat.target.view.aspects; + let attachment = match aspects { + crate::FormatAspects::DEPTH => glow::DEPTH_ATTACHMENT, + crate::FormatAspects::STENCIL => glow::STENCIL_ATTACHMENT, + _ => glow::DEPTH_STENCIL_ATTACHMENT, + }; + self.cmd_buffer.commands.push(C::BindAttachment { + attachment, + view: dsat.target.view.clone(), + depth_slice: None, + sample_count: 1, + }); + if aspects.contains(crate::FormatAspects::DEPTH) + && dsat.depth_ops.contains(crate::AttachmentOps::STORE_DISCARD) + { + self.state + .invalidate_attachments + .push(glow::DEPTH_ATTACHMENT); + } + if aspects.contains(crate::FormatAspects::STENCIL) + && dsat + .stencil_ops + .contains(crate::AttachmentOps::STORE_DISCARD) + { + self.state + .invalidate_attachments + .push(glow::STENCIL_ATTACHMENT); + } + } + } + } + + let rect = crate::Rect { + x: 0, + y: 0, + w: desc.extent.width as i32, + h: desc.extent.height as i32, + }; + self.cmd_buffer.commands.push(C::SetScissor(rect.clone())); + self.cmd_buffer.commands.push(C::SetViewport { + rect, + depth: 0.0..1.0, + }); + + if !rendering_to_external_framebuffer { + // set the draw buffers and states + self.cmd_buffer + .commands + .push(C::SetDrawColorBuffers(desc.color_attachments.len() as u8)); + } + + // issue the clears + for (i, cat) in desc + .color_attachments + .iter() + .filter_map(|at| at.as_ref()) + .enumerate() + { + if cat.ops.contains(crate::AttachmentOps::LOAD_CLEAR) { + let c = &cat.clear_value; + self.cmd_buffer.commands.push( + match cat.target.view.format.sample_type(None, None).unwrap() { + wgt::TextureSampleType::Float { .. } => C::ClearColorF { + draw_buffer: i as u32, + color: [c.r as f32, c.g as f32, c.b as f32, c.a as f32], + is_srgb: cat.target.view.format.is_srgb(), + }, + wgt::TextureSampleType::Uint => C::ClearColorU( + i as u32, + [c.r as u32, c.g as u32, c.b as u32, c.a as u32], + ), + wgt::TextureSampleType::Sint => C::ClearColorI( + i as u32, + [c.r as i32, c.g as i32, c.b as i32, c.a as i32], + ), + wgt::TextureSampleType::Depth => unreachable!(), + }, + ); + } + } + + if let Some(ref dsat) = desc.depth_stencil_attachment { + let clear_depth = dsat.depth_ops.contains(crate::AttachmentOps::LOAD_CLEAR); + let clear_stencil = dsat.stencil_ops.contains(crate::AttachmentOps::LOAD_CLEAR); + + if clear_depth && clear_stencil { + self.cmd_buffer.commands.push(C::ClearDepthAndStencil( + dsat.clear_value.0, + dsat.clear_value.1, + )); + } else if clear_depth { + self.cmd_buffer + .commands + .push(C::ClearDepth(dsat.clear_value.0)); + } else if clear_stencil { + self.cmd_buffer + .commands + .push(C::ClearStencil(dsat.clear_value.1)); + } + } + Ok(()) + } + unsafe fn end_render_pass(&mut self) { + for (attachment, dst) in self.state.resolve_attachments.drain(..) { + self.cmd_buffer.commands.push(C::ResolveAttachment { + attachment, + dst, + size: self.state.render_size, + }); + } + if !self.state.invalidate_attachments.is_empty() { + self.cmd_buffer.commands.push(C::InvalidateAttachments( + self.state.invalidate_attachments.clone(), + )); + self.state.invalidate_attachments.clear(); + } + if self.state.has_pass_label { + self.cmd_buffer.commands.push(C::PopDebugGroup); + self.state.has_pass_label = false; + } + self.state.instance_vbuf_mask = 0; + self.state.dirty_vbuf_mask = 0; + self.state.active_first_instance = 0; + self.state.color_targets.clear(); + for vat in &self.state.vertex_attributes { + self.cmd_buffer + .commands + .push(C::UnsetVertexAttribute(vat.location)); + } + self.state.vertex_attributes.clear(); + self.state.primitive = super::PrimitiveState::default(); + + if let Some(query) = self.state.end_of_pass_timestamp.take() { + self.cmd_buffer.commands.push(C::TimestampQuery(query)); + } + } + + unsafe fn set_bind_group( + &mut self, + layout: &super::PipelineLayout, + index: u32, + group: &super::BindGroup, + dynamic_offsets: &[wgt::DynamicOffset], + ) { + let mut do_index = 0; + let mut dirty_textures = 0u32; + let mut dirty_samplers = 0u32; + let group_info = layout.group_infos[index as usize].as_ref().unwrap(); + + for (binding_layout, raw_binding) in group_info.entries.iter().zip(group.contents.iter()) { + let slot = group_info.binding_to_slot[binding_layout.binding as usize] as u32; + match *raw_binding { + super::RawBinding::Buffer { + raw, + offset: base_offset, + size, + } => { + let mut offset = base_offset; + let target = match binding_layout.ty { + wgt::BindingType::Buffer { + ty, + has_dynamic_offset, + min_binding_size: _, + } => { + if has_dynamic_offset { + offset += dynamic_offsets[do_index] as i32; + do_index += 1; + } + match ty { + wgt::BufferBindingType::Uniform => glow::UNIFORM_BUFFER, + wgt::BufferBindingType::Storage { .. } => { + glow::SHADER_STORAGE_BUFFER + } + } + } + _ => unreachable!(), + }; + self.cmd_buffer.commands.push(C::BindBuffer { + target, + slot, + buffer: raw, + offset, + size, + }); + } + super::RawBinding::Sampler(sampler) => { + dirty_samplers |= 1 << slot; + self.state.samplers[slot as usize] = Some(sampler); + } + super::RawBinding::Texture { + raw, + target, + aspects, + ref mip_levels, + } => { + dirty_textures |= 1 << slot; + self.state.texture_slots[slot as usize].tex_target = target; + self.cmd_buffer.commands.push(C::BindTexture { + slot, + texture: raw, + target, + aspects, + mip_levels: mip_levels.clone(), + }); + } + super::RawBinding::Image(ref binding) => { + self.cmd_buffer.commands.push(C::BindImage { + slot, + binding: binding.clone(), + }); + } + } + } + + self.rebind_sampler_states(dirty_textures, dirty_samplers); + } + + unsafe fn set_immediates( + &mut self, + _layout: &super::PipelineLayout, + offset_bytes: u32, + data: &[u32], + ) { + // There is nothing preventing the user from trying to update a single value within + // a vector or matrix in the set_immediates call, as to the user, all of this is + // just memory. However OpenGL does not allow partial uniform updates. + // + // As such, we locally keep a copy of the current state of the immediate data memory + // block. If the user tries to update a single value, we have the data to update the entirety + // of the uniform. + let start_words = offset_bytes / 4; + let end_words = start_words + data.len() as u32; + self.state.current_immediates_data[start_words as usize..end_words as usize] + .copy_from_slice(data); + + // We iterate over the uniform list as there may be multiple uniforms that need + // updating from the same immediate data memory (one for each shader stage). + // + // Additionally, any statically unused uniform descs will have been removed from this list + // by OpenGL, so the uniform list is not contiguous. + for uniform in self.state.immediates_descs.iter().cloned() { + let uniform_size_words = uniform.size_bytes / 4; + let uniform_start_words = uniform.offset / 4; + let uniform_end_words = uniform_start_words + uniform_size_words; + + // Is true if any word within the uniform binding was updated + let needs_updating = + start_words < uniform_end_words || uniform_start_words <= end_words; + + if needs_updating { + let uniform_data = &self.state.current_immediates_data + [uniform_start_words as usize..uniform_end_words as usize]; + + let range = self.cmd_buffer.add_immediates_data(uniform_data); + + self.cmd_buffer.commands.push(C::SetImmediates { + uniform, + offset: range.start, + }); + } + } + } + + unsafe fn insert_debug_marker(&mut self, label: &str) { + let range = self.cmd_buffer.add_marker(label); + self.cmd_buffer.commands.push(C::InsertDebugMarker(range)); + } + unsafe fn begin_debug_marker(&mut self, group_label: &str) { + let range = self.cmd_buffer.add_marker(group_label); + self.cmd_buffer.commands.push(C::PushDebugGroup(range)); + } + unsafe fn end_debug_marker(&mut self) { + self.cmd_buffer.commands.push(C::PopDebugGroup); + } + + unsafe fn set_render_pipeline(&mut self, pipeline: &super::RenderPipeline) { + self.state.topology = conv::map_primitive_topology(pipeline.primitive.topology); + + if self + .private_caps + .contains(super::PrivateCapabilities::VERTEX_BUFFER_LAYOUT) + { + for vat in pipeline.vertex_attributes.iter() { + let vb = &pipeline.vertex_buffers[vat.buffer_index as usize]; + // set the layout + self.cmd_buffer.commands.push(C::SetVertexAttribute { + buffer: None, + buffer_desc: vb.clone(), + attribute_desc: vat.clone(), + }); + } + } else { + for vat in &self.state.vertex_attributes { + self.cmd_buffer + .commands + .push(C::UnsetVertexAttribute(vat.location)); + } + self.state.vertex_attributes.clear(); + + self.state.dirty_vbuf_mask = 0; + // copy vertex attributes + for vat in pipeline.vertex_attributes.iter() { + //Note: we can invalidate more carefully here. + self.state.dirty_vbuf_mask |= 1 << vat.buffer_index; + self.state.vertex_attributes.push(vat.clone()); + } + } + + self.state.instance_vbuf_mask = 0; + // copy vertex state + for (index, (&mut (ref mut state_desc, _), pipe_desc)) in self + .state + .vertex_buffers + .iter_mut() + .zip(pipeline.vertex_buffers.iter()) + .enumerate() + { + if pipe_desc.step == wgt::VertexStepMode::Instance { + self.state.instance_vbuf_mask |= 1 << index; + } + if state_desc != pipe_desc { + self.state.dirty_vbuf_mask |= 1 << index; + *state_desc = pipe_desc.clone(); + } + } + + self.set_pipeline_inner(&pipeline.inner); + + // set primitive state + let prim_state = conv::map_primitive_state(&pipeline.primitive); + if prim_state != self.state.primitive { + self.cmd_buffer + .commands + .push(C::SetPrimitive(prim_state.clone())); + self.state.primitive = prim_state; + } + + // set depth/stencil states + let mut aspects = crate::FormatAspects::empty(); + if pipeline.depth_bias != self.state.depth_bias { + self.state.depth_bias = pipeline.depth_bias; + self.cmd_buffer + .commands + .push(C::SetDepthBias(pipeline.depth_bias)); + } + if let Some(ref depth) = pipeline.depth { + aspects |= crate::FormatAspects::DEPTH; + self.cmd_buffer.commands.push(C::SetDepth(depth.clone())); + } + if let Some(ref stencil) = pipeline.stencil { + aspects |= crate::FormatAspects::STENCIL; + self.state.stencil = stencil.clone(); + self.rebind_stencil_func(); + if stencil.front.ops == stencil.back.ops + && stencil.front.mask_write == stencil.back.mask_write + { + self.cmd_buffer.commands.push(C::SetStencilOps { + face: glow::FRONT_AND_BACK, + write_mask: stencil.front.mask_write, + ops: stencil.front.ops.clone(), + }); + } else { + self.cmd_buffer.commands.push(C::SetStencilOps { + face: glow::FRONT, + write_mask: stencil.front.mask_write, + ops: stencil.front.ops.clone(), + }); + self.cmd_buffer.commands.push(C::SetStencilOps { + face: glow::BACK, + write_mask: stencil.back.mask_write, + ops: stencil.back.ops.clone(), + }); + } + } + self.cmd_buffer + .commands + .push(C::ConfigureDepthStencil(aspects)); + + // set multisampling state + if pipeline.alpha_to_coverage_enabled != self.state.alpha_to_coverage_enabled { + self.state.alpha_to_coverage_enabled = pipeline.alpha_to_coverage_enabled; + self.cmd_buffer + .commands + .push(C::SetAlphaToCoverage(pipeline.alpha_to_coverage_enabled)); + } + + // set blend states + if self.state.color_targets[..] != pipeline.color_targets[..] { + if pipeline + .color_targets + .iter() + .skip(1) + .any(|ct| *ct != pipeline.color_targets[0]) + { + for (index, ct) in pipeline.color_targets.iter().enumerate() { + self.cmd_buffer.commands.push(C::SetColorTarget { + draw_buffer_index: Some(index as u32), + desc: ct.clone(), + }); + } + } else { + self.cmd_buffer.commands.push(C::SetColorTarget { + draw_buffer_index: None, + desc: pipeline.color_targets.first().cloned().unwrap_or_default(), + }); + } + } + self.state.color_targets.clear(); + for ct in pipeline.color_targets.iter() { + self.state.color_targets.push(ct.clone()); + } + + // set clip plane count + if pipeline.inner.clip_distance_count != self.state.clip_distance_count { + self.cmd_buffer.commands.push(C::SetClipDistances { + old_count: self.state.clip_distance_count, + new_count: pipeline.inner.clip_distance_count, + }); + self.state.clip_distance_count = pipeline.inner.clip_distance_count; + } + } + + unsafe fn set_index_buffer<'a>( + &mut self, + binding: crate::BufferBinding<'a, super::Buffer>, + format: wgt::IndexFormat, + ) { + self.state.index_offset = binding.offset; + self.state.index_format = format; + self.cmd_buffer + .commands + .push(C::SetIndexBuffer(binding.buffer.raw.unwrap())); + } + unsafe fn set_vertex_buffer<'a>( + &mut self, + index: u32, + binding: crate::BufferBinding<'a, super::Buffer>, + ) { + self.state.dirty_vbuf_mask |= 1 << index; + let (_, ref mut vb) = self.state.vertex_buffers[index as usize]; + *vb = Some(super::BufferBinding { + raw: binding.buffer.raw.unwrap(), + offset: binding.offset, + }); + } + unsafe fn set_viewport(&mut self, rect: &crate::Rect, depth: Range) { + self.cmd_buffer.commands.push(C::SetViewport { + rect: crate::Rect { + x: rect.x as i32, + y: rect.y as i32, + w: rect.w as i32, + h: rect.h as i32, + }, + depth, + }); + } + unsafe fn set_scissor_rect(&mut self, rect: &crate::Rect) { + self.cmd_buffer.commands.push(C::SetScissor(crate::Rect { + x: rect.x as i32, + y: rect.y as i32, + w: rect.w as i32, + h: rect.h as i32, + })); + } + unsafe fn set_stencil_reference(&mut self, value: u32) { + self.state.stencil.front.reference = value; + self.state.stencil.back.reference = value; + self.rebind_stencil_func(); + } + unsafe fn set_blend_constants(&mut self, color: &[f32; 4]) { + self.cmd_buffer.commands.push(C::SetBlendConstant(*color)); + } + + unsafe fn draw( + &mut self, + first_vertex: u32, + vertex_count: u32, + first_instance: u32, + instance_count: u32, + ) { + self.prepare_draw(first_instance); + #[allow(clippy::clone_on_copy)] // False positive when cloning glow::UniformLocation + self.cmd_buffer.commands.push(C::Draw { + topology: self.state.topology, + first_vertex, + vertex_count, + first_instance, + instance_count, + first_instance_location: self.state.first_instance_location.clone(), + }); + } + unsafe fn draw_indexed( + &mut self, + first_index: u32, + index_count: u32, + base_vertex: i32, + first_instance: u32, + instance_count: u32, + ) { + self.prepare_draw(first_instance); + let (index_size, index_type) = match self.state.index_format { + wgt::IndexFormat::Uint16 => (2, glow::UNSIGNED_SHORT), + wgt::IndexFormat::Uint32 => (4, glow::UNSIGNED_INT), + }; + let index_offset = self.state.index_offset + index_size * first_index as wgt::BufferAddress; + #[allow(clippy::clone_on_copy)] // False positive when cloning glow::UniformLocation + self.cmd_buffer.commands.push(C::DrawIndexed { + topology: self.state.topology, + index_type, + index_offset, + index_count, + base_vertex, + first_instance, + instance_count, + first_instance_location: self.state.first_instance_location.clone(), + }); + } + unsafe fn draw_mesh_tasks( + &mut self, + _group_count_x: u32, + _group_count_y: u32, + _group_count_z: u32, + ) { + unreachable!() + } + unsafe fn draw_indirect( + &mut self, + buffer: &super::Buffer, + offset: wgt::BufferAddress, + draw_count: u32, + ) { + self.prepare_draw(0); + for draw in 0..draw_count as wgt::BufferAddress { + let indirect_offset = + offset + draw * size_of::() as wgt::BufferAddress; + #[allow(clippy::clone_on_copy)] // False positive when cloning glow::UniformLocation + self.cmd_buffer.commands.push(C::DrawIndirect { + topology: self.state.topology, + indirect_buf: buffer.raw.unwrap(), + indirect_offset, + first_instance_location: self.state.first_instance_location.clone(), + }); + } + } + unsafe fn draw_indexed_indirect( + &mut self, + buffer: &super::Buffer, + offset: wgt::BufferAddress, + draw_count: u32, + ) { + self.prepare_draw(0); + let index_type = match self.state.index_format { + wgt::IndexFormat::Uint16 => glow::UNSIGNED_SHORT, + wgt::IndexFormat::Uint32 => glow::UNSIGNED_INT, + }; + for draw in 0..draw_count as wgt::BufferAddress { + let indirect_offset = + offset + draw * size_of::() as wgt::BufferAddress; + #[allow(clippy::clone_on_copy)] // False positive when cloning glow::UniformLocation + self.cmd_buffer.commands.push(C::DrawIndexedIndirect { + topology: self.state.topology, + index_type, + indirect_buf: buffer.raw.unwrap(), + indirect_offset, + first_instance_location: self.state.first_instance_location.clone(), + }); + } + } + unsafe fn draw_mesh_tasks_indirect( + &mut self, + _buffer: &::Buffer, + _offset: wgt::BufferAddress, + _draw_count: u32, + ) { + unreachable!() + } + unsafe fn draw_indirect_count( + &mut self, + _buffer: &super::Buffer, + _offset: wgt::BufferAddress, + _count_buffer: &super::Buffer, + _count_offset: wgt::BufferAddress, + _max_count: u32, + ) { + unreachable!() + } + unsafe fn draw_indexed_indirect_count( + &mut self, + _buffer: &super::Buffer, + _offset: wgt::BufferAddress, + _count_buffer: &super::Buffer, + _count_offset: wgt::BufferAddress, + _max_count: u32, + ) { + unreachable!() + } + unsafe fn draw_mesh_tasks_indirect_count( + &mut self, + _buffer: &::Buffer, + _offset: wgt::BufferAddress, + _count_buffer: &::Buffer, + _count_offset: wgt::BufferAddress, + _max_count: u32, + ) { + unreachable!() + } + + // compute + + unsafe fn begin_compute_pass(&mut self, desc: &crate::ComputePassDescriptor) { + debug_assert!(self.state.end_of_pass_timestamp.is_none()); + if let Some(ref t) = desc.timestamp_writes { + if let Some(index) = t.beginning_of_pass_write_index { + unsafe { self.write_timestamp(t.query_set, index) } + } + self.state.end_of_pass_timestamp = t + .end_of_pass_write_index + .map(|index| t.query_set.queries[index as usize]); + } + + if let Some(label) = desc.label { + let range = self.cmd_buffer.add_marker(label); + self.cmd_buffer.commands.push(C::PushDebugGroup(range)); + self.state.has_pass_label = true; + } + } + unsafe fn end_compute_pass(&mut self) { + if self.state.has_pass_label { + self.cmd_buffer.commands.push(C::PopDebugGroup); + self.state.has_pass_label = false; + } + + if let Some(query) = self.state.end_of_pass_timestamp.take() { + self.cmd_buffer.commands.push(C::TimestampQuery(query)); + } + } + + unsafe fn set_compute_pipeline(&mut self, pipeline: &super::ComputePipeline) { + self.set_pipeline_inner(&pipeline.inner); + } + + unsafe fn dispatch(&mut self, count: [u32; 3]) { + // Empty dispatches are invalid in OpenGL, but valid in WebGPU. + if count.contains(&0) { + return; + } + self.cmd_buffer.commands.push(C::Dispatch(count)); + } + unsafe fn dispatch_indirect(&mut self, buffer: &super::Buffer, offset: wgt::BufferAddress) { + self.cmd_buffer.commands.push(C::DispatchIndirect { + indirect_buf: buffer.raw.unwrap(), + indirect_offset: offset, + }); + } + + unsafe fn build_acceleration_structures<'a, T>( + &mut self, + _descriptor_count: u32, + _descriptors: T, + ) where + super::Api: 'a, + T: IntoIterator< + Item = crate::BuildAccelerationStructureDescriptor< + 'a, + super::Buffer, + super::AccelerationStructure, + >, + >, + { + unimplemented!() + } + + unsafe fn place_acceleration_structure_barrier( + &mut self, + _barriers: crate::AccelerationStructureBarrier, + ) { + unimplemented!() + } + + unsafe fn copy_acceleration_structure_to_acceleration_structure( + &mut self, + _src: &super::AccelerationStructure, + _dst: &super::AccelerationStructure, + _copy: wgt::AccelerationStructureCopy, + ) { + unimplemented!() + } + + unsafe fn read_acceleration_structure_compact_size( + &mut self, + _acceleration_structure: &super::AccelerationStructure, + _buf: &super::Buffer, + ) { + unimplemented!() + } + + unsafe fn set_acceleration_structure_dependencies( + _command_buffers: &[&super::CommandBuffer], + _dependencies: &[&super::AccelerationStructure], + ) { + unimplemented!() + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/conv.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/conv.rs new file mode 100644 index 00000000..5b54f8f1 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/conv.rs @@ -0,0 +1,430 @@ +impl super::AdapterShared { + pub(super) fn describe_texture_format( + &self, + texture_format: wgt::TextureFormat, + ) -> super::TextureFormatDesc { + use wgt::TextureFormat as Tf; + use wgt::{AstcBlock, AstcChannel}; + + let (internal, external, data_type) = match texture_format { + Tf::R8Unorm => (glow::R8, glow::RED, glow::UNSIGNED_BYTE), + Tf::R8Snorm => (glow::R8_SNORM, glow::RED, glow::BYTE), + Tf::R8Uint => (glow::R8UI, glow::RED_INTEGER, glow::UNSIGNED_BYTE), + Tf::R8Sint => (glow::R8I, glow::RED_INTEGER, glow::BYTE), + Tf::R16Uint => (glow::R16UI, glow::RED_INTEGER, glow::UNSIGNED_SHORT), + Tf::R16Sint => (glow::R16I, glow::RED_INTEGER, glow::SHORT), + Tf::R16Unorm => (glow::R16, glow::RED, glow::UNSIGNED_SHORT), + Tf::R16Snorm => (glow::R16_SNORM, glow::RED, glow::SHORT), + Tf::R16Float => (glow::R16F, glow::RED, glow::HALF_FLOAT), + Tf::Rg8Unorm => (glow::RG8, glow::RG, glow::UNSIGNED_BYTE), + Tf::Rg8Snorm => (glow::RG8_SNORM, glow::RG, glow::BYTE), + Tf::Rg8Uint => (glow::RG8UI, glow::RG_INTEGER, glow::UNSIGNED_BYTE), + Tf::Rg8Sint => (glow::RG8I, glow::RG_INTEGER, glow::BYTE), + Tf::R32Uint => (glow::R32UI, glow::RED_INTEGER, glow::UNSIGNED_INT), + Tf::R32Sint => (glow::R32I, glow::RED_INTEGER, glow::INT), + Tf::R32Float => (glow::R32F, glow::RED, glow::FLOAT), + Tf::Rg16Uint => (glow::RG16UI, glow::RG_INTEGER, glow::UNSIGNED_SHORT), + Tf::Rg16Sint => (glow::RG16I, glow::RG_INTEGER, glow::SHORT), + Tf::Rg16Unorm => (glow::RG16, glow::RG, glow::UNSIGNED_SHORT), + Tf::Rg16Snorm => (glow::RG16_SNORM, glow::RG, glow::SHORT), + Tf::Rg16Float => (glow::RG16F, glow::RG, glow::HALF_FLOAT), + Tf::Rgba8Unorm => (glow::RGBA8, glow::RGBA, glow::UNSIGNED_BYTE), + Tf::Rgba8UnormSrgb => (glow::SRGB8_ALPHA8, glow::RGBA, glow::UNSIGNED_BYTE), + Tf::Bgra8UnormSrgb => (glow::SRGB8_ALPHA8, glow::BGRA, glow::UNSIGNED_BYTE), //TODO? + Tf::Rgba8Snorm => (glow::RGBA8_SNORM, glow::RGBA, glow::BYTE), + Tf::Bgra8Unorm => (glow::RGBA8, glow::BGRA, glow::UNSIGNED_BYTE), //TODO? + Tf::Rgba8Uint => (glow::RGBA8UI, glow::RGBA_INTEGER, glow::UNSIGNED_BYTE), + Tf::Rgba8Sint => (glow::RGBA8I, glow::RGBA_INTEGER, glow::BYTE), + Tf::Rgb10a2Uint => ( + glow::RGB10_A2UI, + glow::RGBA_INTEGER, + glow::UNSIGNED_INT_2_10_10_10_REV, + ), + Tf::Rgb10a2Unorm => ( + glow::RGB10_A2, + glow::RGBA, + glow::UNSIGNED_INT_2_10_10_10_REV, + ), + Tf::Rg11b10Ufloat => ( + glow::R11F_G11F_B10F, + glow::RGB, + glow::UNSIGNED_INT_10F_11F_11F_REV, + ), + Tf::R64Uint => (glow::RG32UI, glow::RED_INTEGER, glow::UNSIGNED_INT), + Tf::Rg32Uint => (glow::RG32UI, glow::RG_INTEGER, glow::UNSIGNED_INT), + Tf::Rg32Sint => (glow::RG32I, glow::RG_INTEGER, glow::INT), + Tf::Rg32Float => (glow::RG32F, glow::RG, glow::FLOAT), + Tf::Rgba16Uint => (glow::RGBA16UI, glow::RGBA_INTEGER, glow::UNSIGNED_SHORT), + Tf::Rgba16Sint => (glow::RGBA16I, glow::RGBA_INTEGER, glow::SHORT), + Tf::Rgba16Unorm => (glow::RGBA16, glow::RGBA, glow::UNSIGNED_SHORT), + Tf::Rgba16Snorm => (glow::RGBA16_SNORM, glow::RGBA, glow::SHORT), + Tf::Rgba16Float => (glow::RGBA16F, glow::RGBA, glow::HALF_FLOAT), + Tf::Rgba32Uint => (glow::RGBA32UI, glow::RGBA_INTEGER, glow::UNSIGNED_INT), + Tf::Rgba32Sint => (glow::RGBA32I, glow::RGBA_INTEGER, glow::INT), + Tf::Rgba32Float => (glow::RGBA32F, glow::RGBA, glow::FLOAT), + Tf::Stencil8 => ( + glow::STENCIL_INDEX8, + glow::STENCIL_INDEX, + glow::UNSIGNED_BYTE, + ), + Tf::Depth16Unorm => ( + glow::DEPTH_COMPONENT16, + glow::DEPTH_COMPONENT, + glow::UNSIGNED_SHORT, + ), + Tf::Depth32Float => (glow::DEPTH_COMPONENT32F, glow::DEPTH_COMPONENT, glow::FLOAT), + Tf::Depth32FloatStencil8 => ( + glow::DEPTH32F_STENCIL8, + glow::DEPTH_STENCIL, + glow::FLOAT_32_UNSIGNED_INT_24_8_REV, + ), + Tf::Depth24Plus => ( + glow::DEPTH_COMPONENT24, + glow::DEPTH_COMPONENT, + glow::UNSIGNED_INT, + ), + Tf::Depth24PlusStencil8 => ( + glow::DEPTH24_STENCIL8, + glow::DEPTH_STENCIL, + glow::UNSIGNED_INT_24_8, + ), + Tf::NV12 => unreachable!(), + Tf::P010 => unreachable!(), + Tf::Rgb9e5Ufloat => (glow::RGB9_E5, glow::RGB, glow::UNSIGNED_INT_5_9_9_9_REV), + Tf::Bc1RgbaUnorm => (glow::COMPRESSED_RGBA_S3TC_DXT1_EXT, glow::RGBA, 0), + Tf::Bc1RgbaUnormSrgb => (glow::COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT, glow::RGBA, 0), + Tf::Bc2RgbaUnorm => (glow::COMPRESSED_RGBA_S3TC_DXT3_EXT, glow::RGBA, 0), + Tf::Bc2RgbaUnormSrgb => (glow::COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT, glow::RGBA, 0), + Tf::Bc3RgbaUnorm => (glow::COMPRESSED_RGBA_S3TC_DXT5_EXT, glow::RGBA, 0), + Tf::Bc3RgbaUnormSrgb => (glow::COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT, glow::RGBA, 0), + Tf::Bc4RUnorm => (glow::COMPRESSED_RED_RGTC1, glow::RED, 0), + Tf::Bc4RSnorm => (glow::COMPRESSED_SIGNED_RED_RGTC1, glow::RED, 0), + Tf::Bc5RgUnorm => (glow::COMPRESSED_RG_RGTC2, glow::RG, 0), + Tf::Bc5RgSnorm => (glow::COMPRESSED_SIGNED_RG_RGTC2, glow::RG, 0), + Tf::Bc6hRgbUfloat => (glow::COMPRESSED_RGB_BPTC_UNSIGNED_FLOAT, glow::RGB, 0), + Tf::Bc6hRgbFloat => (glow::COMPRESSED_RGB_BPTC_SIGNED_FLOAT, glow::RGB, 0), + Tf::Bc7RgbaUnorm => (glow::COMPRESSED_RGBA_BPTC_UNORM, glow::RGBA, 0), + Tf::Bc7RgbaUnormSrgb => (glow::COMPRESSED_SRGB_ALPHA_BPTC_UNORM, glow::RGBA, 0), + Tf::Etc2Rgb8Unorm => (glow::COMPRESSED_RGB8_ETC2, glow::RGB, 0), + Tf::Etc2Rgb8UnormSrgb => (glow::COMPRESSED_SRGB8_ETC2, glow::RGB, 0), + Tf::Etc2Rgb8A1Unorm => ( + glow::COMPRESSED_RGB8_PUNCHTHROUGH_ALPHA1_ETC2, + glow::RGBA, + 0, + ), + Tf::Etc2Rgb8A1UnormSrgb => ( + glow::COMPRESSED_SRGB8_PUNCHTHROUGH_ALPHA1_ETC2, + glow::RGBA, + 0, + ), + Tf::Etc2Rgba8Unorm => (glow::COMPRESSED_RGBA8_ETC2_EAC, glow::RGBA, 0), + Tf::Etc2Rgba8UnormSrgb => (glow::COMPRESSED_SRGB8_ALPHA8_ETC2_EAC, glow::RGBA, 0), + Tf::EacR11Unorm => (glow::COMPRESSED_R11_EAC, glow::RED, 0), + Tf::EacR11Snorm => (glow::COMPRESSED_SIGNED_R11_EAC, glow::RED, 0), + Tf::EacRg11Unorm => (glow::COMPRESSED_RG11_EAC, glow::RG, 0), + Tf::EacRg11Snorm => (glow::COMPRESSED_SIGNED_RG11_EAC, glow::RG, 0), + Tf::Astc { block, channel } => match channel { + AstcChannel::Unorm | AstcChannel::Hdr => match block { + AstcBlock::B4x4 => (glow::COMPRESSED_RGBA_ASTC_4x4_KHR, glow::RGBA, 0), + AstcBlock::B5x4 => (glow::COMPRESSED_RGBA_ASTC_5x4_KHR, glow::RGBA, 0), + AstcBlock::B5x5 => (glow::COMPRESSED_RGBA_ASTC_5x5_KHR, glow::RGBA, 0), + AstcBlock::B6x5 => (glow::COMPRESSED_RGBA_ASTC_6x5_KHR, glow::RGBA, 0), + AstcBlock::B6x6 => (glow::COMPRESSED_RGBA_ASTC_6x6_KHR, glow::RGBA, 0), + AstcBlock::B8x5 => (glow::COMPRESSED_RGBA_ASTC_8x5_KHR, glow::RGBA, 0), + AstcBlock::B8x6 => (glow::COMPRESSED_RGBA_ASTC_8x6_KHR, glow::RGBA, 0), + AstcBlock::B8x8 => (glow::COMPRESSED_RGBA_ASTC_8x8_KHR, glow::RGBA, 0), + AstcBlock::B10x5 => (glow::COMPRESSED_RGBA_ASTC_10x5_KHR, glow::RGBA, 0), + AstcBlock::B10x6 => (glow::COMPRESSED_RGBA_ASTC_10x6_KHR, glow::RGBA, 0), + AstcBlock::B10x8 => (glow::COMPRESSED_RGBA_ASTC_10x8_KHR, glow::RGBA, 0), + AstcBlock::B10x10 => (glow::COMPRESSED_RGBA_ASTC_10x10_KHR, glow::RGBA, 0), + AstcBlock::B12x10 => (glow::COMPRESSED_RGBA_ASTC_12x10_KHR, glow::RGBA, 0), + AstcBlock::B12x12 => (glow::COMPRESSED_RGBA_ASTC_12x12_KHR, glow::RGBA, 0), + }, + AstcChannel::UnormSrgb => match block { + AstcBlock::B4x4 => (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_4x4_KHR, glow::RGBA, 0), + AstcBlock::B5x4 => (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_5x4_KHR, glow::RGBA, 0), + AstcBlock::B5x5 => (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_5x5_KHR, glow::RGBA, 0), + AstcBlock::B6x5 => (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_6x5_KHR, glow::RGBA, 0), + AstcBlock::B6x6 => (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_6x6_KHR, glow::RGBA, 0), + AstcBlock::B8x5 => (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_8x5_KHR, glow::RGBA, 0), + AstcBlock::B8x6 => (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_8x6_KHR, glow::RGBA, 0), + AstcBlock::B8x8 => (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_8x8_KHR, glow::RGBA, 0), + AstcBlock::B10x5 => { + (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_10x5_KHR, glow::RGBA, 0) + } + AstcBlock::B10x6 => { + (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_10x6_KHR, glow::RGBA, 0) + } + AstcBlock::B10x8 => { + (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_10x8_KHR, glow::RGBA, 0) + } + AstcBlock::B10x10 => { + (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_10x10_KHR, glow::RGBA, 0) + } + AstcBlock::B12x10 => { + (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_12x10_KHR, glow::RGBA, 0) + } + AstcBlock::B12x12 => { + (glow::COMPRESSED_SRGB8_ALPHA8_ASTC_12x12_KHR, glow::RGBA, 0) + } + }, + }, + }; + + super::TextureFormatDesc { + internal, + external, + data_type, + } + } +} + +pub(super) fn describe_vertex_format(vertex_format: wgt::VertexFormat) -> super::VertexFormatDesc { + use super::VertexAttribKind as Vak; + use wgt::VertexFormat as Vf; + + let (element_count, element_format, attrib_kind) = match vertex_format { + Vf::Unorm8 => (1, glow::UNSIGNED_BYTE, Vak::Float), + Vf::Snorm8 => (1, glow::BYTE, Vak::Float), + Vf::Uint8 => (1, glow::UNSIGNED_BYTE, Vak::Integer), + Vf::Sint8 => (1, glow::BYTE, Vak::Integer), + Vf::Unorm8x2 => (2, glow::UNSIGNED_BYTE, Vak::Float), + Vf::Snorm8x2 => (2, glow::BYTE, Vak::Float), + Vf::Uint8x2 => (2, glow::UNSIGNED_BYTE, Vak::Integer), + Vf::Sint8x2 => (2, glow::BYTE, Vak::Integer), + Vf::Unorm8x4 => (4, glow::UNSIGNED_BYTE, Vak::Float), + Vf::Snorm8x4 => (4, glow::BYTE, Vak::Float), + Vf::Uint8x4 => (4, glow::UNSIGNED_BYTE, Vak::Integer), + Vf::Sint8x4 => (4, glow::BYTE, Vak::Integer), + Vf::Unorm16 => (1, glow::UNSIGNED_SHORT, Vak::Float), + Vf::Snorm16 => (1, glow::SHORT, Vak::Float), + Vf::Uint16 => (1, glow::UNSIGNED_SHORT, Vak::Integer), + Vf::Sint16 => (1, glow::SHORT, Vak::Integer), + Vf::Float16 => (1, glow::HALF_FLOAT, Vak::Float), + Vf::Unorm16x2 => (2, glow::UNSIGNED_SHORT, Vak::Float), + Vf::Snorm16x2 => (2, glow::SHORT, Vak::Float), + Vf::Uint16x2 => (2, glow::UNSIGNED_SHORT, Vak::Integer), + Vf::Sint16x2 => (2, glow::SHORT, Vak::Integer), + Vf::Float16x2 => (2, glow::HALF_FLOAT, Vak::Float), + Vf::Unorm16x4 => (4, glow::UNSIGNED_SHORT, Vak::Float), + Vf::Snorm16x4 => (4, glow::SHORT, Vak::Float), + Vf::Uint16x4 => (4, glow::UNSIGNED_SHORT, Vak::Integer), + Vf::Sint16x4 => (4, glow::SHORT, Vak::Integer), + Vf::Float16x4 => (4, glow::HALF_FLOAT, Vak::Float), + Vf::Uint32 => (1, glow::UNSIGNED_INT, Vak::Integer), + Vf::Sint32 => (1, glow::INT, Vak::Integer), + Vf::Float32 => (1, glow::FLOAT, Vak::Float), + Vf::Uint32x2 => (2, glow::UNSIGNED_INT, Vak::Integer), + Vf::Sint32x2 => (2, glow::INT, Vak::Integer), + Vf::Float32x2 => (2, glow::FLOAT, Vak::Float), + Vf::Uint32x3 => (3, glow::UNSIGNED_INT, Vak::Integer), + Vf::Sint32x3 => (3, glow::INT, Vak::Integer), + Vf::Float32x3 => (3, glow::FLOAT, Vak::Float), + Vf::Uint32x4 => (4, glow::UNSIGNED_INT, Vak::Integer), + Vf::Sint32x4 => (4, glow::INT, Vak::Integer), + Vf::Float32x4 => (4, glow::FLOAT, Vak::Float), + Vf::Unorm10_10_10_2 => (4, glow::UNSIGNED_INT_2_10_10_10_REV, Vak::Float), + Vf::Unorm8x4Bgra => (glow::BGRA as i32, glow::UNSIGNED_BYTE, Vak::Float), + Vf::Float64 | Vf::Float64x2 | Vf::Float64x3 | Vf::Float64x4 => unimplemented!(), + }; + + super::VertexFormatDesc { + element_count, + element_format, + attrib_kind, + } +} + +pub fn map_filter_modes( + min: wgt::FilterMode, + mag: wgt::FilterMode, + mip: wgt::MipmapFilterMode, +) -> (u32, u32) { + use wgt::FilterMode as Fm; + use wgt::MipmapFilterMode as Mfm; + + let mag_filter = match mag { + Fm::Nearest => glow::NEAREST, + Fm::Linear => glow::LINEAR, + }; + + let min_filter = match (min, mip) { + (Fm::Nearest, Mfm::Nearest) => glow::NEAREST_MIPMAP_NEAREST, + (Fm::Nearest, Mfm::Linear) => glow::NEAREST_MIPMAP_LINEAR, + (Fm::Linear, Mfm::Nearest) => glow::LINEAR_MIPMAP_NEAREST, + (Fm::Linear, Mfm::Linear) => glow::LINEAR_MIPMAP_LINEAR, + }; + + (min_filter, mag_filter) +} + +pub fn map_address_mode(mode: wgt::AddressMode) -> u32 { + match mode { + wgt::AddressMode::Repeat => glow::REPEAT, + wgt::AddressMode::MirrorRepeat => glow::MIRRORED_REPEAT, + wgt::AddressMode::ClampToEdge => glow::CLAMP_TO_EDGE, + wgt::AddressMode::ClampToBorder => glow::CLAMP_TO_BORDER, + //wgt::AddressMode::MirrorClamp => glow::MIRROR_CLAMP_TO_EDGE, + } +} + +pub fn map_compare_func(fun: wgt::CompareFunction) -> u32 { + use wgt::CompareFunction as Cf; + match fun { + Cf::Never => glow::NEVER, + Cf::Less => glow::LESS, + Cf::LessEqual => glow::LEQUAL, + Cf::Equal => glow::EQUAL, + Cf::GreaterEqual => glow::GEQUAL, + Cf::Greater => glow::GREATER, + Cf::NotEqual => glow::NOTEQUAL, + Cf::Always => glow::ALWAYS, + } +} + +pub fn map_primitive_topology(topology: wgt::PrimitiveTopology) -> u32 { + use wgt::PrimitiveTopology as Pt; + match topology { + Pt::PointList => glow::POINTS, + Pt::LineList => glow::LINES, + Pt::LineStrip => glow::LINE_STRIP, + Pt::TriangleList => glow::TRIANGLES, + Pt::TriangleStrip => glow::TRIANGLE_STRIP, + } +} + +pub(super) fn map_primitive_state(state: &wgt::PrimitiveState) -> super::PrimitiveState { + super::PrimitiveState { + //Note: we are flipping the front face, so that + // the Y-flip in the generated GLSL keeps the same visibility. + // See `naga::back::glsl::WriterFlags::ADJUST_COORDINATE_SPACE`. + front_face: match state.front_face { + wgt::FrontFace::Cw => glow::CCW, + wgt::FrontFace::Ccw => glow::CW, + }, + cull_face: match state.cull_mode { + Some(wgt::Face::Front) => glow::FRONT, + Some(wgt::Face::Back) => glow::BACK, + None => 0, + }, + unclipped_depth: state.unclipped_depth, + polygon_mode: match state.polygon_mode { + wgt::PolygonMode::Fill => glow::FILL, + wgt::PolygonMode::Line => glow::LINE, + wgt::PolygonMode::Point => glow::POINT, + }, + } +} + +pub fn _map_view_dimension(dim: wgt::TextureViewDimension) -> u32 { + use wgt::TextureViewDimension as Tvd; + match dim { + Tvd::D1 | Tvd::D2 => glow::TEXTURE_2D, + Tvd::D2Array => glow::TEXTURE_2D_ARRAY, + Tvd::Cube => glow::TEXTURE_CUBE_MAP, + Tvd::CubeArray => glow::TEXTURE_CUBE_MAP_ARRAY, + Tvd::D3 => glow::TEXTURE_3D, + } +} + +fn map_stencil_op(operation: wgt::StencilOperation) -> u32 { + use wgt::StencilOperation as So; + match operation { + So::Keep => glow::KEEP, + So::Zero => glow::ZERO, + So::Replace => glow::REPLACE, + So::Invert => glow::INVERT, + So::IncrementClamp => glow::INCR, + So::DecrementClamp => glow::DECR, + So::IncrementWrap => glow::INCR_WRAP, + So::DecrementWrap => glow::DECR_WRAP, + } +} + +fn map_stencil_ops(face: &wgt::StencilFaceState) -> super::StencilOps { + super::StencilOps { + pass: map_stencil_op(face.pass_op), + fail: map_stencil_op(face.fail_op), + depth_fail: map_stencil_op(face.depth_fail_op), + } +} + +pub(super) fn map_stencil(state: &wgt::StencilState) -> super::StencilState { + super::StencilState { + front: super::StencilSide { + function: map_compare_func(state.front.compare), + mask_read: state.read_mask, + mask_write: state.write_mask, + reference: 0, + ops: map_stencil_ops(&state.front), + }, + back: super::StencilSide { + function: map_compare_func(state.back.compare), + mask_read: state.read_mask, + mask_write: state.write_mask, + reference: 0, + ops: map_stencil_ops(&state.back), + }, + } +} + +fn map_blend_factor(factor: wgt::BlendFactor) -> u32 { + use wgt::BlendFactor as Bf; + match factor { + Bf::Zero => glow::ZERO, + Bf::One => glow::ONE, + Bf::Src => glow::SRC_COLOR, + Bf::OneMinusSrc => glow::ONE_MINUS_SRC_COLOR, + Bf::Dst => glow::DST_COLOR, + Bf::OneMinusDst => glow::ONE_MINUS_DST_COLOR, + Bf::SrcAlpha => glow::SRC_ALPHA, + Bf::OneMinusSrcAlpha => glow::ONE_MINUS_SRC_ALPHA, + Bf::DstAlpha => glow::DST_ALPHA, + Bf::OneMinusDstAlpha => glow::ONE_MINUS_DST_ALPHA, + Bf::Constant => glow::CONSTANT_COLOR, + Bf::OneMinusConstant => glow::ONE_MINUS_CONSTANT_COLOR, + Bf::SrcAlphaSaturated => glow::SRC_ALPHA_SATURATE, + Bf::Src1 => glow::SRC1_COLOR, + Bf::OneMinusSrc1 => glow::ONE_MINUS_SRC1_COLOR, + Bf::Src1Alpha => glow::SRC1_ALPHA, + Bf::OneMinusSrc1Alpha => glow::ONE_MINUS_SRC1_ALPHA, + } +} + +fn map_blend_component(component: &wgt::BlendComponent) -> super::BlendComponent { + super::BlendComponent { + src: map_blend_factor(component.src_factor), + dst: map_blend_factor(component.dst_factor), + equation: match component.operation { + wgt::BlendOperation::Add => glow::FUNC_ADD, + wgt::BlendOperation::Subtract => glow::FUNC_SUBTRACT, + wgt::BlendOperation::ReverseSubtract => glow::FUNC_REVERSE_SUBTRACT, + wgt::BlendOperation::Min => glow::MIN, + wgt::BlendOperation::Max => glow::MAX, + }, + } +} + +pub(super) fn map_blend(blend: &wgt::BlendState) -> super::BlendDesc { + super::BlendDesc { + color: map_blend_component(&blend.color), + alpha: map_blend_component(&blend.alpha), + } +} + +pub(super) fn map_storage_access(access: wgt::StorageTextureAccess) -> u32 { + match access { + wgt::StorageTextureAccess::ReadOnly => glow::READ_ONLY, + wgt::StorageTextureAccess::WriteOnly => glow::WRITE_ONLY, + wgt::StorageTextureAccess::ReadWrite => glow::READ_WRITE, + wgt::StorageTextureAccess::Atomic => glow::READ_WRITE, + } +} + +pub(super) fn is_layered_target(target: u32) -> bool { + match target { + glow::TEXTURE_2D | glow::TEXTURE_CUBE_MAP => false, + glow::TEXTURE_2D_ARRAY | glow::TEXTURE_CUBE_MAP_ARRAY | glow::TEXTURE_3D => true, + _ => unreachable!(), + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/device.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/device.rs new file mode 100644 index 00000000..c16a2ab9 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/device.rs @@ -0,0 +1,1677 @@ +use alloc::{ + borrow::ToOwned, format, string::String, string::ToString as _, sync::Arc, vec, vec::Vec, +}; +use core::{cmp::max, convert::TryInto, num::NonZeroU32, ptr, sync::atomic::Ordering}; + +use arrayvec::ArrayVec; +use glow::HasContext; +use naga::FastHashMap; + +use super::{conv, lock, MaybeMutex, PrivateCapabilities}; +use crate::auxil::map_naga_stage; +use crate::TlasInstance; + +type ShaderStage<'a> = ( + naga::ShaderStage, + &'a crate::ProgrammableStage<'a, super::ShaderModule>, +); +type NameBindingMap = FastHashMap; + +struct CompilationContext<'a> { + layout: &'a super::PipelineLayout, + sampler_map: &'a mut super::SamplerBindMap, + name_binding_map: &'a mut NameBindingMap, + immediates_items: &'a mut Vec, + multiview_mask: Option, + clip_distance_count: &'a mut u32, +} + +impl CompilationContext<'_> { + fn consume_reflection( + self, + gl: &glow::Context, + module: &naga::Module, + ep_info: &naga::valid::FunctionInfo, + reflection_info: naga::back::glsl::ReflectionInfo, + naga_stage: naga::ShaderStage, + program: glow::Program, + ) { + for (handle, var) in module.global_variables.iter() { + if ep_info[handle].is_empty() { + continue; + } + let register = match var.space { + naga::AddressSpace::Uniform => super::BindingRegister::UniformBuffers, + naga::AddressSpace::Storage { .. } => super::BindingRegister::StorageBuffers, + _ => continue, + }; + + let br = var.binding.as_ref().unwrap(); + let slot = self.layout.get_slot(br); + + let name = match reflection_info.uniforms.get(&handle) { + Some(name) => name.clone(), + None => continue, + }; + log::trace!( + "Rebind buffer: {:?} -> {}, register={:?}, slot={}", + var.name.as_ref(), + &name, + register, + slot + ); + self.name_binding_map.insert(name, (register, slot)); + } + + for (name, mapping) in reflection_info.texture_mapping { + let var = &module.global_variables[mapping.texture]; + let register = match module.types[var.ty].inner { + naga::TypeInner::Image { + class: naga::ImageClass::Storage { .. }, + .. + } => super::BindingRegister::Images, + _ => super::BindingRegister::Textures, + }; + + let tex_br = var.binding.as_ref().unwrap(); + let texture_linear_index = self.layout.get_slot(tex_br); + + self.name_binding_map + .insert(name, (register, texture_linear_index)); + if let Some(sampler_handle) = mapping.sampler { + let sam_br = module.global_variables[sampler_handle] + .binding + .as_ref() + .unwrap(); + let sampler_linear_index = self.layout.get_slot(sam_br); + self.sampler_map[texture_linear_index as usize] = Some(sampler_linear_index); + } + } + + for (name, location) in reflection_info.varying { + match naga_stage { + naga::ShaderStage::Vertex => { + assert_eq!(location.index, 0); + unsafe { gl.bind_attrib_location(program, location.location, &name) } + } + naga::ShaderStage::Fragment => { + assert_eq!(location.index, 0); + unsafe { gl.bind_frag_data_location(program, location.location, &name) } + } + naga::ShaderStage::Compute => {} + naga::ShaderStage::Task + | naga::ShaderStage::Mesh + | naga::ShaderStage::RayGeneration + | naga::ShaderStage::AnyHit + | naga::ShaderStage::ClosestHit + | naga::ShaderStage::Miss => unreachable!(), + } + } + + *self.immediates_items = reflection_info.immediates_items; + + if naga_stage == naga::ShaderStage::Vertex { + *self.clip_distance_count = reflection_info.clip_distance_count; + } + } +} + +impl super::Device { + /// # Safety + /// + /// - `name` must be created respecting `desc` + /// - `name` must be a texture + /// - If `drop_callback` is [`None`], wgpu-hal will take ownership of the texture. If + /// `drop_callback` is [`Some`], the texture must be valid until the callback is called. + #[cfg(any(native, Emscripten))] + pub unsafe fn texture_from_raw( + &self, + name: NonZeroU32, + desc: &crate::TextureDescriptor, + drop_callback: Option, + ) -> super::Texture { + super::Texture { + inner: super::TextureInner::Texture { + raw: glow::NativeTexture(name), + target: super::Texture::get_info_from_desc(desc), + }, + drop_guard: crate::DropGuard::from_option(drop_callback), + mip_level_count: desc.mip_level_count, + array_layer_count: desc.array_layer_count(), + format: desc.format, + format_desc: self.shared.describe_texture_format(desc.format), + copy_size: desc.copy_extent(), + } + } + + /// # Safety + /// + /// - `name` must be created respecting `desc` + /// - `name` must be a renderbuffer + /// - If `drop_callback` is [`None`], wgpu-hal will take ownership of the renderbuffer. If + /// `drop_callback` is [`Some`], the renderbuffer must be valid until the callback is called. + #[cfg(any(native, Emscripten))] + pub unsafe fn texture_from_raw_renderbuffer( + &self, + name: NonZeroU32, + desc: &crate::TextureDescriptor, + drop_callback: Option, + ) -> super::Texture { + super::Texture { + inner: super::TextureInner::Renderbuffer { + raw: glow::NativeRenderbuffer(name), + }, + drop_guard: crate::DropGuard::from_option(drop_callback), + mip_level_count: desc.mip_level_count, + array_layer_count: desc.array_layer_count(), + format: desc.format, + format_desc: self.shared.describe_texture_format(desc.format), + copy_size: desc.copy_extent(), + } + } + + unsafe fn compile_shader( + gl: &glow::Context, + shader: &str, + naga_stage: naga::ShaderStage, + #[cfg_attr(target_arch = "wasm32", allow(unused))] label: Option<&str>, + ) -> Result { + let target = match naga_stage { + naga::ShaderStage::Vertex => glow::VERTEX_SHADER, + naga::ShaderStage::Fragment => glow::FRAGMENT_SHADER, + naga::ShaderStage::Compute => glow::COMPUTE_SHADER, + naga::ShaderStage::Task + | naga::ShaderStage::Mesh + | naga::ShaderStage::RayGeneration + | naga::ShaderStage::AnyHit + | naga::ShaderStage::ClosestHit + | naga::ShaderStage::Miss => unreachable!(), + }; + + let raw = unsafe { gl.create_shader(target) }.unwrap(); + #[cfg(native)] + if gl.supports_debug() { + let name = raw.0.get(); + unsafe { gl.object_label(glow::SHADER, name, label) }; + } + + unsafe { gl.shader_source(raw, shader) }; + unsafe { gl.compile_shader(raw) }; + + log::debug!("\tCompiled shader {raw:?}"); + + let compiled_ok = unsafe { gl.get_shader_compile_status(raw) }; + let msg = unsafe { gl.get_shader_info_log(raw) }; + if compiled_ok { + if !msg.is_empty() { + log::debug!("\tCompile message: {msg}"); + } + Ok(raw) + } else { + log::error!("\tShader compilation failed: {msg}"); + unsafe { gl.delete_shader(raw) }; + Err(crate::PipelineError::Linkage( + map_naga_stage(naga_stage), + msg, + )) + } + } + + fn create_shader( + gl: &glow::Context, + naga_stage: naga::ShaderStage, + stage: &crate::ProgrammableStage, + context: CompilationContext, + program: glow::Program, + ) -> Result { + use naga::back::glsl; + let pipeline_options = glsl::PipelineOptions { + shader_stage: naga_stage, + entry_point: stage.entry_point.to_owned(), + multiview: context + .multiview_mask + .map(|a| NonZeroU32::new(a.get().count_ones()).unwrap()), + }; + + let (module, info) = naga::back::pipeline_constants::process_overrides( + &stage.module.source.module, + &stage.module.source.info, + Some((naga_stage, stage.entry_point)), + stage.constants, + ) + .map_err(|e| { + let msg = format!("{e}"); + crate::PipelineError::PipelineConstants(map_naga_stage(naga_stage), msg) + })?; + + let entry_point_index = module + .entry_points + .iter() + .position(|ep| ep.name.as_str() == stage.entry_point) + .ok_or(crate::PipelineError::EntryPoint(naga_stage))?; + + use naga::proc::BoundsCheckPolicy; + // The image bounds checks require the TEXTURE_LEVELS feature available in GL core 4.3+. + let version = gl.version(); + let image_check = if !version.is_embedded && (version.major, version.minor) >= (4, 3) { + BoundsCheckPolicy::ReadZeroSkipWrite + } else { + BoundsCheckPolicy::Unchecked + }; + + // Other bounds check are either provided by glsl or not implemented yet. + let policies = naga::proc::BoundsCheckPolicies { + index: BoundsCheckPolicy::Unchecked, + buffer: BoundsCheckPolicy::Unchecked, + image_load: image_check, + binding_array: BoundsCheckPolicy::Unchecked, + }; + + let mut output = String::new(); + let needs_temp_options = stage.zero_initialize_workgroup_memory + != context.layout.naga_options.zero_initialize_workgroup_memory; + let mut temp_options; + let naga_options = if needs_temp_options { + // We use a conditional here, as cloning the naga_options could be expensive + // That is, we want to avoid doing that unless we cannot avoid it + temp_options = context.layout.naga_options.clone(); + temp_options.zero_initialize_workgroup_memory = stage.zero_initialize_workgroup_memory; + &temp_options + } else { + &context.layout.naga_options + }; + let mut writer = glsl::Writer::new( + &mut output, + &module, + &info, + naga_options, + &pipeline_options, + policies, + ) + .map_err(|e| { + let msg = format!("{e}"); + crate::PipelineError::Linkage(map_naga_stage(naga_stage), msg) + })?; + + let reflection_info = writer.write().map_err(|e| { + let msg = format!("{e}"); + crate::PipelineError::Linkage(map_naga_stage(naga_stage), msg) + })?; + + log::debug!("Naga generated shader:\n{output}"); + + context.consume_reflection( + gl, + &module, + info.get_entry_point(entry_point_index), + reflection_info, + naga_stage, + program, + ); + + unsafe { Self::compile_shader(gl, &output, naga_stage, stage.module.label.as_deref()) } + } + + unsafe fn create_pipeline<'a>( + &self, + gl: &glow::Context, + shaders: ArrayVec, { crate::MAX_CONCURRENT_SHADER_STAGES }>, + layout: &super::PipelineLayout, + #[cfg_attr(target_arch = "wasm32", allow(unused))] label: Option<&str>, + multiview_mask: Option, + ) -> Result, crate::PipelineError> { + let mut program_stages = ArrayVec::new(); + let group_to_binding_to_slot = layout + .group_infos + .iter() + .map(|group| group.as_ref().map(|group| group.binding_to_slot.clone())) + .collect::>(); + for &(naga_stage, stage) in &shaders { + program_stages.push(super::ProgramStage { + naga_stage: naga_stage.to_owned(), + shader_id: stage.module.id, + entry_point: stage.entry_point.to_owned(), + zero_initialize_workgroup_memory: stage.zero_initialize_workgroup_memory, + constant_hash: Self::create_constant_hash(stage), + }); + } + let mut guard = self + .shared + .program_cache + .try_lock() + .expect("Couldn't acquire program_cache lock"); + // This guard ensures that we can't accidentally destroy a program whilst we're about to reuse it + // The only place that destroys a pipeline is also locking on `program_cache` + let program = guard + .entry(super::ProgramCacheKey { + stages: program_stages, + group_to_binding_to_slot: group_to_binding_to_slot.into_boxed_slice(), + }) + .or_insert_with(|| unsafe { + Self::create_program( + gl, + shaders, + layout, + label, + multiview_mask, + self.shared.shading_language_version, + self.shared.private_caps, + ) + }) + .to_owned()?; + drop(guard); + + Ok(program) + } + + fn create_constant_hash(stage: &crate::ProgrammableStage) -> Vec { + let mut buf: Vec = Vec::new(); + + for (key, value) in stage.constants.iter() { + buf.extend_from_slice(key.as_bytes()); + buf.extend_from_slice(&value.to_ne_bytes()); + } + + buf + } + + unsafe fn create_program<'a>( + gl: &glow::Context, + shaders: ArrayVec, { crate::MAX_CONCURRENT_SHADER_STAGES }>, + layout: &super::PipelineLayout, + #[cfg_attr(target_arch = "wasm32", allow(unused))] label: Option<&str>, + multiview_mask: Option, + glsl_version: naga::back::glsl::Version, + private_caps: PrivateCapabilities, + ) -> Result, crate::PipelineError> { + let glsl_version = match glsl_version { + naga::back::glsl::Version::Embedded { version, .. } => format!("{version} es"), + naga::back::glsl::Version::Desktop(version) => format!("{version}"), + }; + let program = unsafe { gl.create_program() }.unwrap(); + #[cfg(native)] + if let Some(label) = label { + if private_caps.contains(PrivateCapabilities::DEBUG_FNS) { + let name = program.0.get(); + unsafe { gl.object_label(glow::PROGRAM, name, Some(label)) }; + } + } + + let mut name_binding_map = NameBindingMap::default(); + let mut immediates_items = ArrayVec::<_, { crate::MAX_CONCURRENT_SHADER_STAGES }>::new(); + let mut sampler_map = [None; super::MAX_TEXTURE_SLOTS]; + let mut has_stages = wgt::ShaderStages::empty(); + let mut shaders_to_delete = ArrayVec::<_, { crate::MAX_CONCURRENT_SHADER_STAGES }>::new(); + let mut clip_distance_count = 0; + + for &(naga_stage, stage) in &shaders { + has_stages |= map_naga_stage(naga_stage); + let pc_item = { + immediates_items.push(Vec::new()); + immediates_items.last_mut().unwrap() + }; + let context = CompilationContext { + layout, + sampler_map: &mut sampler_map, + name_binding_map: &mut name_binding_map, + immediates_items: pc_item, + multiview_mask, + clip_distance_count: &mut clip_distance_count, + }; + + let shader = Self::create_shader(gl, naga_stage, stage, context, program)?; + shaders_to_delete.push(shader); + } + + // Create empty fragment shader if only vertex shader is present + if has_stages == wgt::ShaderStages::VERTEX { + let shader_src = format!("#version {glsl_version}\n void main(void) {{}}",); + log::debug!("Only vertex shader is present. Creating an empty fragment shader",); + let shader = unsafe { + Self::compile_shader( + gl, + &shader_src, + naga::ShaderStage::Fragment, + Some("(wgpu internal) dummy fragment shader"), + ) + }?; + shaders_to_delete.push(shader); + } + + for &shader in shaders_to_delete.iter() { + unsafe { gl.attach_shader(program, shader) }; + } + unsafe { gl.link_program(program) }; + + for shader in shaders_to_delete { + unsafe { gl.delete_shader(shader) }; + } + + log::debug!("\tLinked program {program:?}"); + + let linked_ok = unsafe { gl.get_program_link_status(program) }; + let msg = unsafe { gl.get_program_info_log(program) }; + if !linked_ok { + return Err(crate::PipelineError::Linkage(has_stages, msg)); + } + if !msg.is_empty() { + log::debug!("\tLink message: {msg}"); + } + + if !private_caps.contains(PrivateCapabilities::SHADER_BINDING_LAYOUT) { + // This remapping is only needed if we aren't able to put the binding layout + // in the shader. We can't remap storage buffers this way. + unsafe { gl.use_program(Some(program)) }; + for (ref name, (register, slot)) in name_binding_map { + log::trace!("Get binding {name:?} from program {program:?}"); + match register { + super::BindingRegister::UniformBuffers => { + let index = unsafe { gl.get_uniform_block_index(program, name) }.unwrap(); + log::trace!("\tBinding slot {slot} to block index {index}"); + unsafe { gl.uniform_block_binding(program, index, slot as _) }; + } + super::BindingRegister::StorageBuffers => { + let index = + unsafe { gl.get_shader_storage_block_index(program, name) }.unwrap(); + log::error!("Unable to re-map shader storage block {name} to {index}"); + return Err(crate::DeviceError::Lost.into()); + } + super::BindingRegister::Textures | super::BindingRegister::Images => { + let location = unsafe { gl.get_uniform_location(program, name) }; + unsafe { gl.uniform_1_i32(location.as_ref(), slot as _) }; + } + } + } + } + + let mut uniforms = ArrayVec::new(); + + for (stage_idx, stage_items) in immediates_items.into_iter().enumerate() { + for item in stage_items { + let naga_module = &shaders[stage_idx].1.module.source.module; + let type_inner = &naga_module.types[item.ty].inner; + + let location = unsafe { gl.get_uniform_location(program, &item.access_path) }; + + log::trace!( + "immediate data item: name={}, ty={:?}, offset={}, location={:?}", + item.access_path, + type_inner, + item.offset, + location, + ); + + if let Some(location) = location { + uniforms.push(super::ImmediateDesc { + location, + offset: item.offset, + size_bytes: type_inner.size(naga_module.to_ctx()), + ty: type_inner.clone(), + }); + } + } + } + + let first_instance_location = if has_stages.contains(wgt::ShaderStages::VERTEX) { + // If this returns none (the uniform isn't active), that's fine, we just won't set it. + unsafe { gl.get_uniform_location(program, naga::back::glsl::FIRST_INSTANCE_BINDING) } + } else { + None + }; + + Ok(Arc::new(super::PipelineInner { + program, + sampler_map, + first_instance_location, + immediates_descs: uniforms, + clip_distance_count, + })) + } +} + +impl crate::Device for super::Device { + type A = super::Api; + + unsafe fn create_buffer( + &self, + desc: &crate::BufferDescriptor, + ) -> Result { + let target = if desc.usage.contains(wgt::BufferUses::INDEX) { + glow::ELEMENT_ARRAY_BUFFER + } else { + glow::ARRAY_BUFFER + }; + + let emulate_map = self + .shared + .workarounds + .contains(super::Workarounds::EMULATE_BUFFER_MAP) + || !self + .shared + .private_caps + .contains(PrivateCapabilities::BUFFER_ALLOCATION); + + if emulate_map && desc.usage.intersects(wgt::BufferUses::MAP_WRITE) { + return Ok(super::Buffer { + raw: None, + target, + size: desc.size, + map_flags: 0, + data: Some(Arc::new(MaybeMutex::new(vec![0; desc.size as usize]))), + offset_of_current_mapping: Arc::new(MaybeMutex::new(0)), + }); + } + + let gl = &self.shared.context.lock(); + + let target = if desc.usage.contains(wgt::BufferUses::INDEX) { + glow::ELEMENT_ARRAY_BUFFER + } else { + glow::ARRAY_BUFFER + }; + + let is_host_visible = desc + .usage + .intersects(wgt::BufferUses::MAP_READ | wgt::BufferUses::MAP_WRITE); + let is_coherent = desc + .memory_flags + .contains(crate::MemoryFlags::PREFER_COHERENT); + + let mut map_flags = 0; + if desc.usage.contains(wgt::BufferUses::MAP_READ) { + map_flags |= glow::MAP_READ_BIT; + } + if desc.usage.contains(wgt::BufferUses::MAP_WRITE) { + map_flags |= glow::MAP_WRITE_BIT; + } + + let raw = Some(unsafe { gl.create_buffer() }.map_err(|_| crate::DeviceError::OutOfMemory)?); + unsafe { gl.bind_buffer(target, raw) }; + let raw_size = desc + .size + .try_into() + .map_err(|_| crate::DeviceError::OutOfMemory)?; + + if self + .shared + .private_caps + .contains(PrivateCapabilities::BUFFER_ALLOCATION) + { + if is_host_visible { + map_flags |= glow::MAP_PERSISTENT_BIT; + if is_coherent { + map_flags |= glow::MAP_COHERENT_BIT; + } + } + // TODO: may also be required for other calls involving `buffer_sub_data_u8_slice` (e.g. copy buffer to buffer and clear buffer) + if desc.usage.intersects(wgt::BufferUses::QUERY_RESOLVE) { + map_flags |= glow::DYNAMIC_STORAGE_BIT; + } + unsafe { gl.buffer_storage(target, raw_size, None, map_flags) }; + } else { + assert!(!is_coherent); + let usage = if is_host_visible { + if desc.usage.contains(wgt::BufferUses::MAP_READ) { + glow::STREAM_READ + } else { + glow::DYNAMIC_DRAW + } + } else { + // Even if the usage doesn't contain SRC_READ, we update it internally at least once + // Some vendors take usage very literally and STATIC_DRAW will freeze us with an empty buffer + // https://github.com/gfx-rs/wgpu/issues/3371 + glow::DYNAMIC_DRAW + }; + unsafe { gl.buffer_data_size(target, raw_size, usage) }; + } + + unsafe { gl.bind_buffer(target, None) }; + + if !is_coherent && desc.usage.contains(wgt::BufferUses::MAP_WRITE) { + map_flags |= glow::MAP_FLUSH_EXPLICIT_BIT; + } + //TODO: do we need `glow::MAP_UNSYNCHRONIZED_BIT`? + + #[cfg(native)] + if let Some(label) = desc.label { + if self + .shared + .private_caps + .contains(PrivateCapabilities::DEBUG_FNS) + { + let name = raw.map_or(0, |buf| buf.0.get()); + unsafe { gl.object_label(glow::BUFFER, name, Some(label)) }; + } + } + + let data = if emulate_map && desc.usage.contains(wgt::BufferUses::MAP_READ) { + Some(Arc::new(MaybeMutex::new(vec![0; desc.size as usize]))) + } else { + None + }; + + self.counters.buffers.add(1); + + Ok(super::Buffer { + raw, + target, + size: desc.size, + map_flags, + data, + offset_of_current_mapping: Arc::new(MaybeMutex::new(0)), + }) + } + + unsafe fn destroy_buffer(&self, buffer: super::Buffer) { + if let Some(raw) = buffer.raw { + let gl = &self.shared.context.lock(); + unsafe { gl.delete_buffer(raw) }; + } + + self.counters.buffers.sub(1); + } + + unsafe fn add_raw_buffer(&self, _buffer: &super::Buffer) { + self.counters.buffers.add(1); + } + + unsafe fn map_buffer( + &self, + buffer: &super::Buffer, + range: crate::MemoryRange, + ) -> Result { + let is_coherent = buffer.map_flags & glow::MAP_COHERENT_BIT != 0; + let ptr = match buffer.raw { + None => { + let mut vec = lock(buffer.data.as_ref().unwrap()); + let slice = &mut vec.as_mut_slice()[range.start as usize..range.end as usize]; + slice.as_mut_ptr() + } + Some(raw) => { + let gl = &self.shared.context.lock(); + unsafe { gl.bind_buffer(buffer.target, Some(raw)) }; + let ptr = if let Some(ref map_read_allocation) = buffer.data { + let mut guard = lock(map_read_allocation); + let slice = guard.as_mut_slice(); + unsafe { self.shared.get_buffer_sub_data(gl, buffer.target, 0, slice) }; + slice.as_mut_ptr() + } else { + *lock(&buffer.offset_of_current_mapping) = range.start; + unsafe { + gl.map_buffer_range( + buffer.target, + range.start as i32, + (range.end - range.start) as i32, + buffer.map_flags, + ) + } + }; + unsafe { gl.bind_buffer(buffer.target, None) }; + ptr + } + }; + Ok(crate::BufferMapping { + ptr: ptr::NonNull::new(ptr).ok_or(crate::DeviceError::Lost)?, + is_coherent, + }) + } + unsafe fn unmap_buffer(&self, buffer: &super::Buffer) { + if let Some(raw) = buffer.raw { + if buffer.data.is_none() { + let gl = &self.shared.context.lock(); + unsafe { gl.bind_buffer(buffer.target, Some(raw)) }; + unsafe { gl.unmap_buffer(buffer.target) }; + unsafe { gl.bind_buffer(buffer.target, None) }; + *lock(&buffer.offset_of_current_mapping) = 0; + } + } + } + unsafe fn flush_mapped_ranges(&self, buffer: &super::Buffer, ranges: I) + where + I: Iterator, + { + if let Some(raw) = buffer.raw { + if buffer.data.is_none() { + let gl = &self.shared.context.lock(); + unsafe { gl.bind_buffer(buffer.target, Some(raw)) }; + for range in ranges { + let offset_of_current_mapping = *lock(&buffer.offset_of_current_mapping); + unsafe { + gl.flush_mapped_buffer_range( + buffer.target, + (range.start - offset_of_current_mapping) as i32, + (range.end - range.start) as i32, + ) + }; + } + } + } + } + unsafe fn invalidate_mapped_ranges(&self, _buffer: &super::Buffer, _ranges: I) { + //TODO: do we need to do anything? + } + + unsafe fn create_texture( + &self, + desc: &crate::TextureDescriptor, + ) -> Result { + let gl = &self.shared.context.lock(); + + let render_usage = wgt::TextureUses::COLOR_TARGET + | wgt::TextureUses::DEPTH_STENCIL_WRITE + | wgt::TextureUses::DEPTH_STENCIL_READ + | wgt::TextureUses::TRANSIENT; + let format_desc = self.shared.describe_texture_format(desc.format); + + let inner = if render_usage.contains(desc.usage) + && desc.dimension == wgt::TextureDimension::D2 + && desc.size.depth_or_array_layers == 1 + { + let raw = unsafe { gl.create_renderbuffer().unwrap() }; + unsafe { gl.bind_renderbuffer(glow::RENDERBUFFER, Some(raw)) }; + if desc.sample_count > 1 { + unsafe { + gl.renderbuffer_storage_multisample( + glow::RENDERBUFFER, + desc.sample_count as i32, + format_desc.internal, + desc.size.width as i32, + desc.size.height as i32, + ) + }; + } else { + unsafe { + gl.renderbuffer_storage( + glow::RENDERBUFFER, + format_desc.internal, + desc.size.width as i32, + desc.size.height as i32, + ) + }; + } + + #[cfg(native)] + if let Some(label) = desc.label { + if self + .shared + .private_caps + .contains(PrivateCapabilities::DEBUG_FNS) + { + let name = raw.0.get(); + unsafe { gl.object_label(glow::RENDERBUFFER, name, Some(label)) }; + } + } + + unsafe { gl.bind_renderbuffer(glow::RENDERBUFFER, None) }; + super::TextureInner::Renderbuffer { raw } + } else { + let raw = unsafe { gl.create_texture().unwrap() }; + let target = super::Texture::get_info_from_desc(desc); + + unsafe { gl.bind_texture(target, Some(raw)) }; + //Note: this has to be done before defining the storage! + match desc.format.sample_type(None, Some(self.shared.features)) { + Some( + wgt::TextureSampleType::Float { filterable: false } + | wgt::TextureSampleType::Uint + | wgt::TextureSampleType::Sint, + ) => { + // reset default filtering mode + unsafe { + gl.tex_parameter_i32(target, glow::TEXTURE_MIN_FILTER, glow::NEAREST as i32) + }; + unsafe { + gl.tex_parameter_i32(target, glow::TEXTURE_MAG_FILTER, glow::NEAREST as i32) + }; + } + _ => {} + } + + if conv::is_layered_target(target) { + unsafe { + if self + .shared + .private_caps + .contains(PrivateCapabilities::TEXTURE_STORAGE) + { + gl.tex_storage_3d( + target, + desc.mip_level_count as i32, + format_desc.internal, + desc.size.width as i32, + desc.size.height as i32, + desc.size.depth_or_array_layers as i32, + ) + } else if target == glow::TEXTURE_3D { + let mut width = desc.size.width; + let mut height = desc.size.height; + let mut depth = desc.size.depth_or_array_layers; + for i in 0..desc.mip_level_count { + gl.tex_image_3d( + target, + i as i32, + format_desc.internal as i32, + width as i32, + height as i32, + depth as i32, + 0, + format_desc.external, + format_desc.data_type, + glow::PixelUnpackData::Slice(None), + ); + width = max(1, width / 2); + height = max(1, height / 2); + depth = max(1, depth / 2); + } + } else { + let mut width = desc.size.width; + let mut height = desc.size.height; + for i in 0..desc.mip_level_count { + gl.tex_image_3d( + target, + i as i32, + format_desc.internal as i32, + width as i32, + height as i32, + desc.size.depth_or_array_layers as i32, + 0, + format_desc.external, + format_desc.data_type, + glow::PixelUnpackData::Slice(None), + ); + width = max(1, width / 2); + height = max(1, height / 2); + } + } + }; + } else if desc.sample_count > 1 { + unsafe { + gl.tex_storage_2d_multisample( + target, + desc.sample_count as i32, + format_desc.internal, + desc.size.width as i32, + desc.size.height as i32, + true, + ) + }; + } else { + unsafe { + if self + .shared + .private_caps + .contains(PrivateCapabilities::TEXTURE_STORAGE) + { + gl.tex_storage_2d( + target, + desc.mip_level_count as i32, + format_desc.internal, + desc.size.width as i32, + desc.size.height as i32, + ) + } else if target == glow::TEXTURE_CUBE_MAP { + let mut width = desc.size.width; + let mut height = desc.size.height; + for i in 0..desc.mip_level_count { + for face in [ + glow::TEXTURE_CUBE_MAP_POSITIVE_X, + glow::TEXTURE_CUBE_MAP_NEGATIVE_X, + glow::TEXTURE_CUBE_MAP_POSITIVE_Y, + glow::TEXTURE_CUBE_MAP_NEGATIVE_Y, + glow::TEXTURE_CUBE_MAP_POSITIVE_Z, + glow::TEXTURE_CUBE_MAP_NEGATIVE_Z, + ] { + gl.tex_image_2d( + face, + i as i32, + format_desc.internal as i32, + width as i32, + height as i32, + 0, + format_desc.external, + format_desc.data_type, + glow::PixelUnpackData::Slice(None), + ); + } + width = max(1, width / 2); + height = max(1, height / 2); + } + } else { + let mut width = desc.size.width; + let mut height = desc.size.height; + for i in 0..desc.mip_level_count { + gl.tex_image_2d( + target, + i as i32, + format_desc.internal as i32, + width as i32, + height as i32, + 0, + format_desc.external, + format_desc.data_type, + glow::PixelUnpackData::Slice(None), + ); + width = max(1, width / 2); + height = max(1, height / 2); + } + } + }; + } + + #[cfg(native)] + if let Some(label) = desc.label { + if self + .shared + .private_caps + .contains(PrivateCapabilities::DEBUG_FNS) + { + let name = raw.0.get(); + unsafe { gl.object_label(glow::TEXTURE, name, Some(label)) }; + } + } + + unsafe { gl.bind_texture(target, None) }; + super::TextureInner::Texture { raw, target } + }; + + self.counters.textures.add(1); + + Ok(super::Texture { + inner, + drop_guard: None, + mip_level_count: desc.mip_level_count, + array_layer_count: desc.array_layer_count(), + format: desc.format, + format_desc, + copy_size: desc.copy_extent(), + }) + } + + unsafe fn destroy_texture(&self, texture: super::Texture) { + if texture.drop_guard.is_none() { + let gl = &self.shared.context.lock(); + match texture.inner { + super::TextureInner::Renderbuffer { raw, .. } => { + unsafe { gl.delete_renderbuffer(raw) }; + } + super::TextureInner::DefaultRenderbuffer => {} + super::TextureInner::Texture { raw, .. } => { + unsafe { gl.delete_texture(raw) }; + } + #[cfg(webgl)] + super::TextureInner::ExternalFramebuffer { .. } => {} + #[cfg(native)] + super::TextureInner::ExternalNativeFramebuffer { .. } => {} + } + } + + // For clarity, we explicitly drop the drop guard. Although this has no real semantic effect as the + // end of the scope will drop the drop guard since this function takes ownership of the texture. + drop(texture.drop_guard); + + self.counters.textures.sub(1); + } + + unsafe fn add_raw_texture(&self, _texture: &super::Texture) { + self.counters.textures.add(1); + } + + unsafe fn create_texture_view( + &self, + texture: &super::Texture, + desc: &crate::TextureViewDescriptor, + ) -> Result { + self.counters.texture_views.add(1); + Ok(super::TextureView { + //TODO: use `conv::map_view_dimension(desc.dimension)`? + inner: texture.inner.clone(), + aspects: crate::FormatAspects::new(texture.format, desc.range.aspect), + mip_levels: desc.range.mip_range(texture.mip_level_count), + array_layers: desc.range.layer_range(texture.array_layer_count), + format: texture.format, + }) + } + + unsafe fn destroy_texture_view(&self, _view: super::TextureView) { + self.counters.texture_views.sub(1); + } + + unsafe fn create_sampler( + &self, + desc: &crate::SamplerDescriptor, + ) -> Result { + let gl = &self.shared.context.lock(); + + let raw = unsafe { gl.create_sampler().unwrap() }; + + let (min, mag) = + conv::map_filter_modes(desc.min_filter, desc.mag_filter, desc.mipmap_filter); + + unsafe { gl.sampler_parameter_i32(raw, glow::TEXTURE_MIN_FILTER, min as i32) }; + unsafe { gl.sampler_parameter_i32(raw, glow::TEXTURE_MAG_FILTER, mag as i32) }; + + unsafe { + gl.sampler_parameter_i32( + raw, + glow::TEXTURE_WRAP_S, + conv::map_address_mode(desc.address_modes[0]) as i32, + ) + }; + unsafe { + gl.sampler_parameter_i32( + raw, + glow::TEXTURE_WRAP_T, + conv::map_address_mode(desc.address_modes[1]) as i32, + ) + }; + unsafe { + gl.sampler_parameter_i32( + raw, + glow::TEXTURE_WRAP_R, + conv::map_address_mode(desc.address_modes[2]) as i32, + ) + }; + + if let Some(border_color) = desc.border_color { + let border = match border_color { + wgt::SamplerBorderColor::TransparentBlack | wgt::SamplerBorderColor::Zero => { + [0.0; 4] + } + wgt::SamplerBorderColor::OpaqueBlack => [0.0, 0.0, 0.0, 1.0], + wgt::SamplerBorderColor::OpaqueWhite => [1.0; 4], + }; + unsafe { gl.sampler_parameter_f32_slice(raw, glow::TEXTURE_BORDER_COLOR, &border) }; + } + + unsafe { gl.sampler_parameter_f32(raw, glow::TEXTURE_MIN_LOD, desc.lod_clamp.start) }; + unsafe { gl.sampler_parameter_f32(raw, glow::TEXTURE_MAX_LOD, desc.lod_clamp.end) }; + + // If clamp is not 1, we know anisotropy is supported up to 16x + if desc.anisotropy_clamp != 1 { + unsafe { + gl.sampler_parameter_i32( + raw, + glow::TEXTURE_MAX_ANISOTROPY, + desc.anisotropy_clamp as i32, + ) + }; + } + + //set_param_float(glow::TEXTURE_LOD_BIAS, info.lod_bias.0); + + if let Some(compare) = desc.compare { + unsafe { + gl.sampler_parameter_i32( + raw, + glow::TEXTURE_COMPARE_MODE, + glow::COMPARE_REF_TO_TEXTURE as i32, + ) + }; + unsafe { + gl.sampler_parameter_i32( + raw, + glow::TEXTURE_COMPARE_FUNC, + conv::map_compare_func(compare) as i32, + ) + }; + } + + #[cfg(native)] + if let Some(label) = desc.label { + if self + .shared + .private_caps + .contains(PrivateCapabilities::DEBUG_FNS) + { + let name = raw.0.get(); + unsafe { gl.object_label(glow::SAMPLER, name, Some(label)) }; + } + } + + self.counters.samplers.add(1); + + Ok(super::Sampler { raw }) + } + + unsafe fn destroy_sampler(&self, sampler: super::Sampler) { + let gl = &self.shared.context.lock(); + unsafe { gl.delete_sampler(sampler.raw) }; + self.counters.samplers.sub(1); + } + + unsafe fn create_command_encoder( + &self, + _desc: &crate::CommandEncoderDescriptor, + ) -> Result { + self.counters.command_encoders.add(1); + + Ok(super::CommandEncoder { + cmd_buffer: super::CommandBuffer::default(), + state: Default::default(), + private_caps: self.shared.private_caps, + counters: Arc::clone(&self.counters), + }) + } + + unsafe fn create_bind_group_layout( + &self, + desc: &crate::BindGroupLayoutDescriptor, + ) -> Result { + self.counters.bind_group_layouts.add(1); + Ok(super::BindGroupLayout { + entries: Arc::from(desc.entries), + }) + } + + unsafe fn destroy_bind_group_layout(&self, _bg_layout: super::BindGroupLayout) { + self.counters.bind_group_layouts.sub(1); + } + + unsafe fn create_pipeline_layout( + &self, + desc: &crate::PipelineLayoutDescriptor, + ) -> Result { + use naga::back::glsl; + + let mut group_infos = Vec::with_capacity(desc.bind_group_layouts.len()); + let mut num_samplers = 0u8; + let mut num_textures = 0u8; + let mut num_images = 0u8; + let mut num_uniform_buffers = 0u8; + let mut num_storage_buffers = 0u8; + + let mut writer_flags = glsl::WriterFlags::ADJUST_COORDINATE_SPACE; + writer_flags.set( + glsl::WriterFlags::TEXTURE_SHADOW_LOD, + self.shared + .private_caps + .contains(PrivateCapabilities::SHADER_TEXTURE_SHADOW_LOD), + ); + writer_flags.set( + glsl::WriterFlags::DRAW_PARAMETERS, + self.shared + .private_caps + .contains(PrivateCapabilities::FULLY_FEATURED_INSTANCING), + ); + // We always force point size to be written and it will be ignored by the driver if it's not a point list primitive. + // https://github.com/gfx-rs/wgpu/pull/3440/files#r1095726950 + writer_flags.set(glsl::WriterFlags::FORCE_POINT_SIZE, true); + let mut binding_map = glsl::BindingMap::default(); + + for (group_index, bg_layout) in desc.bind_group_layouts.iter().enumerate() { + let Some(bg_layout) = bg_layout else { + group_infos.push(None); + continue; + }; + + // create a vector with the size enough to hold all the bindings, filled with `!0` + let mut binding_to_slot = vec![ + !0; + bg_layout + .entries + .iter() + .map(|b| b.binding) + .max() + .map_or(0, |idx| idx as usize + 1) + ] + .into_boxed_slice(); + + for entry in bg_layout.entries.iter() { + let counter = match entry.ty { + wgt::BindingType::Sampler { .. } => &mut num_samplers, + wgt::BindingType::Texture { .. } => &mut num_textures, + wgt::BindingType::StorageTexture { .. } => &mut num_images, + wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Uniform, + .. + } => &mut num_uniform_buffers, + wgt::BindingType::Buffer { + ty: wgt::BufferBindingType::Storage { .. }, + .. + } => &mut num_storage_buffers, + wgt::BindingType::AccelerationStructure { .. } => unimplemented!(), + wgt::BindingType::ExternalTexture => unimplemented!(), + }; + + binding_to_slot[entry.binding as usize] = *counter; + let br = naga::ResourceBinding { + group: group_index as u32, + binding: entry.binding, + }; + binding_map.insert(br, *counter); + *counter += entry.count.map_or(1, |c| c.get() as u8); + } + + group_infos.push(Some(super::BindGroupLayoutInfo { + entries: Arc::clone(&bg_layout.entries), + binding_to_slot, + })); + } + + self.counters.pipeline_layouts.add(1); + + Ok(super::PipelineLayout { + group_infos: group_infos.into_boxed_slice(), + naga_options: glsl::Options { + version: self.shared.shading_language_version, + writer_flags, + binding_map, + zero_initialize_workgroup_memory: true, + }, + }) + } + + unsafe fn destroy_pipeline_layout(&self, _pipeline_layout: super::PipelineLayout) { + self.counters.pipeline_layouts.sub(1); + } + + unsafe fn create_bind_group( + &self, + desc: &crate::BindGroupDescriptor< + super::BindGroupLayout, + super::Buffer, + super::Sampler, + super::TextureView, + super::AccelerationStructure, + >, + ) -> Result { + let mut contents = Vec::new(); + + let layout_and_entry_iter = desc.entries.iter().map(|entry| { + let layout = desc + .layout + .entries + .iter() + .find(|layout_entry| layout_entry.binding == entry.binding) + .expect("internal error: no layout entry found with binding slot"); + (entry, layout) + }); + for (entry, layout) in layout_and_entry_iter { + let binding = match layout.ty { + wgt::BindingType::Buffer { .. } => { + let bb = &desc.buffers[entry.resource_index as usize]; + super::RawBinding::Buffer { + raw: bb.buffer.raw.unwrap(), + offset: bb.offset as i32, + size: match bb.size { + Some(s) => s.get() as i32, + None => (bb.buffer.size - bb.offset) as i32, + }, + } + } + wgt::BindingType::Sampler { .. } => { + let sampler = desc.samplers[entry.resource_index as usize]; + super::RawBinding::Sampler(sampler.raw) + } + wgt::BindingType::Texture { view_dimension, .. } => { + let view = desc.textures[entry.resource_index as usize].view; + if view.array_layers.start != 0 { + log::error!("Unable to create a sampled texture binding for non-zero array layer.\n{}", + "This is an implementation problem of wgpu-hal/gles backend.") + } + let (raw, target) = view.inner.as_native(); + + super::Texture::log_failing_target_heuristics(view_dimension, target); + + super::RawBinding::Texture { + raw, + target, + aspects: view.aspects, + mip_levels: view.mip_levels.clone(), + } + } + wgt::BindingType::StorageTexture { + access, + format, + view_dimension, + } => { + let view = desc.textures[entry.resource_index as usize].view; + let format_desc = self.shared.describe_texture_format(format); + let (raw, _target) = view.inner.as_native(); + super::RawBinding::Image(super::ImageBinding { + raw, + mip_level: view.mip_levels.start, + array_layer: match view_dimension { + wgt::TextureViewDimension::D2Array + | wgt::TextureViewDimension::CubeArray => None, + _ => Some(view.array_layers.start), + }, + access: conv::map_storage_access(access), + format: format_desc.internal, + }) + } + wgt::BindingType::AccelerationStructure { .. } => unimplemented!(), + wgt::BindingType::ExternalTexture => unimplemented!(), + }; + contents.push(binding); + } + + self.counters.bind_groups.add(1); + + Ok(super::BindGroup { + contents: contents.into_boxed_slice(), + }) + } + + unsafe fn destroy_bind_group(&self, _group: super::BindGroup) { + self.counters.bind_groups.sub(1); + } + + unsafe fn create_shader_module( + &self, + desc: &crate::ShaderModuleDescriptor, + shader: crate::ShaderInput, + ) -> Result { + self.counters.shader_modules.add(1); + + Ok(super::ShaderModule { + source: match shader { + crate::ShaderInput::Naga(naga) => naga, + // The backend doesn't yet expose this feature so it should be fine + crate::ShaderInput::Glsl { .. } => unimplemented!(), + crate::ShaderInput::SpirV(_) + | crate::ShaderInput::MetalLib { .. } + | crate::ShaderInput::Msl { .. } + | crate::ShaderInput::Dxil { .. } + | crate::ShaderInput::Hlsl { .. } => { + unreachable!() + } + }, + label: desc.label.map(|str| str.to_string()), + id: self.shared.next_shader_id.fetch_add(1, Ordering::Relaxed), + }) + } + + unsafe fn destroy_shader_module(&self, _module: super::ShaderModule) { + self.counters.shader_modules.sub(1); + } + + unsafe fn create_render_pipeline( + &self, + desc: &crate::RenderPipelineDescriptor< + super::PipelineLayout, + super::ShaderModule, + super::PipelineCache, + >, + ) -> Result { + let (vertex_stage, vertex_buffers) = match &desc.vertex_processor { + crate::VertexProcessor::Standard { + vertex_buffers, + ref vertex_stage, + } => (vertex_stage, vertex_buffers), + crate::VertexProcessor::Mesh { .. } => unreachable!(), + }; + let gl = &self.shared.context.lock(); + let mut shaders = ArrayVec::new(); + shaders.push((naga::ShaderStage::Vertex, vertex_stage)); + if let Some(ref fs) = desc.fragment_stage { + shaders.push((naga::ShaderStage::Fragment, fs)); + } + let inner = unsafe { + self.create_pipeline(gl, shaders, desc.layout, desc.label, desc.multiview_mask) + }?; + + let (vertex_buffers, vertex_attributes) = { + let mut buffers = Vec::new(); + let mut attributes = Vec::new(); + for (index, vb_layout) in vertex_buffers.iter().enumerate() { + buffers.push(super::VertexBufferDesc { + step: vb_layout.step_mode, + stride: vb_layout.array_stride as u32, + }); + for vat in vb_layout.attributes.iter() { + let format_desc = conv::describe_vertex_format(vat.format); + attributes.push(super::AttributeDesc { + location: vat.shader_location, + offset: vat.offset as u32, + buffer_index: index as u32, + format_desc, + }); + } + } + (buffers.into_boxed_slice(), attributes.into_boxed_slice()) + }; + + let color_targets = { + let mut targets = Vec::new(); + for ct in desc.color_targets.iter().filter_map(|at| at.as_ref()) { + targets.push(super::ColorTargetDesc { + mask: ct.write_mask, + blend: ct.blend.as_ref().map(conv::map_blend), + }); + } + //Note: if any of the states are different, and `INDEPENDENT_BLEND` flag + // is not exposed, then this pipeline will not bind correctly. + targets.into_boxed_slice() + }; + + self.counters.render_pipelines.add(1); + + Ok(super::RenderPipeline { + inner, + primitive: desc.primitive, + vertex_buffers, + vertex_attributes, + color_targets, + depth: desc.depth_stencil.as_ref().map(|ds| super::DepthState { + function: conv::map_compare_func(ds.depth_compare.unwrap_or_default()), + mask: ds.depth_write_enabled.unwrap_or_default(), + }), + depth_bias: desc + .depth_stencil + .as_ref() + .map(|ds| ds.bias) + .unwrap_or_default(), + stencil: desc + .depth_stencil + .as_ref() + .map(|ds| conv::map_stencil(&ds.stencil)), + alpha_to_coverage_enabled: desc.multisample.alpha_to_coverage_enabled, + }) + } + + unsafe fn destroy_render_pipeline(&self, pipeline: super::RenderPipeline) { + // If the pipeline only has 2 strong references remaining, they're `pipeline` and `program_cache` + // This is safe to assume as long as: + // - `RenderPipeline` can't be cloned + // - The only place that we can get a new reference is during `program_cache.lock()` + if Arc::strong_count(&pipeline.inner) == 2 { + let gl = &self.shared.context.lock(); + let mut program_cache = self.shared.program_cache.lock(); + program_cache.retain(|_, v| match *v { + Ok(ref p) => p.program != pipeline.inner.program, + Err(_) => false, + }); + unsafe { gl.delete_program(pipeline.inner.program) }; + } + + self.counters.render_pipelines.sub(1); + } + + unsafe fn create_compute_pipeline( + &self, + desc: &crate::ComputePipelineDescriptor< + super::PipelineLayout, + super::ShaderModule, + super::PipelineCache, + >, + ) -> Result { + let gl = &self.shared.context.lock(); + let mut shaders = ArrayVec::new(); + shaders.push((naga::ShaderStage::Compute, &desc.stage)); + let inner = unsafe { self.create_pipeline(gl, shaders, desc.layout, desc.label, None) }?; + + self.counters.compute_pipelines.add(1); + + Ok(super::ComputePipeline { inner }) + } + + unsafe fn destroy_compute_pipeline(&self, pipeline: super::ComputePipeline) { + // If the pipeline only has 2 strong references remaining, they're `pipeline` and `program_cache`` + // This is safe to assume as long as: + // - `ComputePipeline` can't be cloned + // - The only place that we can get a new reference is during `program_cache.lock()` + if Arc::strong_count(&pipeline.inner) == 2 { + let gl = &self.shared.context.lock(); + let mut program_cache = self.shared.program_cache.lock(); + program_cache.retain(|_, v| match *v { + Ok(ref p) => p.program != pipeline.inner.program, + Err(_) => false, + }); + unsafe { gl.delete_program(pipeline.inner.program) }; + } + + self.counters.compute_pipelines.sub(1); + } + + unsafe fn create_pipeline_cache( + &self, + _: &crate::PipelineCacheDescriptor<'_>, + ) -> Result { + // Even though the cache doesn't do anything, we still return something here + // as the least bad option + Ok(super::PipelineCache) + } + unsafe fn destroy_pipeline_cache(&self, _: super::PipelineCache) {} + + #[cfg_attr(target_arch = "wasm32", allow(unused))] + unsafe fn create_query_set( + &self, + desc: &wgt::QuerySetDescriptor, + ) -> Result { + let gl = &self.shared.context.lock(); + + let mut queries = Vec::with_capacity(desc.count as usize); + for _ in 0..desc.count { + let query = + unsafe { gl.create_query() }.map_err(|_| crate::DeviceError::OutOfMemory)?; + + // We aren't really able to, in general, label queries. + // + // We could take a timestamp here to "initialize" the query, + // but that's a bit of a hack, and we don't want to insert + // random timestamps into the command stream of we don't have to. + + queries.push(query); + } + + self.counters.query_sets.add(1); + + Ok(super::QuerySet { + queries: queries.into_boxed_slice(), + target: match desc.ty { + wgt::QueryType::Occlusion => glow::ANY_SAMPLES_PASSED_CONSERVATIVE, + wgt::QueryType::Timestamp => glow::TIMESTAMP, + _ => unimplemented!(), + }, + }) + } + + unsafe fn destroy_query_set(&self, set: super::QuerySet) { + let gl = &self.shared.context.lock(); + for &query in set.queries.iter() { + unsafe { gl.delete_query(query) }; + } + self.counters.query_sets.sub(1); + } + + unsafe fn create_fence(&self) -> Result { + self.counters.fences.add(1); + Ok(super::Fence::new(&self.shared.options)) + } + + unsafe fn destroy_fence(&self, fence: super::Fence) { + let gl = &self.shared.context.lock(); + fence.destroy(gl); + self.counters.fences.sub(1); + } + + unsafe fn get_fence_value( + &self, + fence: &super::Fence, + ) -> Result { + #[cfg_attr(target_arch = "wasm32", allow(clippy::needless_borrow))] + Ok(fence.get_latest(&self.shared.context.lock())) + } + unsafe fn wait( + &self, + fence: &super::Fence, + wait_value: crate::FenceValue, + timeout: Option, + ) -> Result { + if fence.satisfied(wait_value) { + return Ok(true); + } + + let gl = &self.shared.context.lock(); + // MAX_CLIENT_WAIT_TIMEOUT_WEBGL is: + // - 1s in Gecko https://searchfox.org/mozilla-central/rev/754074e05178e017ef6c3d8e30428ffa8f1b794d/dom/canvas/WebGLTypes.h#1386 + // - 0 in WebKit https://github.com/WebKit/WebKit/blob/4ef90d4672ca50267c0971b85db403d9684508ea/Source/WebCore/html/canvas/WebGL2RenderingContext.cpp#L110 + // - 0 in Chromium https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/modules/webgl/webgl2_rendering_context_base.cc;l=112;drc=a3cb0ac4c71ec04abfeaed199e5d63230eca2551 + let timeout_ns = if cfg!(any(webgl, Emscripten)) { + 0 + } else { + timeout + .map(|t| t.as_nanos().min(u32::MAX as u128) as u32) + .unwrap_or(u32::MAX) + }; + fence.wait(gl, wait_value, timeout_ns) + } + + unsafe fn start_graphics_debugger_capture(&self) -> bool { + #[cfg(all(native, feature = "renderdoc"))] + return unsafe { + self.render_doc + .start_frame_capture(self.shared.context.raw_context(), ptr::null_mut()) + }; + #[allow(unreachable_code)] + false + } + unsafe fn stop_graphics_debugger_capture(&self) { + #[cfg(all(native, feature = "renderdoc"))] + unsafe { + self.render_doc + .end_frame_capture(ptr::null_mut(), ptr::null_mut()) + } + } + unsafe fn create_acceleration_structure( + &self, + _desc: &crate::AccelerationStructureDescriptor, + ) -> Result { + unimplemented!() + } + unsafe fn get_acceleration_structure_build_sizes<'a>( + &self, + _desc: &crate::GetAccelerationStructureBuildSizesDescriptor<'a, super::Buffer>, + ) -> crate::AccelerationStructureBuildSizes { + unimplemented!() + } + unsafe fn get_acceleration_structure_device_address( + &self, + _acceleration_structure: &super::AccelerationStructure, + ) -> wgt::BufferAddress { + unimplemented!() + } + unsafe fn destroy_acceleration_structure( + &self, + _acceleration_structure: super::AccelerationStructure, + ) { + } + + fn tlas_instance_to_bytes(&self, _instance: TlasInstance) -> Vec { + unimplemented!() + } + + fn get_internal_counters(&self) -> wgt::HalCounters { + self.counters.as_ref().clone() + } + + fn check_if_oom(&self) -> Result<(), crate::DeviceError> { + Ok(()) + } +} + +#[cfg(send_sync)] +unsafe impl Sync for super::Device {} +#[cfg(send_sync)] +unsafe impl Send for super::Device {} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/egl.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/egl.rs new file mode 100644 index 00000000..9bcac23b --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/egl.rs @@ -0,0 +1,1460 @@ +use alloc::{string::String, sync::Arc, vec::Vec}; +use core::{ffi, mem::ManuallyDrop, ptr, time::Duration}; +use std::sync::LazyLock; + +use glow::HasContext; +use hashbrown::HashMap; +use parking_lot::{MappedMutexGuard, Mutex, MutexGuard, RwLock}; + +/// The amount of time to wait while trying to obtain a lock to the adapter context +const CONTEXT_LOCK_TIMEOUT_SECS: u64 = 6; + +const EGL_CONTEXT_FLAGS_KHR: i32 = 0x30FC; +const EGL_CONTEXT_OPENGL_DEBUG_BIT_KHR: i32 = 0x0001; +const EGL_CONTEXT_OPENGL_ROBUST_ACCESS_EXT: i32 = 0x30BF; +const EGL_PLATFORM_WAYLAND_KHR: u32 = 0x31D8; +const EGL_PLATFORM_X11_KHR: u32 = 0x31D5; +const EGL_PLATFORM_XCB_EXT: u32 = 0x31DC; +const EGL_PLATFORM_XCB_SCREEN_EXT: u32 = 0x31DE; +const EGL_PLATFORM_ANGLE_ANGLE: u32 = 0x3202; +const EGL_PLATFORM_ANGLE_NATIVE_PLATFORM_TYPE_ANGLE: u32 = 0x348F; +const EGL_PLATFORM_ANGLE_DEBUG_LAYERS_ENABLED: u32 = 0x3451; +const EGL_PLATFORM_SURFACELESS_MESA: u32 = 0x31DD; +const EGL_GL_COLORSPACE_KHR: u32 = 0x309D; +const EGL_GL_COLORSPACE_SRGB_KHR: u32 = 0x3089; + +#[cfg(not(Emscripten))] +type EglInstance = khronos_egl::DynamicInstance; + +#[cfg(Emscripten)] +type EglInstance = khronos_egl::Instance; + +type EglLabel = *const ffi::c_void; + +#[allow(clippy::upper_case_acronyms)] +type EGLDEBUGPROCKHR = Option< + unsafe extern "system" fn( + error: khronos_egl::Enum, + command: *const ffi::c_char, + message_type: u32, + thread_label: EglLabel, + object_label: EglLabel, + message: *const ffi::c_char, + ), +>; + +const EGL_DEBUG_MSG_CRITICAL_KHR: u32 = 0x33B9; +const EGL_DEBUG_MSG_ERROR_KHR: u32 = 0x33BA; +const EGL_DEBUG_MSG_WARN_KHR: u32 = 0x33BB; +const EGL_DEBUG_MSG_INFO_KHR: u32 = 0x33BC; + +type EglDebugMessageControlFun = unsafe extern "system" fn( + proc: EGLDEBUGPROCKHR, + attrib_list: *const khronos_egl::Attrib, +) -> ffi::c_int; + +unsafe extern "system" fn egl_debug_proc( + error: khronos_egl::Enum, + command_raw: *const ffi::c_char, + message_type: u32, + _thread_label: EglLabel, + _object_label: EglLabel, + message_raw: *const ffi::c_char, +) { + let log_severity = match message_type { + EGL_DEBUG_MSG_CRITICAL_KHR | EGL_DEBUG_MSG_ERROR_KHR => log::Level::Error, + EGL_DEBUG_MSG_WARN_KHR => log::Level::Warn, + // We intentionally suppress info messages down to debug + // so that users are not inundated with info messages from + // the runtime. + EGL_DEBUG_MSG_INFO_KHR => log::Level::Debug, + _ => log::Level::Trace, + }; + let command = unsafe { ffi::CStr::from_ptr(command_raw) }.to_string_lossy(); + let message = if message_raw.is_null() { + "".into() + } else { + unsafe { ffi::CStr::from_ptr(message_raw) }.to_string_lossy() + }; + + log::log!(log_severity, "EGL '{command}' code 0x{error:x}: {message}",); +} + +#[derive(Clone, Copy, Debug)] +enum SrgbFrameBufferKind { + /// No support for SRGB surface + None, + /// Using EGL 1.5's support for colorspaces + Core, + /// Using EGL_KHR_gl_colorspace + Khr, +} + +/// Choose GLES framebuffer configuration. +fn choose_config( + egl: &EglInstance, + display: khronos_egl::Display, + srgb_kind: SrgbFrameBufferKind, +) -> Result<(khronos_egl::Config, bool), crate::InstanceError> { + //TODO: EGL_SLOW_CONFIG + let tiers = [ + ( + "off-screen", + &[ + khronos_egl::SURFACE_TYPE, + khronos_egl::PBUFFER_BIT, + khronos_egl::RENDERABLE_TYPE, + khronos_egl::OPENGL_ES2_BIT, + ][..], + ), + ( + "presentation", + &[khronos_egl::SURFACE_TYPE, khronos_egl::WINDOW_BIT][..], + ), + #[cfg(not(target_os = "android"))] + ( + "native-render", + &[khronos_egl::NATIVE_RENDERABLE, khronos_egl::TRUE as _][..], + ), + ]; + + let mut attributes = Vec::with_capacity(9); + for tier_max in (0..tiers.len()).rev() { + let name = tiers[tier_max].0; + log::debug!("\tTrying {name}"); + + attributes.clear(); + for &(_, tier_attr) in tiers[..=tier_max].iter() { + attributes.extend_from_slice(tier_attr); + } + // make sure the Alpha is enough to support sRGB + match srgb_kind { + SrgbFrameBufferKind::None => {} + _ => { + attributes.push(khronos_egl::ALPHA_SIZE); + attributes.push(8); + } + } + attributes.push(khronos_egl::NONE); + + match egl.choose_first_config(display, &attributes) { + Ok(Some(config)) => { + if tier_max == 1 { + //Note: this has been confirmed to malfunction on Intel+NV laptops, + // but also on Angle. + log::info!("EGL says it can present to the window but not natively",); + } + // Android emulator can't natively present either. + let tier_threshold = + if cfg!(target_os = "android") || cfg!(windows) || cfg!(target_env = "ohos") { + 1 + } else { + 2 + }; + return Ok((config, tier_max >= tier_threshold)); + } + Ok(None) => { + log::debug!("No config found!"); + } + Err(e) => { + log::error!("error in choose_first_config: {e:?}"); + } + } + } + + // TODO: include diagnostic details that are currently logged + Err(crate::InstanceError::new(String::from( + "unable to find an acceptable EGL framebuffer configuration", + ))) +} + +#[derive(Clone, Debug)] +struct EglContext { + instance: Arc, + version: (i32, i32), + display: khronos_egl::Display, + raw: khronos_egl::Context, + pbuffer: Option, +} + +impl EglContext { + fn make_current(&self) { + self.instance + .make_current(self.display, self.pbuffer, self.pbuffer, Some(self.raw)) + .unwrap(); + } + + fn unmake_current(&self) { + self.instance + .make_current(self.display, None, None, None) + .unwrap(); + } +} + +/// A wrapper around a [`glow::Context`] and the required EGL context that uses locking to guarantee +/// exclusive access when shared with multiple threads. +pub struct AdapterContext { + glow: Mutex>, + egl: Option, +} + +unsafe impl Sync for AdapterContext {} +unsafe impl Send for AdapterContext {} + +impl AdapterContext { + pub fn is_owned(&self) -> bool { + self.egl.is_some() + } + + /// Returns the EGL instance. + /// + /// This provides access to EGL functions and the ability to load GL and EGL extension functions. + pub fn egl_instance(&self) -> Option<&EglInstance> { + self.egl.as_ref().map(|egl| &*egl.instance) + } + + /// Returns the EGLDisplay corresponding to the adapter context. + /// + /// Returns [`None`] if the adapter was externally created. + pub fn raw_display(&self) -> Option<&khronos_egl::Display> { + self.egl.as_ref().map(|egl| &egl.display) + } + + /// Returns the EGL version the adapter context was created with. + /// + /// Returns [`None`] if the adapter was externally created. + pub fn egl_version(&self) -> Option<(i32, i32)> { + self.egl.as_ref().map(|egl| egl.version) + } + + pub fn raw_context(&self) -> *mut ffi::c_void { + match self.egl { + Some(ref egl) => egl.raw.as_ptr(), + None => ptr::null_mut(), + } + } +} + +impl Drop for AdapterContext { + fn drop(&mut self) { + struct CurrentGuard<'a>(&'a EglContext); + impl Drop for CurrentGuard<'_> { + fn drop(&mut self) { + self.0.unmake_current(); + } + } + + // Context must be current when dropped. See safety docs on + // `glow::HasContext`. + // + // NOTE: This is only set to `None` by `Adapter::new_external` which + // requires the context to be current when anything that may be holding + // the `Arc` is dropped. + let _guard = self.egl.as_ref().map(|egl| { + egl.make_current(); + CurrentGuard(egl) + }); + let glow = self.glow.get_mut(); + // SAFETY: Field not used after this. + unsafe { ManuallyDrop::drop(glow) }; + } +} + +struct EglContextLock<'a> { + instance: &'a Arc, + display: khronos_egl::Display, +} + +/// A guard containing a lock to an [`AdapterContext`], while the GL context is kept current. +pub struct AdapterContextLock<'a> { + glow: MutexGuard<'a, ManuallyDrop>, + egl: Option>, +} + +impl<'a> core::ops::Deref for AdapterContextLock<'a> { + type Target = glow::Context; + + fn deref(&self) -> &Self::Target { + &self.glow + } +} + +impl<'a> Drop for AdapterContextLock<'a> { + fn drop(&mut self) { + if let Some(egl) = self.egl.take() { + if let Err(err) = egl.instance.make_current(egl.display, None, None, None) { + log::error!("Failed to make EGL context current: {err:?}"); + } + } + } +} + +impl AdapterContext { + /// Get's the [`glow::Context`] without waiting for a lock + /// + /// # Safety + /// + /// This should only be called when you have manually made sure that the current thread has made + /// the EGL context current and that no other thread also has the EGL context current. + /// Additionally, you must manually make the EGL context **not** current after you are done with + /// it, so that future calls to `lock()` will not fail. + /// + /// > **Note:** Calling this function **will** still lock the [`glow::Context`] which adds an + /// > extra safe-guard against accidental concurrent access to the context. + pub unsafe fn get_without_egl_lock(&self) -> MappedMutexGuard<'_, glow::Context> { + let guard = self + .glow + .try_lock_for(Duration::from_secs(CONTEXT_LOCK_TIMEOUT_SECS)) + .expect("Could not lock adapter context. This is most-likely a deadlock."); + MutexGuard::map(guard, |glow| &mut **glow) + } + + /// Obtain a lock to the EGL context and get handle to the [`glow::Context`] that can be used to + /// do rendering. + #[track_caller] + pub fn lock<'a>(&'a self) -> AdapterContextLock<'a> { + let glow = self + .glow + // Don't lock forever. If it takes longer than 1 second to get the lock we've got a + // deadlock and should panic to show where we got stuck + .try_lock_for(Duration::from_secs(CONTEXT_LOCK_TIMEOUT_SECS)) + .expect("Could not lock adapter context. This is most-likely a deadlock."); + + let egl = self.egl.as_ref().map(|egl| { + egl.make_current(); + EglContextLock { + instance: &egl.instance, + display: egl.display, + } + }); + + AdapterContextLock { glow, egl } + } +} + +#[derive(Debug)] +struct Inner { + /// Note: the context contains a dummy pbuffer (1x1). + /// Required for `eglMakeCurrent` on platforms that doesn't supports `EGL_KHR_surfaceless_context`. + egl: EglContext, + version: (i32, i32), + supports_native_window: bool, + config: khronos_egl::Config, + /// Method by which the framebuffer should support srgb + srgb_kind: SrgbFrameBufferKind, +} + +// Different calls to `eglGetPlatformDisplay` may return the same `Display`, making it a global +// state of all our `EglContext`s. This forces us to track the number of such context to prevent +// terminating the display if it's currently used by another `EglContext`. +static DISPLAYS_REFERENCE_COUNT: LazyLock>> = + LazyLock::new(Default::default); + +fn initialize_display( + egl: &EglInstance, + display: khronos_egl::Display, +) -> Result<(i32, i32), khronos_egl::Error> { + let mut guard = DISPLAYS_REFERENCE_COUNT.lock(); + *guard.entry(display.as_ptr() as usize).or_default() += 1; + + // We don't need to check the reference count here since according to the `eglInitialize` + // documentation, initializing an already initialized EGL display connection has no effect + // besides returning the version numbers. + egl.initialize(display) +} + +fn terminate_display( + egl: &EglInstance, + display: khronos_egl::Display, +) -> Result<(), khronos_egl::Error> { + let key = &(display.as_ptr() as usize); + let mut guard = DISPLAYS_REFERENCE_COUNT.lock(); + let count_ref = guard + .get_mut(key) + .expect("Attempted to decref a display before incref was called"); + + if *count_ref > 1 { + *count_ref -= 1; + + Ok(()) + } else { + guard.remove(key); + + egl.terminate(display) + } +} + +fn instance_err( + message: impl Into, +) -> impl FnOnce(E) -> crate::InstanceError { + move |e| crate::InstanceError::with_source(message.into(), e) +} + +impl Inner { + fn create( + flags: wgt::InstanceFlags, + egl: Arc, + display: khronos_egl::Display, + force_gles_minor_version: wgt::Gles3MinorVersion, + ) -> Result { + let version = initialize_display(&egl, display) + .map_err(instance_err("failed to initialize EGL display connection"))?; + let vendor = egl + .query_string(Some(display), khronos_egl::VENDOR) + .map_err(instance_err("failed to query EGL vendor"))?; + let display_extensions = egl + .query_string(Some(display), khronos_egl::EXTENSIONS) + .map_err(instance_err("failed to query EGL display extensions"))? + .to_string_lossy(); + log::debug!("Display vendor {vendor:?}, version {version:?}",); + log::debug!( + "Display extensions: {:#?}", + display_extensions.split_whitespace().collect::>() + ); + + let srgb_kind = if version >= (1, 5) { + log::debug!("\tEGL surface: +srgb"); + SrgbFrameBufferKind::Core + } else if display_extensions.contains("EGL_KHR_gl_colorspace") { + log::debug!("\tEGL surface: +srgb khr"); + SrgbFrameBufferKind::Khr + } else { + log::debug!("\tEGL surface: -srgb"); + SrgbFrameBufferKind::None + }; + + if log::max_level() >= log::LevelFilter::Trace { + log::trace!("Configurations:"); + let config_count = egl + .get_config_count(display) + .map_err(instance_err("failed to get config count"))?; + let mut configurations = Vec::with_capacity(config_count); + egl.get_configs(display, &mut configurations) + .map_err(instance_err("failed to get configs"))?; + for &config in configurations.iter() { + log::trace!("\tCONFORMANT=0x{:X?}, RENDERABLE=0x{:X?}, NATIVE_RENDERABLE=0x{:X?}, SURFACE_TYPE=0x{:X?}, ALPHA_SIZE={:?}", + egl.get_config_attrib(display, config, khronos_egl::CONFORMANT), + egl.get_config_attrib(display, config, khronos_egl::RENDERABLE_TYPE), + egl.get_config_attrib(display, config, khronos_egl::NATIVE_RENDERABLE), + egl.get_config_attrib(display, config, khronos_egl::SURFACE_TYPE), + egl.get_config_attrib(display, config, khronos_egl::ALPHA_SIZE), + ); + } + } + + let (config, supports_native_window) = choose_config(&egl, display, srgb_kind)?; + + let supports_opengl = if version >= (1, 4) { + let client_apis = egl + .query_string(Some(display), khronos_egl::CLIENT_APIS) + .map_err(instance_err("failed to query EGL client APIs string"))? + .to_string_lossy(); + client_apis + .split(' ') + .any(|client_api| client_api == "OpenGL") + } else { + false + }; + + let mut khr_context_flags = 0; + let supports_khr_context = display_extensions.contains("EGL_KHR_create_context"); + + let mut context_attributes = vec![]; + let mut gl_context_attributes = vec![]; + let mut gles_context_attributes = vec![]; + gl_context_attributes.push(khronos_egl::CONTEXT_MAJOR_VERSION); + gl_context_attributes.push(3); + gl_context_attributes.push(khronos_egl::CONTEXT_MINOR_VERSION); + gl_context_attributes.push(3); + if supports_opengl && force_gles_minor_version != wgt::Gles3MinorVersion::Automatic { + log::warn!("Ignoring specified GLES minor version as OpenGL is used"); + } + gles_context_attributes.push(khronos_egl::CONTEXT_MAJOR_VERSION); + gles_context_attributes.push(3); // Request GLES 3.0 or higher + if force_gles_minor_version != wgt::Gles3MinorVersion::Automatic { + gles_context_attributes.push(khronos_egl::CONTEXT_MINOR_VERSION); + gles_context_attributes.push(match force_gles_minor_version { + wgt::Gles3MinorVersion::Automatic => unreachable!(), + wgt::Gles3MinorVersion::Version0 => 0, + wgt::Gles3MinorVersion::Version1 => 1, + wgt::Gles3MinorVersion::Version2 => 2, + }); + } + if flags.contains(wgt::InstanceFlags::DEBUG) { + if version >= (1, 5) { + log::debug!("\tEGL context: +debug"); + context_attributes.push(khronos_egl::CONTEXT_OPENGL_DEBUG); + context_attributes.push(khronos_egl::TRUE as _); + } else if supports_khr_context { + log::debug!("\tEGL context: +debug KHR"); + khr_context_flags |= EGL_CONTEXT_OPENGL_DEBUG_BIT_KHR; + } else { + log::debug!("\tEGL context: -debug"); + } + } + + if khr_context_flags != 0 { + context_attributes.push(EGL_CONTEXT_FLAGS_KHR); + context_attributes.push(khr_context_flags); + } + + gl_context_attributes.extend(&context_attributes); + gles_context_attributes.extend(&context_attributes); + + let context = { + #[derive(Copy, Clone)] + enum Robustness { + Core, + Ext, + } + + let robustness = if version >= (1, 5) { + Some(Robustness::Core) + } else if display_extensions.contains("EGL_EXT_create_context_robustness") { + Some(Robustness::Ext) + } else { + None + }; + + let create_context = |api, base_attributes: &[khronos_egl::Int]| { + egl.bind_api(api)?; + + let mut robustness = robustness; + loop { + let robustness_attributes = match robustness { + Some(Robustness::Core) => { + vec![ + khronos_egl::CONTEXT_OPENGL_ROBUST_ACCESS, + khronos_egl::TRUE as _, + khronos_egl::NONE, + ] + } + Some(Robustness::Ext) => { + vec![ + EGL_CONTEXT_OPENGL_ROBUST_ACCESS_EXT, + khronos_egl::TRUE as _, + khronos_egl::NONE, + ] + } + None => vec![khronos_egl::NONE], + }; + + let mut context_attributes = base_attributes.to_vec(); + context_attributes.extend(&robustness_attributes); + + match egl.create_context(display, config, None, &context_attributes) { + Ok(context) => { + match robustness { + Some(Robustness::Core) => { + log::debug!("\tEGL context: +robust access"); + } + Some(Robustness::Ext) => { + log::debug!("\tEGL context: +robust access EXT"); + } + None => { + log::debug!("\tEGL context: -robust access"); + } + } + return Ok(context); + } + + // Robust access context creation can fail with different error codes + // depending on the EGL path. Retry with a lower robustness level. + Err( + khronos_egl::Error::BadAttribute + | khronos_egl::Error::BadMatch + | khronos_egl::Error::BadConfig, + ) if robustness.is_some() => { + robustness = match robustness { + Some(Robustness::Core) + if display_extensions + .contains("EGL_EXT_create_context_robustness") => + { + Some(Robustness::Ext) + } + _ => None, + }; + continue; + } + + Err(e) => return Err(e), + } + } + }; + + let result = if supports_opengl { + create_context(khronos_egl::OPENGL_API, &gl_context_attributes).or_else( + |gl_error| { + log::debug!("Failed to create desktop OpenGL context: {gl_error}, falling back to OpenGL ES"); + create_context(khronos_egl::OPENGL_ES_API, &gles_context_attributes) + }, + ) + } else { + create_context(khronos_egl::OPENGL_ES_API, &gles_context_attributes) + }; + + result.map_err(|e| { + crate::InstanceError::with_source( + String::from("unable to create OpenGL or GLES 3.x context"), + e, + ) + }) + }?; + + // Testing if context can be binded without surface + // and creating dummy pbuffer surface if not. + let pbuffer = if version >= (1, 5) + || display_extensions.contains("EGL_KHR_surfaceless_context") + || cfg!(Emscripten) + { + log::debug!("\tEGL context: +surfaceless"); + None + } else { + let attributes = [ + khronos_egl::WIDTH, + 1, + khronos_egl::HEIGHT, + 1, + khronos_egl::NONE, + ]; + egl.create_pbuffer_surface(display, config, &attributes) + .map(Some) + .map_err(|e| { + crate::InstanceError::with_source( + String::from("error in create_pbuffer_surface"), + e, + ) + })? + }; + + Ok(Self { + egl: EglContext { + instance: egl, + display, + raw: context, + pbuffer, + version, + }, + version, + supports_native_window, + config, + srgb_kind, + }) + } +} + +impl Drop for Inner { + fn drop(&mut self) { + // ERROR: Since EglContext is erroneously Clone, these handles could be copied and + // accidentally used elsewhere outside of Inner, despite us assuming ownership and + // destroying the handles here. + if let Err(e) = self + .egl + .instance + .destroy_context(self.egl.display, self.egl.raw) + { + log::warn!("Error in destroy_context: {e:?}"); + } + + if let Err(e) = terminate_display(&self.egl.instance, self.egl.display) { + log::warn!("Error in terminate: {e:?}"); + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +enum WindowKind { + Wayland, + X11, + AngleX11, + Unknown, +} + +#[derive(Clone, Debug)] +struct WindowSystemInterface { + kind: WindowKind, +} + +pub struct Instance { + wsi: WindowSystemInterface, + flags: wgt::InstanceFlags, + options: wgt::GlBackendOptions, + inner: Mutex, +} + +impl Instance { + pub fn raw_display(&self) -> khronos_egl::Display { + self.inner + .try_lock() + .expect("Could not lock instance. This is most-likely a deadlock.") + .egl + .display + } + + /// Returns the version of the EGL display. + pub fn egl_version(&self) -> (i32, i32) { + self.inner + .try_lock() + .expect("Could not lock instance. This is most-likely a deadlock.") + .version + } + + pub fn egl_config(&self) -> khronos_egl::Config { + self.inner + .try_lock() + .expect("Could not lock instance. This is most-likely a deadlock.") + .config + } +} + +unsafe impl Send for Instance {} +unsafe impl Sync for Instance {} + +impl crate::Instance for Instance { + type A = super::Api; + + unsafe fn init(desc: &crate::InstanceDescriptor<'_>) -> Result { + use raw_window_handle::RawDisplayHandle as Rdh; + + profiling::scope!("Init OpenGL (EGL) Backend"); + #[cfg(Emscripten)] + let egl_result: Result = + Ok(khronos_egl::Instance::new(khronos_egl::Static)); + + #[cfg(not(Emscripten))] + let egl_result = if cfg!(windows) { + unsafe { + khronos_egl::DynamicInstance::::load_required_from_filename( + "libEGL.dll", + ) + } + } else if cfg!(target_vendor = "apple") { + unsafe { + khronos_egl::DynamicInstance::::load_required_from_filename( + "libEGL.dylib", + ) + } + } else { + unsafe { khronos_egl::DynamicInstance::::load_required() } + }; + let egl = egl_result + .map(Arc::new) + .map_err(instance_err("unable to open libEGL"))?; + + let client_extensions = egl.query_string(None, khronos_egl::EXTENSIONS); + + let client_ext_str = match client_extensions { + Ok(ext) => ext.to_string_lossy().into_owned(), + Err(_) => String::new(), + }; + log::debug!( + "Client extensions: {:#?}", + client_ext_str.split_whitespace().collect::>() + ); + + #[cfg(not(Emscripten))] + let egl1_5 = egl.upcast::(); + + #[cfg(Emscripten)] + let egl1_5: Option<&Arc> = Some(&egl); + + let (display, wsi_kind) = match (desc.display.map(|d| d.as_raw()), egl1_5) { + (Some(Rdh::Wayland(wayland_display_handle)), Some(egl)) + if client_ext_str.contains("EGL_EXT_platform_wayland") => + { + log::debug!("Using Wayland platform"); + let display_attributes = [khronos_egl::ATTRIB_NONE]; + let display = unsafe { + egl.get_platform_display( + EGL_PLATFORM_WAYLAND_KHR, + wayland_display_handle.display.as_ptr(), + &display_attributes, + ) + } + .map_err(instance_err("failed to get Wayland display"))?; + (display, WindowKind::Wayland) + } + (Some(Rdh::Xlib(xlib_display_handle)), Some(egl)) + if client_ext_str.contains("EGL_EXT_platform_x11") => + { + log::debug!("Using X11 platform"); + let display_attributes = [khronos_egl::ATTRIB_NONE]; + let display = unsafe { + egl.get_platform_display( + EGL_PLATFORM_X11_KHR, + xlib_display_handle + .display + .map_or(khronos_egl::DEFAULT_DISPLAY, ptr::NonNull::as_ptr), + &display_attributes, + ) + } + .map_err(instance_err("failed to get X11 display"))?; + (display, WindowKind::X11) + } + (Some(Rdh::Xlib(xlib_display_handle)), Some(egl)) + if client_ext_str.contains("EGL_ANGLE_platform_angle") => + { + log::debug!("Using Angle platform with X11"); + let display_attributes = [ + EGL_PLATFORM_ANGLE_NATIVE_PLATFORM_TYPE_ANGLE as khronos_egl::Attrib, + EGL_PLATFORM_X11_KHR as khronos_egl::Attrib, + EGL_PLATFORM_ANGLE_DEBUG_LAYERS_ENABLED as khronos_egl::Attrib, + usize::from(desc.flags.contains(wgt::InstanceFlags::VALIDATION)), + khronos_egl::ATTRIB_NONE, + ]; + let display = unsafe { + egl.get_platform_display( + EGL_PLATFORM_ANGLE_ANGLE, + xlib_display_handle + .display + .map_or(khronos_egl::DEFAULT_DISPLAY, ptr::NonNull::as_ptr), + &display_attributes, + ) + } + .map_err(instance_err("failed to get Angle display"))?; + (display, WindowKind::AngleX11) + } + (Some(Rdh::Xcb(xcb_display_handle)), Some(egl)) + if client_ext_str.contains("EGL_EXT_platform_xcb") => + { + log::debug!("Using XCB platform"); + let display_attributes = [ + EGL_PLATFORM_XCB_SCREEN_EXT as khronos_egl::Attrib, + xcb_display_handle.screen as khronos_egl::Attrib, + khronos_egl::ATTRIB_NONE, + ]; + let display = unsafe { + egl.get_platform_display( + EGL_PLATFORM_XCB_EXT, + xcb_display_handle + .connection + .map_or(khronos_egl::DEFAULT_DISPLAY, ptr::NonNull::as_ptr), + &display_attributes, + ) + } + .map_err(instance_err("failed to get XCB display"))?; + (display, WindowKind::X11) + } + x if client_ext_str.contains("EGL_MESA_platform_surfaceless") => { + log::debug!( + "No (or unknown) windowing system ({x:?}) present. Using surfaceless platform" + ); + #[allow(clippy::unnecessary_literal_unwrap)] + // This is only a literal on Emscripten + // TODO: This extension is also supported on EGL 1.4 with EGL_EXT_platform_base: https://registry.khronos.org/EGL/extensions/MESA/EGL_MESA_platform_surfaceless.txt + let egl = egl1_5.expect("Failed to get EGL 1.5 for surfaceless"); + let display = unsafe { + egl.get_platform_display( + EGL_PLATFORM_SURFACELESS_MESA, + khronos_egl::DEFAULT_DISPLAY, + &[khronos_egl::ATTRIB_NONE], + ) + } + .map_err(instance_err("failed to get MESA surfaceless display"))?; + (display, WindowKind::Unknown) + } + x => { + log::debug!( + "No (or unknown) windowing system {x:?} and EGL_MESA_platform_surfaceless not available. Using default platform" + ); + let display = + unsafe { egl.get_display(khronos_egl::DEFAULT_DISPLAY) }.ok_or_else(|| { + crate::InstanceError::new("Failed to get default display".into()) + })?; + (display, WindowKind::Unknown) + } + }; + + if desc.flags.contains(wgt::InstanceFlags::VALIDATION) + && client_ext_str.contains("EGL_KHR_debug") + { + log::debug!("Enabling EGL debug output"); + let function: EglDebugMessageControlFun = { + let addr = egl + .get_proc_address("eglDebugMessageControlKHR") + .ok_or_else(|| { + crate::InstanceError::new( + "failed to get `eglDebugMessageControlKHR` proc address".into(), + ) + })?; + unsafe { core::mem::transmute(addr) } + }; + let attributes = [ + EGL_DEBUG_MSG_CRITICAL_KHR as khronos_egl::Attrib, + 1, + EGL_DEBUG_MSG_ERROR_KHR as khronos_egl::Attrib, + 1, + EGL_DEBUG_MSG_WARN_KHR as khronos_egl::Attrib, + 1, + EGL_DEBUG_MSG_INFO_KHR as khronos_egl::Attrib, + 1, + khronos_egl::ATTRIB_NONE, + ]; + unsafe { (function)(Some(egl_debug_proc), attributes.as_ptr()) }; + } + + let inner = Inner::create( + desc.flags, + egl, + display, + desc.backend_options.gl.gles_minor_version, + )?; + + Ok(Instance { + wsi: WindowSystemInterface { kind: wsi_kind }, + flags: desc.flags, + options: desc.backend_options.gl.clone(), + inner: Mutex::new(inner), + }) + } + + unsafe fn create_surface( + &self, + display_handle: raw_window_handle::RawDisplayHandle, + window_handle: raw_window_handle::RawWindowHandle, + ) -> Result { + use raw_window_handle::RawWindowHandle as Rwh; + + let inner = self.inner.lock(); + + match (window_handle, display_handle) { + (Rwh::Xlib(_), _) => {} + (Rwh::Xcb(_), _) => {} + (Rwh::Win32(_), _) => {} + (Rwh::AppKit(_), _) => {} + (Rwh::OhosNdk(_), _) => {} + #[cfg(target_os = "android")] + (Rwh::AndroidNdk(handle), _) => { + let format = inner + .egl + .instance + .get_config_attrib( + inner.egl.display, + inner.config, + khronos_egl::NATIVE_VISUAL_ID, + ) + .map_err(instance_err("failed to get config NATIVE_VISUAL_ID"))?; + + let ret = unsafe { + ndk_sys::ANativeWindow_setBuffersGeometry( + handle + .a_native_window + .as_ptr() + .cast::(), + 0, + 0, + format, + ) + }; + + if ret != 0 { + return Err(crate::InstanceError::new(format!( + "error {ret} returned from ANativeWindow_setBuffersGeometry", + ))); + } + } + (Rwh::Wayland(_), _) => {} + #[cfg(Emscripten)] + (Rwh::Web(_), _) => {} + other => { + return Err(crate::InstanceError::new(format!( + "unsupported window: {other:?}" + ))); + } + }; + + inner.egl.unmake_current(); + + Ok(Surface { + egl: inner.egl.clone(), + wsi: self.wsi.clone(), + config: inner.config, + presentable: inner.supports_native_window, + raw_window_handle: window_handle, + swapchain: RwLock::new(None), + srgb_kind: inner.srgb_kind, + }) + } + + unsafe fn enumerate_adapters( + &self, + _surface_hint: Option<&Surface>, + ) -> Vec> { + let inner = self.inner.lock(); + inner.egl.make_current(); + + let mut gl = unsafe { + glow::Context::from_loader_function(|name| { + inner + .egl + .instance + .get_proc_address(name) + .map_or(ptr::null(), |p| p as *const _) + }) + }; + + // In contrast to OpenGL ES, OpenGL requires explicitly enabling sRGB conversions, + // as otherwise the user has to do the sRGB conversion. + if !matches!(inner.srgb_kind, SrgbFrameBufferKind::None) { + unsafe { gl.enable(glow::FRAMEBUFFER_SRGB) }; + } + + if self.flags.contains(wgt::InstanceFlags::DEBUG) && gl.supports_debug() { + log::debug!("Max label length: {}", unsafe { + gl.get_parameter_i32(glow::MAX_LABEL_LENGTH) + }); + } + + if self.flags.contains(wgt::InstanceFlags::VALIDATION) && gl.supports_debug() { + log::debug!("Enabling GLES debug output"); + unsafe { gl.enable(glow::DEBUG_OUTPUT) }; + unsafe { gl.debug_message_callback(super::gl_debug_message_callback) }; + } + + // Wrap in ManuallyDrop to make it easier to "current" the GL context before dropping this + // GLOW context, which could also happen if a panic occurs after we uncurrent the context + // below but before AdapterContext is constructed. + let gl = ManuallyDrop::new(gl); + inner.egl.unmake_current(); + + unsafe { + super::Adapter::expose( + AdapterContext { + glow: Mutex::new(gl), + // ERROR: Copying owned reference handles here, be careful to not drop them! + egl: Some(inner.egl.clone()), + }, + self.options.clone(), + ) + } + .into_iter() + .collect() + } +} + +impl super::Adapter { + /// Creates a new external adapter using the specified loader function. + /// + /// # Safety + /// + /// - The underlying OpenGL ES context must be current. + /// - The underlying OpenGL ES context must be current when interfacing with any objects returned by + /// wgpu-hal from this adapter. + /// - The underlying OpenGL ES context must be current when dropping this adapter and when + /// dropping any objects returned from this adapter. + pub unsafe fn new_external( + fun: impl FnMut(&str) -> *const ffi::c_void, + options: wgt::GlBackendOptions, + ) -> Option> { + let context = unsafe { glow::Context::from_loader_function(fun) }; + unsafe { + Self::expose( + AdapterContext { + glow: Mutex::new(ManuallyDrop::new(context)), + egl: None, + }, + options, + ) + } + } + + pub fn adapter_context(&self) -> &AdapterContext { + &self.shared.context + } +} + +impl super::Device { + /// Returns the underlying EGL context. + pub fn context(&self) -> &AdapterContext { + &self.shared.context + } +} + +#[derive(Debug)] +pub struct Swapchain { + surface: khronos_egl::Surface, + wl_window: Option<*mut wayland_sys::egl::wl_egl_window>, + framebuffer: glow::Framebuffer, + renderbuffer: glow::Renderbuffer, + /// Extent because the window lies + extent: wgt::Extent3d, + format: wgt::TextureFormat, + format_desc: super::TextureFormatDesc, + #[allow(unused)] + sample_type: wgt::TextureSampleType, +} + +#[derive(Debug)] +pub struct Surface { + egl: EglContext, + wsi: WindowSystemInterface, + config: khronos_egl::Config, + pub(super) presentable: bool, + raw_window_handle: raw_window_handle::RawWindowHandle, + swapchain: RwLock>, + srgb_kind: SrgbFrameBufferKind, +} + +unsafe impl Send for Surface {} +unsafe impl Sync for Surface {} + +impl Surface { + pub(super) unsafe fn present( + &self, + _suf_texture: super::Texture, + context: &AdapterContext, + ) -> Result<(), crate::SurfaceError> { + let gl = unsafe { context.get_without_egl_lock() }; + let swapchain = self.swapchain.read(); + let sc = swapchain.as_ref().ok_or(crate::SurfaceError::Other( + "Surface has no swap-chain configured", + ))?; + + self.egl + .instance + .make_current( + self.egl.display, + Some(sc.surface), + Some(sc.surface), + Some(self.egl.raw), + ) + .map_err(|e| { + log::error!("make_current(surface) failed: {e}"); + crate::SurfaceError::Lost + })?; + + unsafe { gl.disable(glow::SCISSOR_TEST) }; + unsafe { gl.color_mask(true, true, true, true) }; + + unsafe { gl.bind_framebuffer(glow::DRAW_FRAMEBUFFER, None) }; + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, Some(sc.framebuffer)) }; + + if !matches!(self.srgb_kind, SrgbFrameBufferKind::None) { + // Disable sRGB conversions for `glBlitFramebuffer` as behavior does diverge between + // drivers and formats otherwise and we want to ensure no sRGB conversions happen. + unsafe { gl.disable(glow::FRAMEBUFFER_SRGB) }; + } + + // Note the Y-flipping here. GL's presentation is not flipped, + // but main rendering is. Therefore, we Y-flip the output positions + // in the shader, and also this blit. + unsafe { + gl.blit_framebuffer( + 0, + sc.extent.height as i32, + sc.extent.width as i32, + 0, + 0, + 0, + sc.extent.width as i32, + sc.extent.height as i32, + glow::COLOR_BUFFER_BIT, + glow::NEAREST, + ) + }; + + if !matches!(self.srgb_kind, SrgbFrameBufferKind::None) { + unsafe { gl.enable(glow::FRAMEBUFFER_SRGB) }; + } + + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, None) }; + + self.egl + .instance + .swap_buffers(self.egl.display, sc.surface) + .map_err(|e| { + log::error!("swap_buffers failed: {e}"); + crate::SurfaceError::Lost + // TODO: should we unset the current context here? + })?; + self.egl + .instance + .make_current(self.egl.display, None, None, None) + .map_err(|e| { + log::error!("make_current(null) failed: {e}"); + crate::SurfaceError::Lost + })?; + + Ok(()) + } + + unsafe fn unconfigure_impl( + &self, + device: &super::Device, + ) -> Option<( + khronos_egl::Surface, + Option<*mut wayland_sys::egl::wl_egl_window>, + )> { + let gl = &device.shared.context.lock(); + match self.swapchain.write().take() { + Some(sc) => { + unsafe { gl.delete_renderbuffer(sc.renderbuffer) }; + unsafe { gl.delete_framebuffer(sc.framebuffer) }; + Some((sc.surface, sc.wl_window)) + } + None => None, + } + } + + pub fn supports_srgb(&self) -> bool { + match self.srgb_kind { + SrgbFrameBufferKind::None => false, + _ => true, + } + } +} + +impl crate::Surface for Surface { + type A = super::Api; + + unsafe fn configure( + &self, + device: &super::Device, + config: &crate::SurfaceConfiguration, + ) -> Result<(), crate::SurfaceError> { + use raw_window_handle::RawWindowHandle as Rwh; + + let (surface, wl_window) = match unsafe { self.unconfigure_impl(device) } { + Some((sc, wl_window)) => { + if let Some(window) = wl_window { + wayland_sys::ffi_dispatch!( + wayland_sys::egl::wayland_egl_handle(), + wl_egl_window_resize, + window, + config.extent.width as i32, + config.extent.height as i32, + 0, + 0, + ); + } + + (sc, wl_window) + } + None => { + let mut wl_window = None; + let (mut temp_xlib_handle, mut temp_xcb_handle); + let native_window_ptr = match (self.wsi.kind, self.raw_window_handle) { + (WindowKind::Unknown | WindowKind::X11, Rwh::Xlib(handle)) => { + temp_xlib_handle = handle.window; + ptr::from_mut(&mut temp_xlib_handle).cast::() + } + (WindowKind::AngleX11, Rwh::Xlib(handle)) => handle.window as *mut ffi::c_void, + (WindowKind::Unknown | WindowKind::X11, Rwh::Xcb(handle)) => { + temp_xcb_handle = handle.window; + ptr::from_mut(&mut temp_xcb_handle).cast::() + } + (WindowKind::AngleX11, Rwh::Xcb(handle)) => { + handle.window.get() as *mut ffi::c_void + } + (WindowKind::Unknown, Rwh::AndroidNdk(handle)) => { + handle.a_native_window.as_ptr() + } + (WindowKind::Unknown, Rwh::OhosNdk(handle)) => handle.native_window.as_ptr(), + #[cfg(unix)] + (WindowKind::Wayland, Rwh::Wayland(handle)) => { + let window = wayland_sys::ffi_dispatch!( + wayland_sys::egl::wayland_egl_handle(), + wl_egl_window_create, + handle.surface.as_ptr().cast(), + config.extent.width as i32, + config.extent.height as i32, + ); + wl_window = Some(window); + window.cast() + } + #[cfg(Emscripten)] + (WindowKind::Unknown, Rwh::Web(handle)) => handle.id as *mut ffi::c_void, + (WindowKind::Unknown, Rwh::Win32(handle)) => { + handle.hwnd.get() as *mut ffi::c_void + } + (WindowKind::Unknown, Rwh::AppKit(handle)) => { + #[cfg(not(target_os = "macos"))] + let window_ptr = handle.ns_view.as_ptr(); + #[cfg(target_os = "macos")] + let window_ptr = { + use objc2::msg_send; + use objc2::runtime::AnyObject; + // ns_view always have a layer and don't need to verify that it exists. + let layer: *mut AnyObject = + msg_send![handle.ns_view.as_ptr().cast::(), layer]; + layer.cast::() + }; + window_ptr + } + _ => { + log::warn!( + "Initialized platform {:?} doesn't work with window {:?}", + self.wsi.kind, + self.raw_window_handle + ); + return Err(crate::SurfaceError::Other("incompatible window kind")); + } + }; + + let mut attributes = vec![ + khronos_egl::RENDER_BUFFER, + // We don't want any of the buffering done by the driver, because we + // manage a swapchain on our side. + // Some drivers just fail on surface creation seeing `EGL_SINGLE_BUFFER`. + if cfg!(any( + target_os = "android", + target_os = "macos", + target_env = "ohos" + )) || cfg!(windows) + || self.wsi.kind == WindowKind::AngleX11 + { + khronos_egl::BACK_BUFFER + } else { + khronos_egl::SINGLE_BUFFER + }, + ]; + if config.format.is_srgb() { + match self.srgb_kind { + SrgbFrameBufferKind::None => {} + SrgbFrameBufferKind::Core => { + attributes.push(khronos_egl::GL_COLORSPACE); + attributes.push(khronos_egl::GL_COLORSPACE_SRGB); + } + SrgbFrameBufferKind::Khr => { + attributes.push(EGL_GL_COLORSPACE_KHR as i32); + attributes.push(EGL_GL_COLORSPACE_SRGB_KHR as i32); + } + } + } + attributes.push(khronos_egl::ATTRIB_NONE as i32); + + #[cfg(not(Emscripten))] + let egl1_5 = self.egl.instance.upcast::(); + + #[cfg(Emscripten)] + let egl1_5: Option<&Arc> = Some(&self.egl.instance); + + // Careful, we can still be in 1.4 version even if `upcast` succeeds + let raw_result = match egl1_5 { + Some(egl) if self.wsi.kind != WindowKind::Unknown => { + let attributes_usize = attributes + .into_iter() + .map(|v| v as usize) + .collect::>(); + unsafe { + egl.create_platform_window_surface( + self.egl.display, + self.config, + native_window_ptr, + &attributes_usize, + ) + } + } + _ => unsafe { + self.egl.instance.create_window_surface( + self.egl.display, + self.config, + native_window_ptr, + Some(&attributes), + ) + }, + }; + + match raw_result { + Ok(raw) => (raw, wl_window), + Err(e) => { + log::warn!("Error in create_window_surface: {e:?}"); + return Err(crate::SurfaceError::Lost); + } + } + } + }; + + let format_desc = device.shared.describe_texture_format(config.format); + let gl = &device.shared.context.lock(); + let renderbuffer = unsafe { gl.create_renderbuffer() }.map_err(|error| { + log::error!("Internal swapchain renderbuffer creation failed: {error}"); + crate::DeviceError::OutOfMemory + })?; + unsafe { gl.bind_renderbuffer(glow::RENDERBUFFER, Some(renderbuffer)) }; + unsafe { + gl.renderbuffer_storage( + glow::RENDERBUFFER, + format_desc.internal, + config.extent.width as _, + config.extent.height as _, + ) + }; + let framebuffer = unsafe { gl.create_framebuffer() }.map_err(|error| { + log::error!("Internal swapchain framebuffer creation failed: {error}"); + crate::DeviceError::OutOfMemory + })?; + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, Some(framebuffer)) }; + unsafe { + gl.framebuffer_renderbuffer( + glow::READ_FRAMEBUFFER, + glow::COLOR_ATTACHMENT0, + glow::RENDERBUFFER, + Some(renderbuffer), + ) + }; + unsafe { gl.bind_renderbuffer(glow::RENDERBUFFER, None) }; + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, None) }; + + let mut swapchain = self.swapchain.write(); + *swapchain = Some(Swapchain { + surface, + wl_window, + renderbuffer, + framebuffer, + extent: config.extent, + format: config.format, + format_desc, + sample_type: wgt::TextureSampleType::Float { filterable: false }, + }); + + Ok(()) + } + + unsafe fn unconfigure(&self, device: &super::Device) { + if let Some((surface, wl_window)) = unsafe { self.unconfigure_impl(device) } { + self.egl + .instance + .destroy_surface(self.egl.display, surface) + .unwrap(); + if let Some(window) = wl_window { + wayland_sys::ffi_dispatch!( + wayland_sys::egl::wayland_egl_handle(), + wl_egl_window_destroy, + window, + ); + } + } + } + + unsafe fn acquire_texture( + &self, + _timeout_ms: Option, //TODO + _fence: &super::Fence, + ) -> Result, crate::SurfaceError> { + let swapchain = self.swapchain.read(); + let sc = swapchain.as_ref().ok_or(crate::SurfaceError::Other( + "Surface has no swap-chain configured", + ))?; + let texture = super::Texture { + inner: super::TextureInner::Renderbuffer { + raw: sc.renderbuffer, + }, + drop_guard: None, + array_layer_count: 1, + mip_level_count: 1, + format: sc.format, + format_desc: sc.format_desc.clone(), + copy_size: crate::CopyExtent { + width: sc.extent.width, + height: sc.extent.height, + depth: 1, + }, + }; + Ok(crate::AcquiredSurfaceTexture { + texture, + suboptimal: false, + }) + } + unsafe fn discard_texture(&self, _texture: super::Texture) {} +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/emscripten.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/emscripten.rs new file mode 100644 index 00000000..88cc0bec --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/emscripten.rs @@ -0,0 +1,26 @@ +extern "C" { + /// returns 1 if success. 0 if failure. extension name must be null terminated + fn emscripten_webgl_enable_extension( + context: core::ffi::c_int, + extension: *const core::ffi::c_char, + ) -> core::ffi::c_int; + fn emscripten_webgl_get_current_context() -> core::ffi::c_int; +} +/// Webgl requires extensions to be enabled before using them. +/// This function can be used to enable webgl extension on emscripten target. +/// +/// returns true on success +/// +/// # Safety +/// +/// - opengl context MUST BE current +/// - extension_name_null_terminated argument must be a valid string with null terminator. +/// - extension must be present. check `glow_context.supported_extensions()` +pub unsafe fn enable_extension(extension_name_null_terminated: &str) -> bool { + unsafe { + emscripten_webgl_enable_extension( + emscripten_webgl_get_current_context(), + extension_name_null_terminated.as_ptr().cast(), + ) == 1 + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/fence.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/fence.rs new file mode 100644 index 00000000..d5cd0ec4 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/fence.rs @@ -0,0 +1,168 @@ +use alloc::vec::Vec; +use core::sync::atomic::Ordering; + +use glow::HasContext; + +use crate::AtomicFenceValue; + +#[derive(Debug, Copy, Clone)] +struct GLFence { + sync: glow::Fence, + value: crate::FenceValue, +} + +#[derive(Debug)] +pub struct Fence { + last_completed: AtomicFenceValue, + pending: Vec, + fence_behavior: wgt::GlFenceBehavior, +} + +impl crate::DynFence for Fence {} + +#[cfg(send_sync)] +unsafe impl Send for Fence {} +#[cfg(send_sync)] +unsafe impl Sync for Fence {} + +impl Fence { + pub fn new(options: &wgt::GlBackendOptions) -> Self { + Self { + last_completed: AtomicFenceValue::new(0), + pending: Vec::new(), + fence_behavior: options.fence_behavior, + } + } + + pub fn signal( + &mut self, + gl: &glow::Context, + value: crate::FenceValue, + ) -> Result<(), crate::DeviceError> { + if self.fence_behavior.is_auto_finish() { + *self.last_completed.get_mut() = value; + return Ok(()); + } + + let sync = unsafe { gl.fence_sync(glow::SYNC_GPU_COMMANDS_COMPLETE, 0) } + .map_err(|_| crate::DeviceError::OutOfMemory)?; + self.pending.push(GLFence { sync, value }); + + Ok(()) + } + + pub fn satisfied(&self, value: crate::FenceValue) -> bool { + self.last_completed.load(Ordering::Acquire) >= value + } + + pub fn get_latest(&self, gl: &glow::Context) -> crate::FenceValue { + let mut max_value = self.last_completed.load(Ordering::Acquire); + + if self.fence_behavior.is_auto_finish() { + return max_value; + } + + for gl_fence in self.pending.iter() { + if gl_fence.value <= max_value { + // We already know this was good, no need to check again + continue; + } + let status = unsafe { gl.get_sync_status(gl_fence.sync) }; + if status == glow::SIGNALED { + max_value = gl_fence.value; + } else { + // Anything after the first unsignalled is guaranteed to also be unsignalled + break; + } + } + + // Track the latest value, to save ourselves some querying later + self.last_completed.fetch_max(max_value, Ordering::AcqRel); + + max_value + } + + pub fn maintain(&mut self, gl: &glow::Context) { + if self.fence_behavior.is_auto_finish() { + return; + } + + let latest = self.get_latest(gl); + for &gl_fence in self.pending.iter() { + if gl_fence.value <= latest { + unsafe { + gl.delete_sync(gl_fence.sync); + } + } + } + self.pending.retain(|&gl_fence| gl_fence.value > latest); + } + + pub fn wait( + &self, + gl: &glow::Context, + wait_value: crate::FenceValue, + timeout_ns: u32, + ) -> Result { + let last_completed = self.last_completed.load(Ordering::Acquire); + + if self.fence_behavior.is_auto_finish() { + return Ok(last_completed >= wait_value); + } + + // We already know this fence has been signalled to that value. Return signalled. + if last_completed >= wait_value { + return Ok(true); + } + + // Find a matching fence + let gl_fence = self + .pending + .iter() + // Greater or equal as an abundance of caution, but there should be one fence per value + .find(|gl_fence| gl_fence.value >= wait_value); + + let Some(gl_fence) = gl_fence else { + log::warn!("Tried to wait for {wait_value} but that value has not been signalled yet"); + return Ok(false); + }; + + // We should have found a fence with the exact value. + debug_assert_eq!(gl_fence.value, wait_value); + + let status = unsafe { + gl.client_wait_sync( + gl_fence.sync, + glow::SYNC_FLUSH_COMMANDS_BIT, + timeout_ns.min(i32::MAX as u32) as i32, + ) + }; + + let signalled = match status { + glow::ALREADY_SIGNALED | glow::CONDITION_SATISFIED => true, + glow::TIMEOUT_EXPIRED | glow::WAIT_FAILED => false, + _ => { + log::warn!("Unexpected result from client_wait_sync: {status}"); + false + } + }; + + if signalled { + self.last_completed.fetch_max(wait_value, Ordering::AcqRel); + } + + Ok(signalled) + } + + pub fn destroy(self, gl: &glow::Context) { + if self.fence_behavior.is_auto_finish() { + return; + } + + for gl_fence in self.pending { + unsafe { + gl.delete_sync(gl_fence.sync); + } + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/mod.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/mod.rs new file mode 100644 index 00000000..6f8c865b --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/mod.rs @@ -0,0 +1,1140 @@ +/*! +# OpenGL ES3 API (aka GLES3). + +Designed to work on Linux and Android, with context provided by EGL. + +## Texture views + +GLES3 doesn't really have separate texture view objects. We have to remember the +original texture and the sub-range into it. Problem is, however, that there is +no way to expose a subset of array layers or mip levels of a sampled texture. + +## Binding model + +Binding model is very different from WebGPU, especially with regards to samplers. +GLES3 has sampler objects, but they aren't separately bindable to the shaders. +Each sampled texture is exposed to the shader as a combined texture-sampler binding. + +When building the pipeline layout, we linearize binding entries based on the groups +(uniform/storage buffers, uniform/storage textures), and record the mapping into +`BindGroupLayoutInfo`. +When a pipeline gets created, and we track all the texture-sampler associations +from the static use in the shader. +We only support at most one sampler used with each texture so far. The linear index +of this sampler is stored per texture slot in `SamplerBindMap` array. + +The texture-sampler pairs get potentially invalidated in 2 places: + - when a new pipeline is set, we update the linear indices of associated samplers + - when a new bind group is set, we update both the textures and the samplers + +We expect that the changes to sampler states between any 2 pipelines of the same layout +will be minimal, if any. + +## Vertex data + +Generally, vertex buffers are marked as dirty and lazily bound on draw. + +GLES3 doesn't support `first_instance` semantics. However, it's easy to support, +since we are forced to do late binding anyway. We just adjust the offsets +into the vertex data. + +### Old path + +In GLES-3.0 and WebGL2, vertex buffer layout is provided +together with the actual buffer binding. +We invalidate the attributes on the vertex buffer change, and re-bind them. + +### New path + +In GLES-3.1 and higher, the vertex buffer layout can be declared separately +from the vertex data itself. This mostly matches WebGPU, however there is a catch: +`stride` needs to be specified with the data, not as a part of the layout. + +To address this, we invalidate the vertex buffers based on: + - whether or not `first_instance` is used + - stride has changed + +## Handling of `base_vertex`, `first_instance`, and `first_vertex` + +Between indirect, the lack of `first_instance` semantics, and the availability of `gl_BaseInstance` +in shaders, getting buffers and builtins to work correctly is a bit tricky. + +We never emulate `base_vertex` and gl_VertexID behaves as `@builtin(vertex_index)` does, so we +never need to do anything about that. + +### GL 4.2+ with ARB_shader_draw_parameters + +- `@builtin(instance_index)` translates to `gl_InstanceID + gl_BaseInstance` +- We bind instance buffers without any offset emulation. +- We advertise support for the `INDIRECT_FIRST_INSTANCE` feature. + +While we can theoretically have a card with 4.2+ support but without ARB_shader_draw_parameters, +we don't bother with that combination. + +### GLES & GL 4.1 + +- `@builtin(instance_index)` translates to `gl_InstanceID + naga_vs_first_instance` +- We bind instance buffers with offset emulation. +- We _do not_ advertise support for `INDIRECT_FIRST_INSTANCE` and cpu-side pretend the `first_instance` is 0 on indirect calls. + +*/ + +///cbindgen:ignore +#[cfg(not(any(windows, webgl)))] +mod egl; +#[cfg(Emscripten)] +mod emscripten; +#[cfg(webgl)] +mod web; +#[cfg(windows)] +mod wgl; + +mod adapter; +mod command; +mod conv; +mod device; +mod fence; +mod queue; + +pub use fence::Fence; + +#[cfg(not(any(windows, webgl)))] +pub use self::egl::{AdapterContext, AdapterContextLock}; +#[cfg(not(any(windows, webgl)))] +pub use self::egl::{Instance, Surface}; + +#[cfg(webgl)] +pub use self::web::AdapterContext; +#[cfg(webgl)] +pub use self::web::{Instance, Surface}; + +#[cfg(windows)] +use self::wgl::AdapterContext; +#[cfg(windows)] +pub use self::wgl::{Instance, Surface}; + +use alloc::{boxed::Box, string::String, string::ToString as _, sync::Arc, vec::Vec}; +use core::{ + fmt, + ops::Range, + sync::atomic::{AtomicU32, AtomicU8}, +}; +use parking_lot::Mutex; + +use arrayvec::ArrayVec; +use glow::HasContext; +use naga::FastHashMap; + +use crate::{CopyExtent, TextureDescriptor}; + +#[derive(Clone, Debug)] +pub struct Api; + +//Note: we can support more samplers if not every one of them is used at a time, +// but it probably doesn't worth it. +const MAX_TEXTURE_SLOTS: usize = 16; +const MAX_SAMPLERS: usize = 16; +const MAX_VERTEX_ATTRIBUTES: usize = 16; +const ZERO_BUFFER_SIZE: usize = 256 << 10; +const MAX_IMMEDIATES: usize = 64; +// We have to account for each immediate data may need to be set for every shader. +const MAX_IMMEDIATES_COMMANDS: usize = MAX_IMMEDIATES * crate::MAX_CONCURRENT_SHADER_STAGES; + +impl crate::Api for Api { + const VARIANT: wgt::Backend = wgt::Backend::Gl; + + type Instance = Instance; + type Surface = Surface; + type Adapter = Adapter; + type Device = Device; + + type Queue = Queue; + type CommandEncoder = CommandEncoder; + type CommandBuffer = CommandBuffer; + + type Buffer = Buffer; + type Texture = Texture; + type SurfaceTexture = Texture; + type TextureView = TextureView; + type Sampler = Sampler; + type QuerySet = QuerySet; + type Fence = Fence; + type AccelerationStructure = AccelerationStructure; + type PipelineCache = PipelineCache; + + type BindGroupLayout = BindGroupLayout; + type BindGroup = BindGroup; + type PipelineLayout = PipelineLayout; + type ShaderModule = ShaderModule; + type RenderPipeline = RenderPipeline; + type ComputePipeline = ComputePipeline; +} + +crate::impl_dyn_resource!( + Adapter, + AccelerationStructure, + BindGroup, + BindGroupLayout, + Buffer, + CommandBuffer, + CommandEncoder, + ComputePipeline, + Device, + Fence, + Instance, + PipelineCache, + PipelineLayout, + QuerySet, + Queue, + RenderPipeline, + Sampler, + ShaderModule, + Surface, + Texture, + TextureView +); + +bitflags::bitflags! { + /// Flags that affect internal code paths but do not + /// change the exposed feature set. + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + struct PrivateCapabilities: u32 { + /// Indicates support for `glBufferStorage` allocation. + const BUFFER_ALLOCATION = 1 << 0; + /// Support explicit layouts in shader. + const SHADER_BINDING_LAYOUT = 1 << 1; + /// Support extended shadow sampling instructions. + const SHADER_TEXTURE_SHADOW_LOD = 1 << 2; + /// Support memory barriers. + const MEMORY_BARRIERS = 1 << 3; + /// Vertex buffer layouts separate from the data. + const VERTEX_BUFFER_LAYOUT = 1 << 4; + /// Indicates that buffers used as `GL_ELEMENT_ARRAY_BUFFER` may be created / initialized / used + /// as other targets, if not present they must not be mixed with other targets. + const INDEX_BUFFER_ROLE_CHANGE = 1 << 5; + /// Supports `glGetBufferSubData` + const GET_BUFFER_SUB_DATA = 1 << 7; + /// Supports `f16` color buffers + const COLOR_BUFFER_HALF_FLOAT = 1 << 8; + /// Supports `f11/f10` and `f32` color buffers + const COLOR_BUFFER_FLOAT = 1 << 9; + /// Supports query buffer objects. + const QUERY_BUFFERS = 1 << 11; + /// Supports 64 bit queries via `glGetQueryObjectui64v` + const QUERY_64BIT = 1 << 12; + /// Supports `glTexStorage2D`, etc. + const TEXTURE_STORAGE = 1 << 13; + /// Supports `push_debug_group`, `pop_debug_group` and `debug_message_insert`. + const DEBUG_FNS = 1 << 14; + /// Supports framebuffer invalidation. + const INVALIDATE_FRAMEBUFFER = 1 << 15; + /// Indicates support for `glDrawElementsInstancedBaseVertexBaseInstance` and `ARB_shader_draw_parameters` + /// + /// When this is true, instance offset emulation via vertex buffer rebinding and a shader uniform will be disabled. + const FULLY_FEATURED_INSTANCING = 1 << 16; + /// Supports direct multisampled rendering to a texture without needing a resolve texture. + const MULTISAMPLED_RENDER_TO_TEXTURE = 1 << 17; + } +} + +bitflags::bitflags! { + /// Flags that indicate necessary workarounds for specific devices or driver bugs + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + struct Workarounds: u32 { + // Needs workaround for Intel Mesa bug: + // https://gitlab.freedesktop.org/mesa/mesa/-/issues/2565. + // + // This comment + // (https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4972/diffs?diff_id=75888#22f5d1004713c9bbf857988c7efb81631ab88f99_323_327) + // seems to indicate all skylake models are effected. + const MESA_I915_SRGB_SHADER_CLEAR = 1 << 0; + /// Buffer map must emulated because it is not supported natively + const EMULATE_BUFFER_MAP = 1 << 1; + } +} + +type BindTarget = u32; + +#[derive(Debug, Default, Clone, Copy)] +enum VertexAttribKind { + #[default] + Float, // glVertexAttribPointer + Integer, // glVertexAttribIPointer + //Double, // glVertexAttribLPointer +} + +#[derive(Clone, Debug)] +pub struct TextureFormatDesc { + pub internal: u32, + pub external: u32, + pub data_type: u32, +} + +struct AdapterShared { + context: AdapterContext, + private_caps: PrivateCapabilities, + features: wgt::Features, + limits: wgt::Limits, + workarounds: Workarounds, + options: wgt::GlBackendOptions, + shading_language_version: naga::back::glsl::Version, + next_shader_id: AtomicU32, + program_cache: Mutex, + es: bool, + + /// Result of `gl.get_parameter_i32(glow::MAX_SAMPLES)`. + /// Cached here so it doesn't need to be queried every time texture format capabilities are requested. + /// (this has been shown to be a significant enough overhead) + max_msaa_samples: i32, +} + +pub struct Adapter { + shared: Arc, +} + +pub struct Device { + shared: Arc, + main_vao: glow::VertexArray, + #[cfg(all(native, feature = "renderdoc"))] + render_doc: crate::auxil::renderdoc::RenderDoc, + counters: Arc, +} + +impl Drop for Device { + fn drop(&mut self) { + let gl = &self.shared.context.lock(); + unsafe { gl.delete_vertex_array(self.main_vao) }; + } +} + +pub struct ShaderClearProgram { + pub program: glow::Program, + pub color_uniform_location: glow::UniformLocation, +} + +pub struct Queue { + shared: Arc, + features: wgt::Features, + draw_fbo: glow::Framebuffer, + copy_fbo: glow::Framebuffer, + /// Shader program used to clear the screen for [`Workarounds::MESA_I915_SRGB_SHADER_CLEAR`] + /// devices. + shader_clear_program: Option, + /// Keep a reasonably large buffer filled with zeroes, so that we can implement `ClearBuffer` of + /// zeroes by copying from it. + zero_buffer: glow::Buffer, + temp_query_results: Mutex>, + draw_buffer_count: AtomicU8, + current_index_buffer: Mutex>, +} + +impl Drop for Queue { + fn drop(&mut self) { + let gl = &self.shared.context.lock(); + unsafe { gl.delete_framebuffer(self.draw_fbo) }; + unsafe { gl.delete_framebuffer(self.copy_fbo) }; + unsafe { gl.delete_buffer(self.zero_buffer) }; + } +} + +#[derive(Clone, Debug)] +pub struct Buffer { + raw: Option, + target: BindTarget, + size: wgt::BufferAddress, + /// Flags to use within calls to [`Device::map_buffer`](crate::Device::map_buffer). + map_flags: u32, + data: Option>>>, + offset_of_current_mapping: Arc>, +} + +#[cfg(send_sync)] +unsafe impl Sync for Buffer {} +#[cfg(send_sync)] +unsafe impl Send for Buffer {} + +impl crate::DynBuffer for Buffer {} + +#[derive(Clone, Debug)] +pub enum TextureInner { + Renderbuffer { + raw: glow::Renderbuffer, + }, + DefaultRenderbuffer, + Texture { + raw: glow::Texture, + target: BindTarget, + }, + #[cfg(webgl)] + /// Render to a `WebGLFramebuffer` + /// + /// This is a web feature + ExternalFramebuffer { + inner: web_sys::WebGlFramebuffer, + }, + #[cfg(native)] + /// Render to a `glow::NativeFramebuffer` + /// Useful when the framebuffer to draw to + /// has a non-zero framebuffer ID + /// + /// This is a native feature + ExternalNativeFramebuffer { + inner: glow::NativeFramebuffer, + }, +} + +#[cfg(send_sync)] +unsafe impl Sync for TextureInner {} +#[cfg(send_sync)] +unsafe impl Send for TextureInner {} + +impl TextureInner { + fn as_native(&self) -> (glow::Texture, BindTarget) { + match *self { + Self::Renderbuffer { .. } | Self::DefaultRenderbuffer => { + panic!("Unexpected renderbuffer"); + } + Self::Texture { raw, target } => (raw, target), + #[cfg(webgl)] + Self::ExternalFramebuffer { .. } => panic!("Unexpected external framebuffer"), + #[cfg(native)] + Self::ExternalNativeFramebuffer { .. } => panic!("unexpected external framebuffer"), + } + } +} + +#[derive(Debug)] +pub struct Texture { + pub inner: TextureInner, + pub mip_level_count: u32, + pub array_layer_count: u32, + pub format: wgt::TextureFormat, + pub format_desc: TextureFormatDesc, + pub copy_size: CopyExtent, + + // The `drop_guard` field must be the last field of this struct so it is dropped last. + // Do not add new fields after it. + pub drop_guard: Option, +} + +impl crate::DynTexture for Texture {} +impl crate::DynSurfaceTexture for Texture {} + +impl core::borrow::Borrow for Texture { + fn borrow(&self) -> &dyn crate::DynTexture { + self + } +} + +impl Texture { + pub fn default_framebuffer(format: wgt::TextureFormat) -> Self { + Self { + inner: TextureInner::DefaultRenderbuffer, + drop_guard: None, + mip_level_count: 1, + array_layer_count: 1, + format, + format_desc: TextureFormatDesc { + internal: 0, + external: 0, + data_type: 0, + }, + copy_size: CopyExtent { + width: 0, + height: 0, + depth: 0, + }, + } + } + + /// Returns the `target`, whether the image is 3d and whether the image is a cubemap. + fn get_info_from_desc(desc: &TextureDescriptor) -> u32 { + match desc.dimension { + // WebGL (1 and 2) as well as some GLES versions do not have 1D textures, so we are + // doing `TEXTURE_2D` instead + wgt::TextureDimension::D1 => glow::TEXTURE_2D, + wgt::TextureDimension::D2 => { + // HACK: detect a cube map; forces cube compatible textures to be cube textures + match (desc.is_cube_compatible(), desc.size.depth_or_array_layers) { + (false, 1) => glow::TEXTURE_2D, + (false, _) => glow::TEXTURE_2D_ARRAY, + (true, 6) => glow::TEXTURE_CUBE_MAP, + (true, _) => glow::TEXTURE_CUBE_MAP_ARRAY, + } + } + wgt::TextureDimension::D3 => glow::TEXTURE_3D, + } + } + + /// More information can be found in issues #1614 and #1574 + fn log_failing_target_heuristics(view_dimension: wgt::TextureViewDimension, target: u32) { + let expected_target = match view_dimension { + wgt::TextureViewDimension::D1 => glow::TEXTURE_2D, + wgt::TextureViewDimension::D2 => glow::TEXTURE_2D, + wgt::TextureViewDimension::D2Array => glow::TEXTURE_2D_ARRAY, + wgt::TextureViewDimension::Cube => glow::TEXTURE_CUBE_MAP, + wgt::TextureViewDimension::CubeArray => glow::TEXTURE_CUBE_MAP_ARRAY, + wgt::TextureViewDimension::D3 => glow::TEXTURE_3D, + }; + + if expected_target == target { + return; + } + + let buffer; + let got = match target { + glow::TEXTURE_2D => "D2", + glow::TEXTURE_2D_ARRAY => "D2Array", + glow::TEXTURE_CUBE_MAP => "Cube", + glow::TEXTURE_CUBE_MAP_ARRAY => "CubeArray", + glow::TEXTURE_3D => "D3", + target => { + buffer = target.to_string(); + &buffer + } + }; + + log::error!( + concat!( + "wgpu-hal heuristics assumed that ", + "the view dimension will be equal to `{}` rather than `{:?}`.\n", + "`D2` textures with ", + "`depth_or_array_layers == 1` ", + "are assumed to have view dimension `D2`\n", + "`D2` textures with ", + "`depth_or_array_layers > 1` ", + "are assumed to have view dimension `D2Array`\n", + "`D2` textures with ", + "`depth_or_array_layers == 6` ", + "are assumed to have view dimension `Cube`\n", + "`D2` textures with ", + "`depth_or_array_layers > 6 && depth_or_array_layers % 6 == 0` ", + "are assumed to have view dimension `CubeArray`\n", + ), + got, + view_dimension, + ); + } +} + +#[derive(Clone, Debug)] +pub struct TextureView { + inner: TextureInner, + aspects: crate::FormatAspects, + mip_levels: Range, + array_layers: Range, + format: wgt::TextureFormat, +} + +impl crate::DynTextureView for TextureView {} + +#[derive(Debug)] +pub struct Sampler { + raw: glow::Sampler, +} + +impl crate::DynSampler for Sampler {} + +#[derive(Debug)] +pub struct BindGroupLayout { + entries: Arc<[wgt::BindGroupLayoutEntry]>, +} + +impl crate::DynBindGroupLayout for BindGroupLayout {} + +#[derive(Debug)] +struct BindGroupLayoutInfo { + entries: Arc<[wgt::BindGroupLayoutEntry]>, + /// Mapping of resources, indexed by `binding`, into the whole layout space. + /// For texture resources, the value is the texture slot index. + /// For sampler resources, the value is the index of the sampler in the whole layout. + /// For buffers, the value is the uniform or storage slot index. + /// For unused bindings, the value is `!0` + binding_to_slot: Box<[u8]>, +} + +#[derive(Debug)] +pub struct PipelineLayout { + group_infos: Box<[Option]>, + naga_options: naga::back::glsl::Options, +} + +impl crate::DynPipelineLayout for PipelineLayout {} + +impl PipelineLayout { + /// # Panics + /// If the pipeline layout does not contain a bind group layout used by + /// the resource binding. + fn get_slot(&self, br: &naga::ResourceBinding) -> u8 { + let group_info = self.group_infos[br.group as usize].as_ref().unwrap(); + group_info.binding_to_slot[br.binding as usize] + } +} + +#[derive(Debug)] +enum BindingRegister { + UniformBuffers, + StorageBuffers, + Textures, + Images, +} + +#[derive(Debug)] +enum RawBinding { + Buffer { + raw: glow::Buffer, + offset: i32, + size: i32, + }, + Texture { + raw: glow::Texture, + target: BindTarget, + aspects: crate::FormatAspects, + mip_levels: Range, + //TODO: array layers + }, + Image(ImageBinding), + Sampler(glow::Sampler), +} + +#[derive(Debug)] +pub struct BindGroup { + contents: Box<[RawBinding]>, +} + +impl crate::DynBindGroup for BindGroup {} + +type ShaderId = u32; + +#[derive(Debug)] +pub struct ShaderModule { + source: crate::NagaShader, + label: Option, + id: ShaderId, +} + +impl crate::DynShaderModule for ShaderModule {} + +#[derive(Clone, Debug, Default)] +struct VertexFormatDesc { + element_count: i32, + element_format: u32, + attrib_kind: VertexAttribKind, +} + +#[derive(Clone, Debug, Default)] +struct AttributeDesc { + location: u32, + offset: u32, + buffer_index: u32, + format_desc: VertexFormatDesc, +} + +#[derive(Clone, Debug)] +struct BufferBinding { + raw: glow::Buffer, + offset: wgt::BufferAddress, +} + +#[derive(Clone, Debug)] +struct ImageBinding { + raw: glow::Texture, + mip_level: u32, + array_layer: Option, + access: u32, + format: u32, +} + +#[derive(Clone, Debug, Default, PartialEq)] +struct VertexBufferDesc { + step: wgt::VertexStepMode, + stride: u32, +} + +#[derive(Clone, Debug)] +struct ImmediateDesc { + location: glow::UniformLocation, + ty: naga::TypeInner, + offset: u32, + size_bytes: u32, +} + +#[cfg(send_sync)] +unsafe impl Sync for ImmediateDesc {} +#[cfg(send_sync)] +unsafe impl Send for ImmediateDesc {} + +/// For each texture in the pipeline layout, store the index of the only +/// sampler (in this layout) that the texture is used with. +type SamplerBindMap = [Option; MAX_TEXTURE_SLOTS]; + +#[derive(Debug)] +struct PipelineInner { + program: glow::Program, + sampler_map: SamplerBindMap, + first_instance_location: Option, + immediates_descs: ArrayVec, + clip_distance_count: u32, +} + +#[derive(Clone, Debug)] +struct DepthState { + function: u32, + mask: bool, +} + +#[derive(Clone, Debug, PartialEq)] +struct BlendComponent { + src: u32, + dst: u32, + equation: u32, +} + +#[derive(Clone, Debug, PartialEq)] +struct BlendDesc { + alpha: BlendComponent, + color: BlendComponent, +} + +#[derive(Clone, Debug, Default, PartialEq)] +struct ColorTargetDesc { + mask: wgt::ColorWrites, + blend: Option, +} + +#[derive(PartialEq, Eq, Hash)] +struct ProgramStage { + naga_stage: naga::ShaderStage, + shader_id: ShaderId, + entry_point: String, + zero_initialize_workgroup_memory: bool, + constant_hash: Vec, +} + +#[derive(PartialEq, Eq, Hash)] +struct ProgramCacheKey { + stages: ArrayVec, + group_to_binding_to_slot: Box<[Option>]>, +} + +type ProgramCache = FastHashMap, crate::PipelineError>>; + +#[derive(Debug)] +pub struct RenderPipeline { + inner: Arc, + primitive: wgt::PrimitiveState, + vertex_buffers: Box<[VertexBufferDesc]>, + vertex_attributes: Box<[AttributeDesc]>, + color_targets: Box<[ColorTargetDesc]>, + depth: Option, + depth_bias: wgt::DepthBiasState, + stencil: Option, + alpha_to_coverage_enabled: bool, +} + +impl crate::DynRenderPipeline for RenderPipeline {} + +#[cfg(send_sync)] +unsafe impl Sync for RenderPipeline {} +#[cfg(send_sync)] +unsafe impl Send for RenderPipeline {} + +#[derive(Debug)] +pub struct ComputePipeline { + inner: Arc, +} + +impl crate::DynComputePipeline for ComputePipeline {} + +#[cfg(send_sync)] +unsafe impl Sync for ComputePipeline {} +#[cfg(send_sync)] +unsafe impl Send for ComputePipeline {} + +#[derive(Debug)] +pub struct QuerySet { + queries: Box<[glow::Query]>, + target: BindTarget, +} + +impl crate::DynQuerySet for QuerySet {} + +#[derive(Debug)] +pub struct AccelerationStructure; + +impl crate::DynAccelerationStructure for AccelerationStructure {} + +#[derive(Debug)] +pub struct PipelineCache; + +impl crate::DynPipelineCache for PipelineCache {} + +#[derive(Clone, Debug, PartialEq)] +struct StencilOps { + pass: u32, + fail: u32, + depth_fail: u32, +} + +impl Default for StencilOps { + fn default() -> Self { + Self { + pass: glow::KEEP, + fail: glow::KEEP, + depth_fail: glow::KEEP, + } + } +} + +#[derive(Clone, Debug, PartialEq)] +struct StencilSide { + function: u32, + mask_read: u32, + mask_write: u32, + reference: u32, + ops: StencilOps, +} + +impl Default for StencilSide { + fn default() -> Self { + Self { + function: glow::ALWAYS, + mask_read: 0xFF, + mask_write: 0xFF, + reference: 0, + ops: StencilOps::default(), + } + } +} + +#[derive(Debug, Clone, Default)] +struct StencilState { + front: StencilSide, + back: StencilSide, +} + +#[derive(Clone, Debug, Default, PartialEq)] +struct PrimitiveState { + front_face: u32, + cull_face: u32, + unclipped_depth: bool, + polygon_mode: u32, +} + +type InvalidatedAttachments = ArrayVec; + +#[derive(Debug)] +enum Command { + Draw { + topology: u32, + first_vertex: u32, + vertex_count: u32, + first_instance: u32, + instance_count: u32, + first_instance_location: Option, + }, + DrawIndexed { + topology: u32, + index_type: u32, + index_count: u32, + index_offset: wgt::BufferAddress, + base_vertex: i32, + first_instance: u32, + instance_count: u32, + first_instance_location: Option, + }, + DrawIndirect { + topology: u32, + indirect_buf: glow::Buffer, + indirect_offset: wgt::BufferAddress, + first_instance_location: Option, + }, + DrawIndexedIndirect { + topology: u32, + index_type: u32, + indirect_buf: glow::Buffer, + indirect_offset: wgt::BufferAddress, + first_instance_location: Option, + }, + Dispatch([u32; 3]), + DispatchIndirect { + indirect_buf: glow::Buffer, + indirect_offset: wgt::BufferAddress, + }, + ClearBuffer { + dst: Buffer, + dst_target: BindTarget, + range: crate::MemoryRange, + }, + CopyBufferToBuffer { + src: Buffer, + src_target: BindTarget, + dst: Buffer, + dst_target: BindTarget, + copy: crate::BufferCopy, + }, + #[cfg(webgl)] + CopyExternalImageToTexture { + src: wgt::CopyExternalImageSourceInfo, + dst: glow::Texture, + dst_target: BindTarget, + dst_format: wgt::TextureFormat, + dst_premultiplication: bool, + copy: crate::TextureCopy, + }, + CopyTextureToTexture { + src: glow::Texture, + src_target: BindTarget, + dst: glow::Texture, + dst_target: BindTarget, + copy: crate::TextureCopy, + }, + CopyBufferToTexture { + src: Buffer, + #[allow(unused)] + src_target: BindTarget, + dst: glow::Texture, + dst_target: BindTarget, + dst_format: wgt::TextureFormat, + copy: crate::BufferTextureCopy, + }, + CopyTextureToBuffer { + src: glow::Texture, + src_target: BindTarget, + src_format: wgt::TextureFormat, + dst: Buffer, + #[allow(unused)] + dst_target: BindTarget, + copy: crate::BufferTextureCopy, + }, + SetIndexBuffer(glow::Buffer), + BeginQuery(glow::Query, BindTarget), + EndQuery(BindTarget), + TimestampQuery(glow::Query), + CopyQueryResults { + query_range: Range, + dst: Buffer, + dst_target: BindTarget, + dst_offset: wgt::BufferAddress, + }, + ResetFramebuffer { + is_default: bool, + }, + BindAttachment { + attachment: u32, + view: TextureView, + depth_slice: Option, + sample_count: u32, + }, + ResolveAttachment { + attachment: u32, + dst: TextureView, + size: wgt::Extent3d, + }, + InvalidateAttachments(InvalidatedAttachments), + SetDrawColorBuffers(u8), + ClearColorF { + draw_buffer: u32, + color: [f32; 4], + is_srgb: bool, + }, + ClearColorU(u32, [u32; 4]), + ClearColorI(u32, [i32; 4]), + ClearDepth(f32), + ClearStencil(u32), + // Clearing both the depth and stencil buffer individually appears to + // result in the stencil buffer failing to clear, atleast in WebGL. + // It is also more efficient to emit a single command instead of two for + // this. + ClearDepthAndStencil(f32, u32), + BufferBarrier(glow::Buffer, wgt::BufferUses), + TextureBarrier(wgt::TextureUses), + SetViewport { + rect: crate::Rect, + depth: Range, + }, + SetScissor(crate::Rect), + SetStencilFunc { + face: u32, + function: u32, + reference: u32, + read_mask: u32, + }, + SetStencilOps { + face: u32, + write_mask: u32, + ops: StencilOps, + }, + SetDepth(DepthState), + SetDepthBias(wgt::DepthBiasState), + ConfigureDepthStencil(crate::FormatAspects), + SetAlphaToCoverage(bool), + SetVertexAttribute { + buffer: Option, + buffer_desc: VertexBufferDesc, + attribute_desc: AttributeDesc, + }, + UnsetVertexAttribute(u32), + SetVertexBuffer { + index: u32, + buffer: BufferBinding, + buffer_desc: VertexBufferDesc, + }, + SetProgram(glow::Program), + SetPrimitive(PrimitiveState), + SetBlendConstant([f32; 4]), + SetColorTarget { + draw_buffer_index: Option, + desc: ColorTargetDesc, + }, + BindBuffer { + target: BindTarget, + slot: u32, + buffer: glow::Buffer, + offset: i32, + size: i32, + }, + BindSampler(u32, Option), + BindTexture { + slot: u32, + texture: glow::Texture, + target: BindTarget, + aspects: crate::FormatAspects, + mip_levels: Range, + }, + BindImage { + slot: u32, + binding: ImageBinding, + }, + InsertDebugMarker(Range), + PushDebugGroup(Range), + PopDebugGroup, + SetImmediates { + uniform: ImmediateDesc, + /// Offset from the start of the `data_bytes` + offset: u32, + }, + SetClipDistances { + old_count: u32, + new_count: u32, + }, +} + +#[derive(Default)] +pub struct CommandBuffer { + label: Option, + commands: Vec, + data_bytes: Vec, + queries: Vec, +} + +impl crate::DynCommandBuffer for CommandBuffer {} + +impl fmt::Debug for CommandBuffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut builder = f.debug_struct("CommandBuffer"); + if let Some(ref label) = self.label { + builder.field("label", label); + } + builder.finish() + } +} + +#[cfg(send_sync)] +unsafe impl Sync for CommandBuffer {} +#[cfg(send_sync)] +unsafe impl Send for CommandBuffer {} + +//TODO: we would have something like `Arc` +// here and in the command buffers. So that everything grows +// inside the encoder and stays there until `reset_all`. + +pub struct CommandEncoder { + cmd_buffer: CommandBuffer, + state: command::State, + private_caps: PrivateCapabilities, + counters: Arc, +} + +impl fmt::Debug for CommandEncoder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CommandEncoder") + .field("cmd_buffer", &self.cmd_buffer) + .finish() + } +} + +#[cfg(send_sync)] +unsafe impl Sync for CommandEncoder {} +#[cfg(send_sync)] +unsafe impl Send for CommandEncoder {} + +#[cfg(not(webgl))] +fn gl_debug_message_callback(source: u32, gltype: u32, id: u32, severity: u32, message: &str) { + let source_str = match source { + glow::DEBUG_SOURCE_API => "API", + glow::DEBUG_SOURCE_WINDOW_SYSTEM => "Window System", + glow::DEBUG_SOURCE_SHADER_COMPILER => "ShaderCompiler", + glow::DEBUG_SOURCE_THIRD_PARTY => "Third Party", + glow::DEBUG_SOURCE_APPLICATION => "Application", + glow::DEBUG_SOURCE_OTHER => "Other", + _ => unreachable!(), + }; + + let log_severity = match severity { + glow::DEBUG_SEVERITY_HIGH => log::Level::Error, + glow::DEBUG_SEVERITY_MEDIUM => log::Level::Warn, + glow::DEBUG_SEVERITY_LOW => log::Level::Debug, + glow::DEBUG_SEVERITY_NOTIFICATION => log::Level::Trace, + _ => unreachable!(), + }; + + let type_str = match gltype { + glow::DEBUG_TYPE_DEPRECATED_BEHAVIOR => "Deprecated Behavior", + glow::DEBUG_TYPE_ERROR => "Error", + glow::DEBUG_TYPE_MARKER => "Marker", + glow::DEBUG_TYPE_OTHER => "Other", + glow::DEBUG_TYPE_PERFORMANCE => "Performance", + glow::DEBUG_TYPE_POP_GROUP => "Pop Group", + glow::DEBUG_TYPE_PORTABILITY => "Portability", + glow::DEBUG_TYPE_PUSH_GROUP => "Push Group", + glow::DEBUG_TYPE_UNDEFINED_BEHAVIOR => "Undefined Behavior", + _ => unreachable!(), + }; + + let _ = std::panic::catch_unwind(|| { + log::log!( + log_severity, + "GLES: [{source_str}/{type_str}] ID {id} : {message}" + ); + }); + + #[cfg(feature = "validation_canary")] + if cfg!(debug_assertions) && log_severity == log::Level::Error { + // Set canary and continue + crate::VALIDATION_CANARY.add(message.to_string()); + } +} + +// If we are using `std`, then use `Mutex` to provide `Send` and `Sync` +cfg_if::cfg_if! { + if #[cfg(gles_with_std)] { + type MaybeMutex = std::sync::Mutex; + + fn lock(mutex: &MaybeMutex) -> std::sync::MutexGuard<'_, T> { + mutex.lock().unwrap() + } + } else { + // It should be impossible for any build configuration to trigger this error + // It is intended only as a guard against changes elsewhere causing the use of + // `RefCell` here to become unsound. + #[cfg(all(send_sync, not(feature = "fragile-send-sync-non-atomic-wasm")))] + compile_error!("cannot provide non-fragile Send+Sync without std"); + + type MaybeMutex = core::cell::RefCell; + + fn lock(mutex: &MaybeMutex) -> core::cell::RefMut<'_, T> { + mutex.borrow_mut() + } + } +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/queue.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/queue.rs new file mode 100644 index 00000000..a1f09a5c --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/queue.rs @@ -0,0 +1,1963 @@ +use alloc::sync::Arc; +use alloc::vec; +use core::sync::atomic::Ordering; + +use arrayvec::ArrayVec; +use glow::HasContext; + +use super::{conv::is_layered_target, lock, Command as C, PrivateCapabilities}; + +const DEBUG_ID: u32 = 0; + +fn extract_marker<'a>(data: &'a [u8], range: &core::ops::Range) -> &'a str { + core::str::from_utf8(&data[range.start as usize..range.end as usize]).unwrap() +} + +fn to_debug_str(s: &str) -> &str { + // The spec mentions that if the length given to debug functions is negative, + // the implementation will access the ptr and look for a null that terminates + // the string but some implementations will try to access the ptr even if the + // length is 0. + if s.is_empty() { + "" + } else { + s + } +} + +fn get_2d_target(target: u32, array_layer: u32) -> u32 { + const CUBEMAP_FACES: [u32; 6] = [ + glow::TEXTURE_CUBE_MAP_POSITIVE_X, + glow::TEXTURE_CUBE_MAP_NEGATIVE_X, + glow::TEXTURE_CUBE_MAP_POSITIVE_Y, + glow::TEXTURE_CUBE_MAP_NEGATIVE_Y, + glow::TEXTURE_CUBE_MAP_POSITIVE_Z, + glow::TEXTURE_CUBE_MAP_NEGATIVE_Z, + ]; + + match target { + glow::TEXTURE_2D => target, + glow::TEXTURE_CUBE_MAP => CUBEMAP_FACES[array_layer as usize], + _ => unreachable!(), + } +} + +fn get_z_offset(target: u32, base: &crate::TextureCopyBase) -> u32 { + match target { + glow::TEXTURE_2D_ARRAY | glow::TEXTURE_CUBE_MAP_ARRAY => base.array_layer, + glow::TEXTURE_3D => base.origin.z, + _ => unreachable!(), + } +} + +impl super::Queue { + /// Performs a manual shader clear, used as a workaround for a clearing bug on mesa + unsafe fn perform_shader_clear(&self, gl: &glow::Context, draw_buffer: u32, color: [f32; 4]) { + let shader_clear = self + .shader_clear_program + .as_ref() + .expect("shader_clear_program should always be set if the workaround is enabled"); + unsafe { gl.use_program(Some(shader_clear.program)) }; + unsafe { + gl.uniform_4_f32( + Some(&shader_clear.color_uniform_location), + color[0], + color[1], + color[2], + color[3], + ) + }; + unsafe { gl.disable(glow::DEPTH_TEST) }; + unsafe { gl.disable(glow::STENCIL_TEST) }; + unsafe { gl.disable(glow::SCISSOR_TEST) }; + unsafe { gl.disable(glow::BLEND) }; + unsafe { gl.disable(glow::CULL_FACE) }; + unsafe { gl.draw_buffers(&[glow::COLOR_ATTACHMENT0 + draw_buffer]) }; + unsafe { gl.draw_arrays(glow::TRIANGLES, 0, 3) }; + + let draw_buffer_count = self.draw_buffer_count.load(Ordering::Relaxed); + if draw_buffer_count != 0 { + // Reset the draw buffers to what they were before the clear + let indices = (0..draw_buffer_count as u32) + .map(|i| glow::COLOR_ATTACHMENT0 + i) + .collect::>(); + unsafe { gl.draw_buffers(&indices) }; + } + } + + unsafe fn reset_state(&self, gl: &glow::Context) { + unsafe { gl.use_program(None) }; + unsafe { gl.bind_framebuffer(glow::FRAMEBUFFER, None) }; + unsafe { gl.disable(glow::DEPTH_TEST) }; + unsafe { gl.disable(glow::STENCIL_TEST) }; + unsafe { gl.disable(glow::SCISSOR_TEST) }; + unsafe { gl.disable(glow::BLEND) }; + unsafe { gl.disable(glow::CULL_FACE) }; + unsafe { gl.disable(glow::POLYGON_OFFSET_FILL) }; + unsafe { gl.disable(glow::SAMPLE_ALPHA_TO_COVERAGE) }; + if self.features.contains(wgt::Features::DEPTH_CLIP_CONTROL) { + unsafe { gl.disable(glow::DEPTH_CLAMP) }; + } + + unsafe { gl.bind_buffer(glow::ELEMENT_ARRAY_BUFFER, None) }; + let mut current_index_buffer = self.current_index_buffer.lock(); + *current_index_buffer = None; + } + + unsafe fn set_attachment( + &self, + gl: &glow::Context, + fbo_target: u32, + attachment: u32, + view: &super::TextureView, + depth_slice: Option, + sample_count: u32, + ) { + match view.inner { + super::TextureInner::Renderbuffer { raw } => { + unsafe { + gl.framebuffer_renderbuffer( + fbo_target, + attachment, + glow::RENDERBUFFER, + Some(raw), + ) + }; + } + super::TextureInner::DefaultRenderbuffer => panic!("Unexpected default RBO"), + super::TextureInner::Texture { raw, target } => { + let num_layers = view.array_layers.end - view.array_layers.start; + if num_layers > 1 { + #[cfg(webgl)] + unsafe { + gl.framebuffer_texture_multiview_ovr( + fbo_target, + attachment, + Some(raw), + view.mip_levels.start as i32, + view.array_layers.start as i32, + num_layers as i32, + ) + }; + } else if is_layered_target(target) { + let layer = if target == glow::TEXTURE_3D { + depth_slice.unwrap() as i32 + } else { + view.array_layers.start as i32 + }; + unsafe { + gl.framebuffer_texture_layer( + fbo_target, + attachment, + Some(raw), + view.mip_levels.start as i32, + layer, + ) + }; + } else { + unsafe { + assert_eq!(view.mip_levels.len(), 1); + if sample_count != 1 { + gl.framebuffer_texture_2d_multisample( + fbo_target, + attachment, + get_2d_target(target, view.array_layers.start), + Some(raw), + view.mip_levels.start as i32, + sample_count as i32, + ) + } else { + gl.framebuffer_texture_2d( + fbo_target, + attachment, + get_2d_target(target, view.array_layers.start), + Some(raw), + view.mip_levels.start as i32, + ) + } + }; + } + } + #[cfg(webgl)] + super::TextureInner::ExternalFramebuffer { ref inner } => unsafe { + gl.bind_external_framebuffer(glow::FRAMEBUFFER, inner); + }, + #[cfg(native)] + super::TextureInner::ExternalNativeFramebuffer { ref inner } => unsafe { + gl.bind_framebuffer(glow::FRAMEBUFFER, Some(*inner)); + }, + } + } + + unsafe fn process( + &self, + gl: &glow::Context, + command: &C, + #[cfg_attr(target_arch = "wasm32", allow(unused))] data_bytes: &[u8], + queries: &[glow::Query], + ) { + match *command { + C::Draw { + topology, + first_vertex, + vertex_count, + instance_count, + first_instance, + ref first_instance_location, + } => { + let supports_full_instancing = self + .shared + .private_caps + .contains(PrivateCapabilities::FULLY_FEATURED_INSTANCING); + + if supports_full_instancing { + unsafe { + gl.draw_arrays_instanced_base_instance( + topology, + first_vertex as i32, + vertex_count as i32, + instance_count as i32, + first_instance, + ) + } + } else { + unsafe { + gl.uniform_1_u32(first_instance_location.as_ref(), first_instance); + } + + // Don't use `gl.draw_arrays` for `instance_count == 1`. + // Angle has a bug where it doesn't consider the instance divisor when `DYNAMIC_DRAW` is used in `draw_arrays`. + // See https://github.com/gfx-rs/wgpu/issues/3578 + unsafe { + gl.draw_arrays_instanced( + topology, + first_vertex as i32, + vertex_count as i32, + instance_count as i32, + ) + } + }; + } + C::DrawIndexed { + topology, + index_type, + index_count, + index_offset, + base_vertex, + first_instance, + instance_count, + ref first_instance_location, + } => { + let supports_full_instancing = self + .shared + .private_caps + .contains(PrivateCapabilities::FULLY_FEATURED_INSTANCING); + + if supports_full_instancing { + unsafe { + gl.draw_elements_instanced_base_vertex_base_instance( + topology, + index_count as i32, + index_type, + index_offset as i32, + instance_count as i32, + base_vertex, + first_instance, + ) + } + } else { + unsafe { gl.uniform_1_u32(first_instance_location.as_ref(), first_instance) }; + + if base_vertex == 0 { + unsafe { + // Don't use `gl.draw_elements`/`gl.draw_elements_base_vertex` for `instance_count == 1`. + // Angle has a bug where it doesn't consider the instance divisor when `DYNAMIC_DRAW` is used in `gl.draw_elements`/`gl.draw_elements_base_vertex`. + // See https://github.com/gfx-rs/wgpu/issues/3578 + gl.draw_elements_instanced( + topology, + index_count as i32, + index_type, + index_offset as i32, + instance_count as i32, + ) + } + } else { + // If we've gotten here, wgpu-core has already validated that this function exists via the DownlevelFlags::BASE_VERTEX feature. + unsafe { + gl.draw_elements_instanced_base_vertex( + topology, + index_count as _, + index_type, + index_offset as i32, + instance_count as i32, + base_vertex, + ) + } + } + } + } + C::DrawIndirect { + topology, + indirect_buf, + indirect_offset, + ref first_instance_location, + } => { + unsafe { gl.uniform_1_u32(first_instance_location.as_ref(), 0) }; + + unsafe { gl.bind_buffer(glow::DRAW_INDIRECT_BUFFER, Some(indirect_buf)) }; + unsafe { gl.draw_arrays_indirect_offset(topology, indirect_offset as i32) }; + } + C::DrawIndexedIndirect { + topology, + index_type, + indirect_buf, + indirect_offset, + ref first_instance_location, + } => { + unsafe { gl.uniform_1_u32(first_instance_location.as_ref(), 0) }; + + unsafe { gl.bind_buffer(glow::DRAW_INDIRECT_BUFFER, Some(indirect_buf)) }; + unsafe { + gl.draw_elements_indirect_offset(topology, index_type, indirect_offset as i32) + }; + } + C::Dispatch(group_counts) => { + unsafe { gl.dispatch_compute(group_counts[0], group_counts[1], group_counts[2]) }; + } + C::DispatchIndirect { + indirect_buf, + indirect_offset, + } => { + unsafe { gl.bind_buffer(glow::DISPATCH_INDIRECT_BUFFER, Some(indirect_buf)) }; + unsafe { gl.dispatch_compute_indirect(indirect_offset as i32) }; + } + C::ClearBuffer { + ref dst, + dst_target, + ref range, + } => match dst.raw { + Some(buffer) => { + // When `INDEX_BUFFER_ROLE_CHANGE` isn't available, we can't copy into the + // index buffer from the zero buffer. This would fail in Chrome with the + // following message: + // + // > Cannot copy into an element buffer destination from a non-element buffer + // > source + // + // Instead, we'll upload zeroes into the buffer. + let can_use_zero_buffer = self + .shared + .private_caps + .contains(PrivateCapabilities::INDEX_BUFFER_ROLE_CHANGE) + || dst_target != glow::ELEMENT_ARRAY_BUFFER; + + if can_use_zero_buffer { + unsafe { gl.bind_buffer(glow::COPY_READ_BUFFER, Some(self.zero_buffer)) }; + unsafe { gl.bind_buffer(dst_target, Some(buffer)) }; + let mut dst_offset = range.start; + while dst_offset < range.end { + let size = (range.end - dst_offset).min(super::ZERO_BUFFER_SIZE as u64); + unsafe { + gl.copy_buffer_sub_data( + glow::COPY_READ_BUFFER, + dst_target, + 0, + dst_offset as i32, + size as i32, + ) + }; + dst_offset += size; + } + } else { + unsafe { gl.bind_buffer(dst_target, Some(buffer)) }; + let zeroes = vec![0u8; (range.end - range.start) as usize]; + unsafe { + gl.buffer_sub_data_u8_slice(dst_target, range.start as i32, &zeroes) + }; + } + } + None => { + lock(dst.data.as_ref().unwrap()).as_mut_slice() + [range.start as usize..range.end as usize] + .fill(0); + } + }, + C::CopyBufferToBuffer { + ref src, + src_target, + ref dst, + dst_target, + copy, + } => { + let copy_src_target = glow::COPY_READ_BUFFER; + let is_index_buffer_only_element_dst = !self + .shared + .private_caps + .contains(PrivateCapabilities::INDEX_BUFFER_ROLE_CHANGE) + && dst_target == glow::ELEMENT_ARRAY_BUFFER + || src_target == glow::ELEMENT_ARRAY_BUFFER; + + // WebGL not allowed to copy data from other targets to element buffer and can't copy element data to other buffers + let copy_dst_target = if is_index_buffer_only_element_dst { + glow::ELEMENT_ARRAY_BUFFER + } else { + glow::COPY_WRITE_BUFFER + }; + let size = copy.size.get() as usize; + match (src.raw, dst.raw) { + (Some(ref src), Some(ref dst)) => { + unsafe { gl.bind_buffer(copy_src_target, Some(*src)) }; + unsafe { gl.bind_buffer(copy_dst_target, Some(*dst)) }; + unsafe { + gl.copy_buffer_sub_data( + copy_src_target, + copy_dst_target, + copy.src_offset as _, + copy.dst_offset as _, + copy.size.get() as _, + ) + }; + } + (Some(src), None) => { + let mut data = lock(dst.data.as_ref().unwrap()); + let dst_data = &mut data.as_mut_slice() + [copy.dst_offset as usize..copy.dst_offset as usize + size]; + + unsafe { gl.bind_buffer(copy_src_target, Some(src)) }; + unsafe { + self.shared.get_buffer_sub_data( + gl, + copy_src_target, + copy.src_offset as i32, + dst_data, + ) + }; + } + (None, Some(dst)) => { + let data = lock(src.data.as_ref().unwrap()); + let src_data = &data.as_slice() + [copy.src_offset as usize..copy.src_offset as usize + size]; + unsafe { gl.bind_buffer(copy_dst_target, Some(dst)) }; + unsafe { + gl.buffer_sub_data_u8_slice( + copy_dst_target, + copy.dst_offset as i32, + src_data, + ) + }; + } + (None, None) => { + todo!() + } + } + unsafe { gl.bind_buffer(copy_src_target, None) }; + if is_index_buffer_only_element_dst { + unsafe { + gl.bind_buffer( + glow::ELEMENT_ARRAY_BUFFER, + *self.current_index_buffer.lock(), + ) + }; + } else { + unsafe { gl.bind_buffer(copy_dst_target, None) }; + } + } + #[cfg(webgl)] + C::CopyExternalImageToTexture { + ref src, + dst, + dst_target, + dst_format, + dst_premultiplication, + ref copy, + } => { + const UNPACK_FLIP_Y_WEBGL: u32 = + web_sys::WebGl2RenderingContext::UNPACK_FLIP_Y_WEBGL; + const UNPACK_PREMULTIPLY_ALPHA_WEBGL: u32 = + web_sys::WebGl2RenderingContext::UNPACK_PREMULTIPLY_ALPHA_WEBGL; + + unsafe { + if src.flip_y { + gl.pixel_store_bool(UNPACK_FLIP_Y_WEBGL, true); + } + if dst_premultiplication { + gl.pixel_store_bool(UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); + } + } + + unsafe { gl.bind_texture(dst_target, Some(dst)) }; + let format_desc = self.shared.describe_texture_format(dst_format); + if is_layered_target(dst_target) { + let z_offset = get_z_offset(dst_target, ©.dst_base); + + match src.source { + wgt::ExternalImageSource::ImageBitmap(ref b) => unsafe { + gl.tex_sub_image_3d_with_image_bitmap( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + z_offset as i32, + copy.size.width as i32, + copy.size.height as i32, + copy.size.depth as i32, + format_desc.external, + format_desc.data_type, + b, + ); + }, + wgt::ExternalImageSource::HTMLImageElement(ref i) => unsafe { + gl.tex_sub_image_3d_with_html_image_element( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + z_offset as i32, + copy.size.width as i32, + copy.size.height as i32, + copy.size.depth as i32, + format_desc.external, + format_desc.data_type, + i, + ); + }, + wgt::ExternalImageSource::HTMLVideoElement(ref v) => unsafe { + gl.tex_sub_image_3d_with_html_video_element( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + z_offset as i32, + copy.size.width as i32, + copy.size.height as i32, + copy.size.depth as i32, + format_desc.external, + format_desc.data_type, + v, + ); + }, + #[cfg(web_sys_unstable_apis)] + wgt::ExternalImageSource::VideoFrame(ref v) => unsafe { + gl.tex_sub_image_3d_with_video_frame( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + z_offset as i32, + copy.size.width as i32, + copy.size.height as i32, + copy.size.depth as i32, + format_desc.external, + format_desc.data_type, + v, + ) + }, + wgt::ExternalImageSource::ImageData(ref i) => unsafe { + gl.tex_sub_image_3d_with_image_data( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + z_offset as i32, + copy.size.width as i32, + copy.size.height as i32, + copy.size.depth as i32, + format_desc.external, + format_desc.data_type, + i, + ); + }, + wgt::ExternalImageSource::HTMLCanvasElement(ref c) => unsafe { + gl.tex_sub_image_3d_with_html_canvas_element( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + z_offset as i32, + copy.size.width as i32, + copy.size.height as i32, + copy.size.depth as i32, + format_desc.external, + format_desc.data_type, + c, + ); + }, + wgt::ExternalImageSource::OffscreenCanvas(_) => unreachable!(), + } + } else { + let dst_target = get_2d_target(dst_target, copy.dst_base.array_layer); + + match src.source { + wgt::ExternalImageSource::ImageBitmap(ref b) => unsafe { + gl.tex_sub_image_2d_with_image_bitmap_and_width_and_height( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + format_desc.external, + format_desc.data_type, + b, + ); + }, + wgt::ExternalImageSource::HTMLImageElement(ref i) => unsafe { + gl.tex_sub_image_2d_with_html_image_and_width_and_height( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + format_desc.external, + format_desc.data_type, + i, + ) + }, + wgt::ExternalImageSource::HTMLVideoElement(ref v) => unsafe { + gl.tex_sub_image_2d_with_html_video_and_width_and_height( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + format_desc.external, + format_desc.data_type, + v, + ) + }, + #[cfg(web_sys_unstable_apis)] + wgt::ExternalImageSource::VideoFrame(ref v) => unsafe { + gl.tex_sub_image_2d_with_video_frame_and_width_and_height( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + format_desc.external, + format_desc.data_type, + v, + ) + }, + wgt::ExternalImageSource::ImageData(ref i) => unsafe { + gl.tex_sub_image_2d_with_image_data_and_width_and_height( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + format_desc.external, + format_desc.data_type, + i, + ); + }, + wgt::ExternalImageSource::HTMLCanvasElement(ref c) => unsafe { + gl.tex_sub_image_2d_with_html_canvas_and_width_and_height( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + format_desc.external, + format_desc.data_type, + c, + ) + }, + wgt::ExternalImageSource::OffscreenCanvas(_) => unreachable!(), + } + } + + unsafe { + if src.flip_y { + gl.pixel_store_bool(UNPACK_FLIP_Y_WEBGL, false); + } + if dst_premultiplication { + gl.pixel_store_bool(UNPACK_PREMULTIPLY_ALPHA_WEBGL, false); + } + } + } + C::CopyTextureToTexture { + src, + src_target, + dst, + dst_target, + ref copy, + } => { + //TODO: handle 3D copies + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, Some(self.copy_fbo)) }; + if is_layered_target(src_target) { + //TODO: handle GLES without framebuffer_texture_3d + unsafe { + gl.framebuffer_texture_layer( + glow::READ_FRAMEBUFFER, + glow::COLOR_ATTACHMENT0, + Some(src), + copy.src_base.mip_level as i32, + copy.src_base.array_layer as i32, + ) + }; + } else { + unsafe { + gl.framebuffer_texture_2d( + glow::READ_FRAMEBUFFER, + glow::COLOR_ATTACHMENT0, + src_target, + Some(src), + copy.src_base.mip_level as i32, + ) + }; + } + + unsafe { gl.bind_texture(dst_target, Some(dst)) }; + if is_layered_target(dst_target) { + unsafe { + gl.copy_tex_sub_image_3d( + dst_target, + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + get_z_offset(dst_target, ©.dst_base) as i32, + copy.src_base.origin.x as i32, + copy.src_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + ) + }; + } else { + unsafe { + gl.copy_tex_sub_image_2d( + get_2d_target(dst_target, copy.dst_base.array_layer), + copy.dst_base.mip_level as i32, + copy.dst_base.origin.x as i32, + copy.dst_base.origin.y as i32, + copy.src_base.origin.x as i32, + copy.src_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + ) + }; + } + } + C::CopyBufferToTexture { + ref src, + src_target: _, + dst, + dst_target, + dst_format, + ref copy, + } => { + let (block_width, block_height) = dst_format.block_dimensions(); + let block_size = dst_format.block_copy_size(None).unwrap(); + let format_desc = self.shared.describe_texture_format(dst_format); + let row_texels = copy + .buffer_layout + .bytes_per_row + .map_or(0, |bpr| block_width * bpr / block_size); + let column_texels = copy + .buffer_layout + .rows_per_image + .map_or(0, |rpi| block_height * rpi); + + unsafe { gl.bind_texture(dst_target, Some(dst)) }; + unsafe { gl.pixel_store_i32(glow::UNPACK_ROW_LENGTH, row_texels as i32) }; + unsafe { gl.pixel_store_i32(glow::UNPACK_IMAGE_HEIGHT, column_texels as i32) }; + let mut unbind_unpack_buffer = false; + if !dst_format.is_compressed() { + let buffer_data; + let unpack_data = match src.raw { + Some(buffer) => { + unsafe { gl.bind_buffer(glow::PIXEL_UNPACK_BUFFER, Some(buffer)) }; + unbind_unpack_buffer = true; + glow::PixelUnpackData::BufferOffset(copy.buffer_layout.offset as u32) + } + None => { + buffer_data = lock(src.data.as_ref().unwrap()); + let src_data = + &buffer_data.as_slice()[copy.buffer_layout.offset as usize..]; + glow::PixelUnpackData::Slice(Some(src_data)) + } + }; + if is_layered_target(dst_target) { + unsafe { + gl.tex_sub_image_3d( + dst_target, + copy.texture_base.mip_level as i32, + copy.texture_base.origin.x as i32, + copy.texture_base.origin.y as i32, + get_z_offset(dst_target, ©.texture_base) as i32, + copy.size.width as i32, + copy.size.height as i32, + copy.size.depth as i32, + format_desc.external, + format_desc.data_type, + unpack_data, + ) + }; + } else { + unsafe { + gl.tex_sub_image_2d( + get_2d_target(dst_target, copy.texture_base.array_layer), + copy.texture_base.mip_level as i32, + copy.texture_base.origin.x as i32, + copy.texture_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + format_desc.external, + format_desc.data_type, + unpack_data, + ) + }; + } + } else { + let bytes_per_row = copy + .buffer_layout + .bytes_per_row + .unwrap_or(copy.size.width * block_size); + let minimum_rows_per_image = copy.size.height.div_ceil(block_height); + let rows_per_image = copy + .buffer_layout + .rows_per_image + .unwrap_or(minimum_rows_per_image); + + let bytes_per_image = bytes_per_row * rows_per_image; + let minimum_bytes_per_image = bytes_per_row * minimum_rows_per_image; + let bytes_in_upload = + (bytes_per_image * (copy.size.depth - 1)) + minimum_bytes_per_image; + let offset = copy.buffer_layout.offset as u32; + + let buffer_data; + let unpack_data = match src.raw { + Some(buffer) => { + unsafe { gl.bind_buffer(glow::PIXEL_UNPACK_BUFFER, Some(buffer)) }; + unbind_unpack_buffer = true; + glow::CompressedPixelUnpackData::BufferRange( + offset..offset + bytes_in_upload, + ) + } + None => { + buffer_data = lock(src.data.as_ref().unwrap()); + let src_data = &buffer_data.as_slice() + [(offset as usize)..(offset + bytes_in_upload) as usize]; + glow::CompressedPixelUnpackData::Slice(src_data) + } + }; + + if is_layered_target(dst_target) { + unsafe { + gl.compressed_tex_sub_image_3d( + dst_target, + copy.texture_base.mip_level as i32, + copy.texture_base.origin.x as i32, + copy.texture_base.origin.y as i32, + get_z_offset(dst_target, ©.texture_base) as i32, + copy.size.width as i32, + copy.size.height as i32, + copy.size.depth as i32, + format_desc.internal, + unpack_data, + ) + }; + } else { + unsafe { + gl.compressed_tex_sub_image_2d( + get_2d_target(dst_target, copy.texture_base.array_layer), + copy.texture_base.mip_level as i32, + copy.texture_base.origin.x as i32, + copy.texture_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + format_desc.internal, + unpack_data, + ) + }; + } + } + if unbind_unpack_buffer { + unsafe { gl.bind_buffer(glow::PIXEL_UNPACK_BUFFER, None) }; + } + } + C::CopyTextureToBuffer { + src, + src_target, + src_format, + ref dst, + dst_target: _, + ref copy, + } => { + let block_size = src_format.block_copy_size(None).unwrap(); + if src_format.is_compressed() { + log::error!("Not implemented yet: compressed texture copy to buffer"); + return; + } + if src_target == glow::TEXTURE_CUBE_MAP + || src_target == glow::TEXTURE_CUBE_MAP_ARRAY + { + log::error!("Not implemented yet: cubemap texture copy to buffer"); + return; + } + let format_desc = self.shared.describe_texture_format(src_format); + let row_texels = copy + .buffer_layout + .bytes_per_row + .map_or(copy.size.width, |bpr| bpr / block_size); + let column_texels = copy + .buffer_layout + .rows_per_image + .unwrap_or(copy.size.height); + + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, Some(self.copy_fbo)) }; + + let read_pixels = |offset| { + let mut buffer_data; + let unpack_data = match dst.raw { + Some(buffer) => { + unsafe { gl.pixel_store_i32(glow::PACK_ROW_LENGTH, row_texels as i32) }; + unsafe { gl.bind_buffer(glow::PIXEL_PACK_BUFFER, Some(buffer)) }; + glow::PixelPackData::BufferOffset(offset as u32) + } + None => { + buffer_data = lock(dst.data.as_ref().unwrap()); + let dst_data = &mut buffer_data.as_mut_slice()[offset as usize..]; + glow::PixelPackData::Slice(Some(dst_data)) + } + }; + unsafe { + gl.read_pixels( + copy.texture_base.origin.x as i32, + copy.texture_base.origin.y as i32, + copy.size.width as i32, + copy.size.height as i32, + format_desc.external, + format_desc.data_type, + unpack_data, + ) + }; + }; + + match src_target { + glow::TEXTURE_2D => { + unsafe { + gl.framebuffer_texture_2d( + glow::READ_FRAMEBUFFER, + glow::COLOR_ATTACHMENT0, + src_target, + Some(src), + copy.texture_base.mip_level as i32, + ) + }; + read_pixels(copy.buffer_layout.offset); + } + glow::TEXTURE_2D_ARRAY => { + unsafe { + gl.framebuffer_texture_layer( + glow::READ_FRAMEBUFFER, + glow::COLOR_ATTACHMENT0, + Some(src), + copy.texture_base.mip_level as i32, + copy.texture_base.array_layer as i32, + ) + }; + read_pixels(copy.buffer_layout.offset); + } + glow::TEXTURE_3D => { + for z in copy.texture_base.origin.z..copy.size.depth { + unsafe { + gl.framebuffer_texture_layer( + glow::READ_FRAMEBUFFER, + glow::COLOR_ATTACHMENT0, + Some(src), + copy.texture_base.mip_level as i32, + z as i32, + ) + }; + let offset = copy.buffer_layout.offset + + (z * block_size * row_texels * column_texels) as u64; + read_pixels(offset); + } + } + glow::TEXTURE_CUBE_MAP | glow::TEXTURE_CUBE_MAP_ARRAY => unimplemented!(), + _ => unreachable!(), + } + } + C::SetIndexBuffer(buffer) => { + unsafe { gl.bind_buffer(glow::ELEMENT_ARRAY_BUFFER, Some(buffer)) }; + let mut current_index_buffer = self.current_index_buffer.lock(); + *current_index_buffer = Some(buffer); + } + C::BeginQuery(query, target) => { + unsafe { gl.begin_query(target, query) }; + } + C::EndQuery(target) => { + unsafe { gl.end_query(target) }; + } + C::TimestampQuery(query) => { + unsafe { gl.query_counter(query, glow::TIMESTAMP) }; + } + C::CopyQueryResults { + ref query_range, + ref dst, + dst_target, + dst_offset, + } => { + if self + .shared + .private_caps + .contains(PrivateCapabilities::QUERY_BUFFERS) + && dst.raw.is_some() + { + unsafe { + // We're assuming that the only relevant queries are 8 byte timestamps or + // occlusion tests. + let query_size = 8; + + let query_range_size = query_size * query_range.len(); + + let buffer = gl.create_buffer().ok(); + gl.bind_buffer(glow::QUERY_BUFFER, buffer); + gl.buffer_data_size( + glow::QUERY_BUFFER, + query_range_size as _, + glow::STREAM_COPY, + ); + + for (i, &query) in queries + [query_range.start as usize..query_range.end as usize] + .iter() + .enumerate() + { + gl.get_query_parameter_u64_with_offset( + query, + glow::QUERY_RESULT, + query_size * i, + ) + } + gl.bind_buffer(dst_target, dst.raw); + gl.copy_buffer_sub_data( + glow::QUERY_BUFFER, + dst_target, + 0, + dst_offset as _, + query_range_size as _, + ); + if let Some(buffer) = buffer { + gl.delete_buffer(buffer) + } + } + } else { + let mut temp_query_results = self.temp_query_results.lock(); + temp_query_results.clear(); + for &query in + queries[query_range.start as usize..query_range.end as usize].iter() + { + let mut result: u64 = 0; + unsafe { + if self + .shared + .private_caps + .contains(PrivateCapabilities::QUERY_64BIT) + { + let result: *mut u64 = &mut result; + gl.get_query_parameter_u64_with_offset( + query, + glow::QUERY_RESULT, + result as usize, + ) + } else { + result = + gl.get_query_parameter_u32(query, glow::QUERY_RESULT) as u64; + } + }; + temp_query_results.push(result); + } + let query_data = bytemuck::cast_slice(&temp_query_results); + match dst.raw { + Some(buffer) => { + unsafe { gl.bind_buffer(dst_target, Some(buffer)) }; + unsafe { + gl.buffer_sub_data_u8_slice( + dst_target, + dst_offset as i32, + query_data, + ) + }; + } + None => { + let data = &mut lock(dst.data.as_ref().unwrap()); + let len = query_data.len().min(data.len()); + data[..len].copy_from_slice(&query_data[..len]); + } + } + } + } + C::ResetFramebuffer { is_default } => { + if is_default { + unsafe { gl.bind_framebuffer(glow::DRAW_FRAMEBUFFER, None) }; + } else { + unsafe { gl.bind_framebuffer(glow::DRAW_FRAMEBUFFER, Some(self.draw_fbo)) }; + unsafe { + gl.framebuffer_texture_2d( + glow::DRAW_FRAMEBUFFER, + glow::DEPTH_STENCIL_ATTACHMENT, + glow::TEXTURE_2D, + None, + 0, + ) + }; + for i in 0..self.shared.limits.max_color_attachments { + let target = glow::COLOR_ATTACHMENT0 + i; + unsafe { + gl.framebuffer_texture_2d( + glow::DRAW_FRAMEBUFFER, + target, + glow::TEXTURE_2D, + None, + 0, + ) + }; + } + } + unsafe { gl.color_mask(true, true, true, true) }; + unsafe { gl.depth_mask(true) }; + unsafe { gl.stencil_mask(!0) }; + unsafe { gl.disable(glow::DEPTH_TEST) }; + unsafe { gl.disable(glow::STENCIL_TEST) }; + unsafe { gl.disable(glow::SCISSOR_TEST) }; + } + C::BindAttachment { + attachment, + ref view, + depth_slice, + sample_count, + } => { + unsafe { + self.set_attachment( + gl, + glow::DRAW_FRAMEBUFFER, + attachment, + view, + depth_slice, + sample_count, + ) + }; + } + C::ResolveAttachment { + attachment, + ref dst, + ref size, + } => { + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, Some(self.draw_fbo)) }; + unsafe { gl.read_buffer(attachment) }; + unsafe { gl.bind_framebuffer(glow::DRAW_FRAMEBUFFER, Some(self.copy_fbo)) }; + unsafe { + self.set_attachment( + gl, + glow::DRAW_FRAMEBUFFER, + glow::COLOR_ATTACHMENT0, + dst, + None, + 1, + ) + }; + unsafe { + gl.blit_framebuffer( + 0, + 0, + size.width as i32, + size.height as i32, + 0, + 0, + size.width as i32, + size.height as i32, + glow::COLOR_BUFFER_BIT, + glow::NEAREST, + ) + }; + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, None) }; + unsafe { gl.bind_framebuffer(glow::DRAW_FRAMEBUFFER, Some(self.draw_fbo)) }; + } + C::InvalidateAttachments(ref list) => { + if self + .shared + .private_caps + .contains(PrivateCapabilities::INVALIDATE_FRAMEBUFFER) + { + unsafe { gl.invalidate_framebuffer(glow::DRAW_FRAMEBUFFER, list) }; + } + } + C::SetDrawColorBuffers(count) => { + self.draw_buffer_count.store(count, Ordering::Relaxed); + let indices = (0..count as u32) + .map(|i| glow::COLOR_ATTACHMENT0 + i) + .collect::>(); + unsafe { gl.draw_buffers(&indices) }; + } + C::ClearColorF { + draw_buffer, + ref color, + is_srgb, + } => { + if self + .shared + .workarounds + .contains(super::Workarounds::MESA_I915_SRGB_SHADER_CLEAR) + && is_srgb + { + unsafe { self.perform_shader_clear(gl, draw_buffer, *color) }; + } else { + unsafe { gl.clear_buffer_f32_slice(glow::COLOR, draw_buffer, color) }; + } + } + C::ClearColorU(draw_buffer, ref color) => { + unsafe { gl.clear_buffer_u32_slice(glow::COLOR, draw_buffer, color) }; + } + C::ClearColorI(draw_buffer, ref color) => { + unsafe { gl.clear_buffer_i32_slice(glow::COLOR, draw_buffer, color) }; + } + C::ClearDepth(depth) => { + // Prefer `clear` as `clear_buffer` functions have issues on Sandy Bridge + // on Windows. + unsafe { + gl.clear_depth_f32(depth); + gl.clear(glow::DEPTH_BUFFER_BIT); + } + } + C::ClearStencil(value) => { + // Prefer `clear` as `clear_buffer` functions have issues on Sandy Bridge + // on Windows. + unsafe { + gl.clear_stencil(value as i32); + gl.clear(glow::STENCIL_BUFFER_BIT); + } + } + C::ClearDepthAndStencil(depth, stencil_value) => { + // Prefer `clear` as `clear_buffer` functions have issues on Sandy Bridge + // on Windows. + unsafe { + gl.clear_depth_f32(depth); + gl.clear_stencil(stencil_value as i32); + gl.clear(glow::DEPTH_BUFFER_BIT | glow::STENCIL_BUFFER_BIT); + } + } + C::BufferBarrier(raw, usage) => { + let mut flags = 0; + if usage.contains(wgt::BufferUses::VERTEX) { + flags |= glow::VERTEX_ATTRIB_ARRAY_BARRIER_BIT; + unsafe { gl.bind_buffer(glow::ARRAY_BUFFER, Some(raw)) }; + unsafe { gl.vertex_attrib_pointer_f32(0, 1, glow::BYTE, true, 0, 0) }; + } + if usage.contains(wgt::BufferUses::INDEX) { + flags |= glow::ELEMENT_ARRAY_BARRIER_BIT; + unsafe { gl.bind_buffer(glow::ELEMENT_ARRAY_BUFFER, Some(raw)) }; + } + if usage.contains(wgt::BufferUses::UNIFORM) { + flags |= glow::UNIFORM_BARRIER_BIT; + } + if usage.contains(wgt::BufferUses::INDIRECT) { + flags |= glow::COMMAND_BARRIER_BIT; + unsafe { gl.bind_buffer(glow::DRAW_INDIRECT_BUFFER, Some(raw)) }; + } + if usage.contains(wgt::BufferUses::COPY_SRC) { + flags |= glow::PIXEL_BUFFER_BARRIER_BIT; + unsafe { gl.bind_buffer(glow::PIXEL_UNPACK_BUFFER, Some(raw)) }; + } + if usage.contains(wgt::BufferUses::COPY_DST) { + flags |= glow::PIXEL_BUFFER_BARRIER_BIT; + unsafe { gl.bind_buffer(glow::PIXEL_PACK_BUFFER, Some(raw)) }; + } + if usage.intersects(wgt::BufferUses::MAP_READ | wgt::BufferUses::MAP_WRITE) { + flags |= glow::BUFFER_UPDATE_BARRIER_BIT; + } + if usage.intersects( + wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE, + ) { + flags |= glow::SHADER_STORAGE_BARRIER_BIT; + } + unsafe { gl.memory_barrier(flags) }; + } + // because `STORAGE_WRITE_ONLY` and `STORAGE_READ_WRITE` are only states + // we can transit from due OpenGL memory barriers are used to make _subsequent_ + // operations see changes from the _shader_ side. We filter out usage changes that are + // does not comes from the shader side in `transition_textures` + C::TextureBarrier(usage) => { + let mut flags = 0; + if usage.contains(wgt::TextureUses::RESOURCE) { + flags |= glow::TEXTURE_FETCH_BARRIER_BIT; + } + if usage.intersects( + wgt::TextureUses::STORAGE_READ_ONLY + | wgt::TextureUses::STORAGE_WRITE_ONLY + | wgt::TextureUses::STORAGE_READ_WRITE, + ) { + flags |= glow::SHADER_IMAGE_ACCESS_BARRIER_BIT; + } + if usage.intersects(wgt::TextureUses::COPY_SRC) { + flags |= glow::PIXEL_BUFFER_BARRIER_BIT; + } + if usage.contains(wgt::TextureUses::COPY_DST) { + flags |= glow::TEXTURE_UPDATE_BARRIER_BIT; + } + if usage.intersects( + wgt::TextureUses::COLOR_TARGET + | wgt::TextureUses::DEPTH_STENCIL_READ + | wgt::TextureUses::DEPTH_STENCIL_WRITE, + ) { + flags |= glow::FRAMEBUFFER_BARRIER_BIT; + } + unsafe { gl.memory_barrier(flags) }; + } + C::SetViewport { + ref rect, + ref depth, + } => { + unsafe { gl.viewport(rect.x, rect.y, rect.w, rect.h) }; + unsafe { gl.depth_range_f32(depth.start, depth.end) }; + } + C::SetScissor(ref rect) => { + unsafe { gl.scissor(rect.x, rect.y, rect.w, rect.h) }; + unsafe { gl.enable(glow::SCISSOR_TEST) }; + } + C::SetStencilFunc { + face, + function, + reference, + read_mask, + } => { + unsafe { gl.stencil_func_separate(face, function, reference as i32, read_mask) }; + } + C::SetStencilOps { + face, + write_mask, + ref ops, + } => { + unsafe { gl.stencil_mask_separate(face, write_mask) }; + unsafe { gl.stencil_op_separate(face, ops.fail, ops.depth_fail, ops.pass) }; + } + C::SetVertexAttribute { + buffer, + ref buffer_desc, + attribute_desc: ref vat, + } => { + unsafe { gl.bind_buffer(glow::ARRAY_BUFFER, buffer) }; + unsafe { gl.enable_vertex_attrib_array(vat.location) }; + + if buffer.is_none() { + match vat.format_desc.attrib_kind { + super::VertexAttribKind::Float => unsafe { + gl.vertex_attrib_format_f32( + vat.location, + vat.format_desc.element_count, + vat.format_desc.element_format, + true, // always normalized + vat.offset, + ) + }, + super::VertexAttribKind::Integer => unsafe { + gl.vertex_attrib_format_i32( + vat.location, + vat.format_desc.element_count, + vat.format_desc.element_format, + vat.offset, + ) + }, + } + + //Note: there is apparently a bug on AMD 3500U: + // this call is ignored if the current array is disabled. + unsafe { gl.vertex_attrib_binding(vat.location, vat.buffer_index) }; + } else { + match vat.format_desc.attrib_kind { + super::VertexAttribKind::Float => unsafe { + gl.vertex_attrib_pointer_f32( + vat.location, + vat.format_desc.element_count, + vat.format_desc.element_format, + true, // always normalized + buffer_desc.stride as i32, + vat.offset as i32, + ) + }, + super::VertexAttribKind::Integer => unsafe { + gl.vertex_attrib_pointer_i32( + vat.location, + vat.format_desc.element_count, + vat.format_desc.element_format, + buffer_desc.stride as i32, + vat.offset as i32, + ) + }, + } + unsafe { gl.vertex_attrib_divisor(vat.location, buffer_desc.step as u32) }; + } + } + C::UnsetVertexAttribute(location) => { + unsafe { gl.disable_vertex_attrib_array(location) }; + } + C::SetVertexBuffer { + index, + ref buffer, + ref buffer_desc, + } => { + unsafe { gl.vertex_binding_divisor(index, buffer_desc.step as u32) }; + unsafe { + gl.bind_vertex_buffer( + index, + Some(buffer.raw), + buffer.offset as i32, + buffer_desc.stride as i32, + ) + }; + } + C::SetDepth(ref depth) => { + unsafe { gl.depth_func(depth.function) }; + unsafe { gl.depth_mask(depth.mask) }; + } + C::SetDepthBias(bias) => { + if bias.is_enabled() { + unsafe { gl.enable(glow::POLYGON_OFFSET_FILL) }; + unsafe { gl.polygon_offset(bias.slope_scale, bias.constant as f32) }; + } else { + unsafe { gl.disable(glow::POLYGON_OFFSET_FILL) }; + } + } + C::ConfigureDepthStencil(aspects) => { + if aspects.contains(crate::FormatAspects::DEPTH) { + unsafe { gl.enable(glow::DEPTH_TEST) }; + } else { + unsafe { gl.disable(glow::DEPTH_TEST) }; + } + if aspects.contains(crate::FormatAspects::STENCIL) { + unsafe { gl.enable(glow::STENCIL_TEST) }; + } else { + unsafe { gl.disable(glow::STENCIL_TEST) }; + } + } + C::SetAlphaToCoverage(enabled) => { + if enabled { + unsafe { gl.enable(glow::SAMPLE_ALPHA_TO_COVERAGE) }; + } else { + unsafe { gl.disable(glow::SAMPLE_ALPHA_TO_COVERAGE) }; + } + } + C::SetProgram(program) => { + unsafe { gl.use_program(Some(program)) }; + } + C::SetPrimitive(ref state) => { + unsafe { gl.front_face(state.front_face) }; + if state.cull_face != 0 { + unsafe { gl.enable(glow::CULL_FACE) }; + unsafe { gl.cull_face(state.cull_face) }; + } else { + unsafe { gl.disable(glow::CULL_FACE) }; + } + if self.features.contains(wgt::Features::DEPTH_CLIP_CONTROL) { + //Note: this is a bit tricky, since we are controlling the clip, not the clamp. + if state.unclipped_depth { + unsafe { gl.enable(glow::DEPTH_CLAMP) }; + } else { + unsafe { gl.disable(glow::DEPTH_CLAMP) }; + } + } + // POLYGON_MODE_LINE also implies POLYGON_MODE_POINT + if self.features.contains(wgt::Features::POLYGON_MODE_LINE) { + unsafe { gl.polygon_mode(glow::FRONT_AND_BACK, state.polygon_mode) }; + } + } + C::SetBlendConstant(c) => { + unsafe { gl.blend_color(c[0], c[1], c[2], c[3]) }; + } + C::SetColorTarget { + draw_buffer_index, + desc: super::ColorTargetDesc { mask, ref blend }, + } => { + use wgt::ColorWrites as Cw; + if let Some(index) = draw_buffer_index { + unsafe { + gl.color_mask_draw_buffer( + index, + mask.contains(Cw::RED), + mask.contains(Cw::GREEN), + mask.contains(Cw::BLUE), + mask.contains(Cw::ALPHA), + ) + }; + if let Some(ref blend) = *blend { + unsafe { gl.enable_draw_buffer(glow::BLEND, index) }; + if blend.color != blend.alpha { + unsafe { + gl.blend_equation_separate_draw_buffer( + index, + blend.color.equation, + blend.alpha.equation, + ) + }; + unsafe { + gl.blend_func_separate_draw_buffer( + index, + blend.color.src, + blend.color.dst, + blend.alpha.src, + blend.alpha.dst, + ) + }; + } else { + unsafe { gl.blend_equation_draw_buffer(index, blend.color.equation) }; + unsafe { + gl.blend_func_draw_buffer(index, blend.color.src, blend.color.dst) + }; + } + } else { + unsafe { gl.disable_draw_buffer(glow::BLEND, index) }; + } + } else { + unsafe { + gl.color_mask( + mask.contains(Cw::RED), + mask.contains(Cw::GREEN), + mask.contains(Cw::BLUE), + mask.contains(Cw::ALPHA), + ) + }; + if let Some(ref blend) = *blend { + unsafe { gl.enable(glow::BLEND) }; + if blend.color != blend.alpha { + unsafe { + gl.blend_equation_separate( + blend.color.equation, + blend.alpha.equation, + ) + }; + unsafe { + gl.blend_func_separate( + blend.color.src, + blend.color.dst, + blend.alpha.src, + blend.alpha.dst, + ) + }; + } else { + unsafe { gl.blend_equation(blend.color.equation) }; + unsafe { gl.blend_func(blend.color.src, blend.color.dst) }; + } + } else { + unsafe { gl.disable(glow::BLEND) }; + } + } + } + C::BindBuffer { + target, + slot, + buffer, + offset, + size, + } => { + unsafe { gl.bind_buffer_range(target, slot, Some(buffer), offset, size) }; + } + C::BindSampler(texture_index, sampler) => { + unsafe { gl.bind_sampler(texture_index, sampler) }; + } + C::BindTexture { + slot, + texture, + target, + aspects, + ref mip_levels, + } => { + unsafe { gl.active_texture(glow::TEXTURE0 + slot) }; + unsafe { gl.bind_texture(target, Some(texture)) }; + + unsafe { + gl.tex_parameter_i32(target, glow::TEXTURE_BASE_LEVEL, mip_levels.start as i32) + }; + unsafe { + gl.tex_parameter_i32( + target, + glow::TEXTURE_MAX_LEVEL, + (mip_levels.end - 1) as i32, + ) + }; + + let version = gl.version(); + let is_min_es_3_1 = version.is_embedded && (version.major, version.minor) >= (3, 1); + let is_min_4_3 = !version.is_embedded && (version.major, version.minor) >= (4, 3); + if is_min_es_3_1 || is_min_4_3 { + let mode = match aspects { + crate::FormatAspects::DEPTH => Some(glow::DEPTH_COMPONENT), + crate::FormatAspects::STENCIL => Some(glow::STENCIL_INDEX), + _ => None, + }; + if let Some(mode) = mode { + unsafe { + gl.tex_parameter_i32( + target, + glow::DEPTH_STENCIL_TEXTURE_MODE, + mode as _, + ) + }; + } + } + } + C::BindImage { slot, ref binding } => { + unsafe { + gl.bind_image_texture( + slot, + Some(binding.raw), + binding.mip_level as i32, + binding.array_layer.is_none(), + binding.array_layer.unwrap_or_default() as i32, + binding.access, + binding.format, + ) + }; + } + C::InsertDebugMarker(ref range) => { + let marker = extract_marker(data_bytes, range); + unsafe { + if self + .shared + .private_caps + .contains(PrivateCapabilities::DEBUG_FNS) + { + gl.debug_message_insert( + glow::DEBUG_SOURCE_APPLICATION, + glow::DEBUG_TYPE_MARKER, + DEBUG_ID, + glow::DEBUG_SEVERITY_NOTIFICATION, + to_debug_str(marker), + ) + } + }; + } + C::PushDebugGroup(ref range) => { + let marker = extract_marker(data_bytes, range); + unsafe { + if self + .shared + .private_caps + .contains(PrivateCapabilities::DEBUG_FNS) + { + gl.push_debug_group( + glow::DEBUG_SOURCE_APPLICATION, + DEBUG_ID, + to_debug_str(marker), + ) + } + }; + } + C::PopDebugGroup => { + unsafe { + if self + .shared + .private_caps + .contains(PrivateCapabilities::DEBUG_FNS) + { + gl.pop_debug_group() + } + }; + } + C::SetImmediates { + ref uniform, + offset, + } => { + fn get_data(data: &[u8], offset: u32) -> [T; COUNT] + where + [T; COUNT]: bytemuck::AnyBitPattern, + { + let data_required = size_of::() * COUNT; + let raw = &data[(offset as usize)..][..data_required]; + bytemuck::pod_read_unaligned(raw) + } + + let location = Some(&uniform.location); + + match uniform.ty { + // + // --- Float 1-4 Component --- + // + naga::TypeInner::Scalar(naga::Scalar::F32) => { + let data = get_data::(data_bytes, offset)[0]; + unsafe { gl.uniform_1_f32(location, data) }; + } + naga::TypeInner::Vector { + size: naga::VectorSize::Bi, + scalar: naga::Scalar::F32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_2_f32_slice(location, data) }; + } + naga::TypeInner::Vector { + size: naga::VectorSize::Tri, + scalar: naga::Scalar::F32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_3_f32_slice(location, data) }; + } + naga::TypeInner::Vector { + size: naga::VectorSize::Quad, + scalar: naga::Scalar::F32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_4_f32_slice(location, data) }; + } + + // + // --- Int 1-4 Component --- + // + naga::TypeInner::Scalar(naga::Scalar::I32) => { + let data = get_data::(data_bytes, offset)[0]; + unsafe { gl.uniform_1_i32(location, data) }; + } + naga::TypeInner::Vector { + size: naga::VectorSize::Bi, + scalar: naga::Scalar::I32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_2_i32_slice(location, data) }; + } + naga::TypeInner::Vector { + size: naga::VectorSize::Tri, + scalar: naga::Scalar::I32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_3_i32_slice(location, data) }; + } + naga::TypeInner::Vector { + size: naga::VectorSize::Quad, + scalar: naga::Scalar::I32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_4_i32_slice(location, data) }; + } + + // + // --- Uint 1-4 Component --- + // + naga::TypeInner::Scalar(naga::Scalar::U32) => { + let data = get_data::(data_bytes, offset)[0]; + unsafe { gl.uniform_1_u32(location, data) }; + } + naga::TypeInner::Vector { + size: naga::VectorSize::Bi, + scalar: naga::Scalar::U32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_2_u32_slice(location, data) }; + } + naga::TypeInner::Vector { + size: naga::VectorSize::Tri, + scalar: naga::Scalar::U32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_3_u32_slice(location, data) }; + } + naga::TypeInner::Vector { + size: naga::VectorSize::Quad, + scalar: naga::Scalar::U32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_4_u32_slice(location, data) }; + } + + // + // --- Matrix 2xR --- + // + naga::TypeInner::Matrix { + columns: naga::VectorSize::Bi, + rows: naga::VectorSize::Bi, + scalar: naga::Scalar::F32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_matrix_2_f32_slice(location, false, data) }; + } + naga::TypeInner::Matrix { + columns: naga::VectorSize::Bi, + rows: naga::VectorSize::Tri, + scalar: naga::Scalar::F32, + } => { + // repack 2 vec3s into 6 values. + let unpacked_data = &get_data::(data_bytes, offset); + #[rustfmt::skip] + let packed_data = [ + unpacked_data[0], unpacked_data[1], unpacked_data[2], + unpacked_data[4], unpacked_data[5], unpacked_data[6], + ]; + unsafe { gl.uniform_matrix_2x3_f32_slice(location, false, &packed_data) }; + } + naga::TypeInner::Matrix { + columns: naga::VectorSize::Bi, + rows: naga::VectorSize::Quad, + scalar: naga::Scalar::F32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_matrix_2x4_f32_slice(location, false, data) }; + } + + // + // --- Matrix 3xR --- + // + naga::TypeInner::Matrix { + columns: naga::VectorSize::Tri, + rows: naga::VectorSize::Bi, + scalar: naga::Scalar::F32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_matrix_3x2_f32_slice(location, false, data) }; + } + naga::TypeInner::Matrix { + columns: naga::VectorSize::Tri, + rows: naga::VectorSize::Tri, + scalar: naga::Scalar::F32, + } => { + // repack 3 vec3s into 9 values. + let unpacked_data = &get_data::(data_bytes, offset); + #[rustfmt::skip] + let packed_data = [ + unpacked_data[0], unpacked_data[1], unpacked_data[2], + unpacked_data[4], unpacked_data[5], unpacked_data[6], + unpacked_data[8], unpacked_data[9], unpacked_data[10], + ]; + unsafe { gl.uniform_matrix_3_f32_slice(location, false, &packed_data) }; + } + naga::TypeInner::Matrix { + columns: naga::VectorSize::Tri, + rows: naga::VectorSize::Quad, + scalar: naga::Scalar::F32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_matrix_3x4_f32_slice(location, false, data) }; + } + + // + // --- Matrix 4xR --- + // + naga::TypeInner::Matrix { + columns: naga::VectorSize::Quad, + rows: naga::VectorSize::Bi, + scalar: naga::Scalar::F32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_matrix_4x2_f32_slice(location, false, data) }; + } + naga::TypeInner::Matrix { + columns: naga::VectorSize::Quad, + rows: naga::VectorSize::Tri, + scalar: naga::Scalar::F32, + } => { + // repack 4 vec3s into 12 values. + let unpacked_data = &get_data::(data_bytes, offset); + #[rustfmt::skip] + let packed_data = [ + unpacked_data[0], unpacked_data[1], unpacked_data[2], + unpacked_data[4], unpacked_data[5], unpacked_data[6], + unpacked_data[8], unpacked_data[9], unpacked_data[10], + unpacked_data[12], unpacked_data[13], unpacked_data[14], + ]; + unsafe { gl.uniform_matrix_4x3_f32_slice(location, false, &packed_data) }; + } + naga::TypeInner::Matrix { + columns: naga::VectorSize::Quad, + rows: naga::VectorSize::Quad, + scalar: naga::Scalar::F32, + } => { + let data = &get_data::(data_bytes, offset); + unsafe { gl.uniform_matrix_4_f32_slice(location, false, data) }; + } + _ => panic!("Unsupported uniform datatype: {:?}!", uniform.ty), + } + } + C::SetClipDistances { + old_count, + new_count, + } => { + // Disable clip planes that are no longer active + for i in new_count..old_count { + unsafe { gl.disable(glow::CLIP_DISTANCE0 + i) }; + } + + // Enable clip planes that are now active + for i in old_count..new_count { + unsafe { gl.enable(glow::CLIP_DISTANCE0 + i) }; + } + } + } + } +} + +impl crate::Queue for super::Queue { + type A = super::Api; + + unsafe fn submit( + &self, + command_buffers: &[&super::CommandBuffer], + _surface_textures: &[&super::Texture], + (signal_fence, signal_value): (&mut super::Fence, crate::FenceValue), + ) -> Result<(), crate::DeviceError> { + let shared = Arc::clone(&self.shared); + let gl = &shared.context.lock(); + for cmd_buf in command_buffers.iter() { + // The command encoder assumes a default state when encoding the command buffer. + // Always reset the state between command_buffers to reflect this assumption. Do + // this at the beginning of the loop in case something outside of wgpu modified + // this state prior to commit. + unsafe { self.reset_state(gl) }; + if let Some(ref label) = cmd_buf.label { + if self + .shared + .private_caps + .contains(PrivateCapabilities::DEBUG_FNS) + { + unsafe { + gl.push_debug_group( + glow::DEBUG_SOURCE_APPLICATION, + DEBUG_ID, + to_debug_str(label), + ) + }; + } + } + + for command in cmd_buf.commands.iter() { + unsafe { self.process(gl, command, &cmd_buf.data_bytes, &cmd_buf.queries) }; + } + + if cmd_buf.label.is_some() + && self + .shared + .private_caps + .contains(PrivateCapabilities::DEBUG_FNS) + { + unsafe { gl.pop_debug_group() }; + } + } + + signal_fence.maintain(gl); + signal_fence.signal(gl, signal_value)?; + + // This is extremely important. If we don't flush, the above fences may never + // be signaled, particularly in headless contexts. Headed contexts will + // often flush every so often, but headless contexts may not. + unsafe { gl.flush() }; + + Ok(()) + } + + unsafe fn present( + &self, + surface: &super::Surface, + texture: super::Texture, + ) -> Result<(), crate::SurfaceError> { + unsafe { surface.present(texture, &self.shared.context) } + } + + unsafe fn get_timestamp_period(&self) -> f32 { + 1.0 + } +} + +#[cfg(send_sync)] +unsafe impl Sync for super::Queue {} +#[cfg(send_sync)] +unsafe impl Send for super::Queue {} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/clear.frag b/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/clear.frag new file mode 100644 index 00000000..1d0e414b --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/clear.frag @@ -0,0 +1,7 @@ +uniform vec4 color; +//Hack: Some WebGL implementations don't find "color" otherwise. +uniform vec4 color_workaround; +out vec4 frag; +void main() { + frag = color + color_workaround; +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/clear.vert b/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/clear.vert new file mode 100644 index 00000000..341b4e5f --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/clear.vert @@ -0,0 +1,9 @@ +// A triangle that fills the whole screen +vec2[3] TRIANGLE_POS = vec2[]( + vec2( 0.0, -3.0), + vec2(-3.0, 1.0), + vec2( 3.0, 1.0) +); +void main() { + gl_Position = vec4(TRIANGLE_POS[gl_VertexID], 0.0, 1.0); +} \ No newline at end of file diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/srgb_present.frag b/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/srgb_present.frag new file mode 100644 index 00000000..853f82a6 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/srgb_present.frag @@ -0,0 +1,16 @@ +#version 300 es +precision mediump float; +in vec2 uv; +uniform sampler2D present_texture; +out vec4 frag; +vec4 linear_to_srgb(vec4 linear) { + vec3 color_linear = linear.rgb; + vec3 selector = ceil(color_linear - 0.0031308); // 0 if under value, 1 if over + vec3 under = 12.92 * color_linear; + vec3 over = 1.055 * pow(color_linear, vec3(0.41666)) - 0.055; + vec3 result = mix(under, over, selector); + return vec4(result, linear.a); +} +void main() { + frag = linear_to_srgb(texture(present_texture, uv)); +} \ No newline at end of file diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/srgb_present.vert b/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/srgb_present.vert new file mode 100644 index 00000000..922f2a18 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/shaders/srgb_present.vert @@ -0,0 +1,18 @@ +#version 300 es +precision mediump float; +// A triangle that fills the whole screen +const vec2[3] TRIANGLE_POS = vec2[]( + vec2( 0.0, -3.0), + vec2(-3.0, 1.0), + vec2( 3.0, 1.0) +); +const vec2[3] TRIANGLE_UV = vec2[]( + vec2( 0.5, 1.), + vec2( -1.0, -1.0), + vec2( 2.0, -1.0) +); +out vec2 uv; +void main() { + uv = TRIANGLE_UV[gl_VertexID]; + gl_Position = vec4(TRIANGLE_POS[gl_VertexID], 0.0, 1.0); +} \ No newline at end of file diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/web.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/web.rs new file mode 100644 index 00000000..8e404d6b --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/web.rs @@ -0,0 +1,453 @@ +use alloc::{format, string::String, vec::Vec}; + +use glow::HasContext; +use parking_lot::{Mutex, RwLock}; +use wasm_bindgen::{JsCast, JsValue}; + +use super::TextureFormatDesc; + +/// A wrapper around a [`glow::Context`] to provide a fake `lock()` api that makes it compatible +/// with the `AdapterContext` API from the EGL implementation. +pub struct AdapterContext { + pub glow_context: glow::Context, + pub webgl2_context: web_sys::WebGl2RenderingContext, +} + +impl AdapterContext { + pub fn is_owned(&self) -> bool { + false + } + + /// Obtain a lock to the EGL context and get handle to the [`glow::Context`] that can be used to + /// do rendering. + #[track_caller] + pub fn lock(&self) -> &glow::Context { + &self.glow_context + } +} + +#[derive(Debug)] +pub struct Instance { + options: wgt::GlBackendOptions, +} + +impl Instance { + pub fn create_surface_from_canvas( + &self, + canvas: web_sys::HtmlCanvasElement, + ) -> Result { + let result = + canvas.get_context_with_context_options("webgl2", &Self::create_context_options()); + self.create_surface_from_context(Canvas::Canvas(canvas), result) + } + + pub fn create_surface_from_offscreen_canvas( + &self, + canvas: web_sys::OffscreenCanvas, + ) -> Result { + let result = + canvas.get_context_with_context_options("webgl2", &Self::create_context_options()); + self.create_surface_from_context(Canvas::Offscreen(canvas), result) + } + + /// Common portion of public `create_surface_from_*` functions. + /// + /// Note: Analogous code also exists in the WebGPU backend at + /// `wgpu::backend::web::Context`. + fn create_surface_from_context( + &self, + canvas: Canvas, + context_result: Result, JsValue>, + ) -> Result { + let context_object: js_sys::Object = match context_result { + Ok(Some(context)) => context, + Ok(None) => { + // + // A getContext() call “returns null if contextId is not supported, or if the + // canvas has already been initialized with another context type”. Additionally, + // “not supported” could include “insufficient GPU resources” or “the GPU process + // previously crashed”. So, we must return it as an `Err` since it could occur + // for circumstances outside the application author's control. + return Err(crate::InstanceError::new(String::from(concat!( + "canvas.getContext() returned null; ", + "webgl2 not available or canvas already in use" + )))); + } + Err(js_error) => { + // + // A thrown exception indicates misuse of the canvas state. + return Err(crate::InstanceError::new(format!( + "canvas.getContext() threw exception {js_error:?}", + ))); + } + }; + + // Not returning this error because it is a type error that shouldn't happen unless + // the browser, JS builtin objects, or wasm bindings are misbehaving somehow. + let webgl2_context: web_sys::WebGl2RenderingContext = context_object + .dyn_into() + .expect("canvas context is not a WebGl2RenderingContext"); + + Ok(Surface { + canvas, + webgl2_context, + srgb_present_program: Mutex::new(None), + swapchain: RwLock::new(None), + texture: Mutex::new(None), + presentable: true, + }) + } + + fn create_context_options() -> js_sys::Object { + let context_options = js_sys::Object::new(); + js_sys::Reflect::set(&context_options, &"antialias".into(), &JsValue::FALSE) + .expect("Cannot create context options"); + context_options + } +} + +#[cfg(send_sync)] +unsafe impl Sync for Instance {} +#[cfg(send_sync)] +unsafe impl Send for Instance {} + +impl crate::Instance for Instance { + type A = super::Api; + + unsafe fn init(desc: &crate::InstanceDescriptor<'_>) -> Result { + profiling::scope!("Init OpenGL (WebGL) Backend"); + Ok(Instance { + options: desc.backend_options.gl.clone(), + }) + } + + unsafe fn enumerate_adapters( + &self, + surface_hint: Option<&Surface>, + ) -> Vec> { + if let Some(surface_hint) = surface_hint { + let gl = glow::Context::from_webgl2_context(surface_hint.webgl2_context.clone()); + + unsafe { + super::Adapter::expose( + AdapterContext { + glow_context: gl, + webgl2_context: surface_hint.webgl2_context.clone(), + }, + self.options.clone(), + ) + } + .into_iter() + .collect() + } else { + Vec::new() + } + } + + unsafe fn create_surface( + &self, + _display_handle: raw_window_handle::RawDisplayHandle, + window_handle: raw_window_handle::RawWindowHandle, + ) -> Result { + let canvas: web_sys::HtmlCanvasElement = match window_handle { + raw_window_handle::RawWindowHandle::Web(handle) => web_sys::window() + .and_then(|win| win.document()) + .expect("Cannot get document") + .query_selector(&format!("canvas[data-raw-handle=\"{}\"]", handle.id)) + .expect("Cannot query for canvas") + .expect("Canvas is not found") + .dyn_into() + .expect("Failed to downcast to canvas type"), + raw_window_handle::RawWindowHandle::WebCanvas(handle) => { + let value: &JsValue = unsafe { handle.obj.cast().as_ref() }; + value.clone().unchecked_into() + } + raw_window_handle::RawWindowHandle::WebOffscreenCanvas(handle) => { + let value: &JsValue = unsafe { handle.obj.cast().as_ref() }; + let canvas: web_sys::OffscreenCanvas = value.clone().unchecked_into(); + + return self.create_surface_from_offscreen_canvas(canvas); + } + _ => { + return Err(crate::InstanceError::new(format!( + "window handle {window_handle:?} is not a web handle" + ))) + } + }; + + self.create_surface_from_canvas(canvas) + } +} + +#[derive(Debug)] +pub struct Surface { + canvas: Canvas, + pub(super) webgl2_context: web_sys::WebGl2RenderingContext, + pub(super) swapchain: RwLock>, + texture: Mutex>, + pub(super) presentable: bool, + srgb_present_program: Mutex>, +} + +impl Clone for Surface { + fn clone(&self) -> Self { + Self { + canvas: self.canvas.clone(), + webgl2_context: self.webgl2_context.clone(), + swapchain: RwLock::new(self.swapchain.read().clone()), + texture: Mutex::new(*self.texture.lock()), + presentable: self.presentable, + srgb_present_program: Mutex::new(*self.srgb_present_program.lock()), + } + } +} + +#[cfg(send_sync)] +unsafe impl Sync for Surface {} +#[cfg(send_sync)] +unsafe impl Send for Surface {} + +#[derive(Clone, Debug)] +enum Canvas { + Canvas(web_sys::HtmlCanvasElement), + Offscreen(web_sys::OffscreenCanvas), +} + +#[derive(Clone, Debug)] +pub struct Swapchain { + pub(crate) extent: wgt::Extent3d, + // pub(crate) channel: f::ChannelType, + pub(super) format: wgt::TextureFormat, + pub(super) framebuffer: glow::Framebuffer, + pub(super) format_desc: TextureFormatDesc, +} + +impl Surface { + pub(super) unsafe fn present( + &self, + _suf_texture: super::Texture, + context: &AdapterContext, + ) -> Result<(), crate::SurfaceError> { + let gl = &context.glow_context; + let swapchain = self.swapchain.read(); + let swapchain = swapchain.as_ref().ok_or(crate::SurfaceError::Other( + "need to configure surface before presenting", + ))?; + + if swapchain.format.is_srgb() { + // Important to set the viewport since we don't know in what state the user left it. + unsafe { + gl.viewport( + 0, + 0, + swapchain.extent.width as _, + swapchain.extent.height as _, + ) + }; + unsafe { gl.bind_framebuffer(glow::DRAW_FRAMEBUFFER, None) }; + unsafe { gl.bind_sampler(0, None) }; + unsafe { gl.active_texture(glow::TEXTURE0) }; + unsafe { gl.bind_texture(glow::TEXTURE_2D, *self.texture.lock()) }; + unsafe { gl.use_program(*self.srgb_present_program.lock()) }; + unsafe { gl.disable(glow::DEPTH_TEST) }; + unsafe { gl.disable(glow::STENCIL_TEST) }; + unsafe { gl.disable(glow::SCISSOR_TEST) }; + unsafe { gl.disable(glow::BLEND) }; + unsafe { gl.disable(glow::CULL_FACE) }; + unsafe { gl.draw_buffers(&[glow::BACK]) }; + unsafe { gl.draw_arrays(glow::TRIANGLES, 0, 3) }; + } else { + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, Some(swapchain.framebuffer)) }; + unsafe { gl.bind_framebuffer(glow::DRAW_FRAMEBUFFER, None) }; + // Note the Y-flipping here. GL's presentation is not flipped, + // but main rendering is. Therefore, we Y-flip the output positions + // in the shader, and also this blit. + unsafe { + gl.blit_framebuffer( + 0, + swapchain.extent.height as i32, + swapchain.extent.width as i32, + 0, + 0, + 0, + swapchain.extent.width as i32, + swapchain.extent.height as i32, + glow::COLOR_BUFFER_BIT, + glow::NEAREST, + ) + }; + } + + Ok(()) + } + + unsafe fn create_srgb_present_program(gl: &glow::Context) -> glow::Program { + let program = unsafe { gl.create_program() }.expect("Could not create shader program"); + let vertex = + unsafe { gl.create_shader(glow::VERTEX_SHADER) }.expect("Could not create shader"); + unsafe { gl.shader_source(vertex, include_str!("./shaders/srgb_present.vert")) }; + unsafe { gl.compile_shader(vertex) }; + let fragment = + unsafe { gl.create_shader(glow::FRAGMENT_SHADER) }.expect("Could not create shader"); + unsafe { gl.shader_source(fragment, include_str!("./shaders/srgb_present.frag")) }; + unsafe { gl.compile_shader(fragment) }; + unsafe { gl.attach_shader(program, vertex) }; + unsafe { gl.attach_shader(program, fragment) }; + unsafe { gl.link_program(program) }; + unsafe { gl.delete_shader(vertex) }; + unsafe { gl.delete_shader(fragment) }; + unsafe { gl.bind_texture(glow::TEXTURE_2D, None) }; + + program + } + + pub fn supports_srgb(&self) -> bool { + // present.frag takes care of handling srgb conversion + true + } +} + +impl crate::Surface for Surface { + type A = super::Api; + + unsafe fn configure( + &self, + device: &super::Device, + config: &crate::SurfaceConfiguration, + ) -> Result<(), crate::SurfaceError> { + match self.canvas { + Canvas::Canvas(ref canvas) => { + canvas.set_width(config.extent.width); + canvas.set_height(config.extent.height); + } + Canvas::Offscreen(ref canvas) => { + canvas.set_width(config.extent.width); + canvas.set_height(config.extent.height); + } + } + + let gl = &device.shared.context.lock(); + + { + let mut swapchain = self.swapchain.write(); + if let Some(swapchain) = swapchain.take() { + // delete all frame buffers already allocated + unsafe { gl.delete_framebuffer(swapchain.framebuffer) }; + } + } + { + let mut srgb_present_program = self.srgb_present_program.lock(); + if srgb_present_program.is_none() && config.format.is_srgb() { + *srgb_present_program = Some(unsafe { Self::create_srgb_present_program(gl) }); + } + } + { + let mut texture = self.texture.lock(); + if let Some(texture) = texture.take() { + unsafe { gl.delete_texture(texture) }; + } + + *texture = Some(unsafe { gl.create_texture() }.map_err(|error| { + log::error!("Internal swapchain texture creation failed: {error}"); + crate::DeviceError::OutOfMemory + })?); + + let desc = device.shared.describe_texture_format(config.format); + unsafe { gl.bind_texture(glow::TEXTURE_2D, *texture) }; + unsafe { + gl.tex_parameter_i32( + glow::TEXTURE_2D, + glow::TEXTURE_MIN_FILTER, + glow::NEAREST as _, + ) + }; + unsafe { + gl.tex_parameter_i32( + glow::TEXTURE_2D, + glow::TEXTURE_MAG_FILTER, + glow::NEAREST as _, + ) + }; + unsafe { + gl.tex_storage_2d( + glow::TEXTURE_2D, + 1, + desc.internal, + config.extent.width as i32, + config.extent.height as i32, + ) + }; + + let framebuffer = unsafe { gl.create_framebuffer() }.map_err(|error| { + log::error!("Internal swapchain framebuffer creation failed: {error}"); + crate::DeviceError::OutOfMemory + })?; + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, Some(framebuffer)) }; + unsafe { + gl.framebuffer_texture_2d( + glow::READ_FRAMEBUFFER, + glow::COLOR_ATTACHMENT0, + glow::TEXTURE_2D, + *texture, + 0, + ) + }; + unsafe { gl.bind_texture(glow::TEXTURE_2D, None) }; + + let mut swapchain = self.swapchain.write(); + *swapchain = Some(Swapchain { + extent: config.extent, + // channel: config.format.base_format().1, + format: config.format, + format_desc: desc, + framebuffer, + }); + } + + Ok(()) + } + + unsafe fn unconfigure(&self, device: &super::Device) { + let gl = device.shared.context.lock(); + { + let mut swapchain = self.swapchain.write(); + if let Some(swapchain) = swapchain.take() { + unsafe { gl.delete_framebuffer(swapchain.framebuffer) }; + } + } + if let Some(renderbuffer) = self.texture.lock().take() { + unsafe { gl.delete_texture(renderbuffer) }; + } + } + + unsafe fn acquire_texture( + &self, + _timeout_ms: Option, //TODO + _fence: &super::Fence, + ) -> Result, crate::SurfaceError> { + let swapchain = self.swapchain.read(); + let sc = swapchain.as_ref().unwrap(); + let texture = super::Texture { + inner: super::TextureInner::Texture { + raw: self.texture.lock().unwrap(), + target: glow::TEXTURE_2D, + }, + drop_guard: None, + array_layer_count: 1, + mip_level_count: 1, + format: sc.format, + format_desc: sc.format_desc.clone(), + copy_size: crate::CopyExtent { + width: sc.extent.width, + height: sc.extent.height, + depth: 1, + }, + }; + Ok(crate::AcquiredSurfaceTexture { + texture, + suboptimal: false, + }) + } + + unsafe fn discard_texture(&self, _texture: super::Texture) {} +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/gles/wgl.rs b/third_party/wgpu-hal-29.0.4-patched/src/gles/wgl.rs new file mode 100644 index 00000000..dba309e5 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/gles/wgl.rs @@ -0,0 +1,900 @@ +use alloc::{borrow::ToOwned as _, ffi::CString, string::String, sync::Arc, vec::Vec}; +use core::{ + ffi::{c_int, c_void, CStr}, + mem::{self, ManuallyDrop}, + ptr, + time::Duration, +}; +use std::{ + sync::{ + mpsc::{sync_channel, SyncSender}, + LazyLock, + }, + thread, +}; + +use glow::HasContext; +use glutin_wgl_sys::wgl_extra::{ + Wgl, CONTEXT_CORE_PROFILE_BIT_ARB, CONTEXT_DEBUG_BIT_ARB, CONTEXT_FLAGS_ARB, + CONTEXT_PROFILE_MASK_ARB, +}; +use hashbrown::HashSet; +use parking_lot::{Mutex, MutexGuard, RwLock}; +use raw_window_handle::{RawDisplayHandle, RawWindowHandle}; +use wgt::InstanceFlags; +use windows::{ + core::{Error, PCSTR}, + Win32::{ + Foundation, + Graphics::{Gdi, OpenGL}, + System::LibraryLoader, + UI::WindowsAndMessaging, + }, +}; + +/// The amount of time to wait while trying to obtain a lock to the adapter context +const CONTEXT_LOCK_TIMEOUT_SECS: u64 = 1; + +/// A wrapper around a `[`glow::Context`]` and the required WGL context that uses locking to +/// guarantee exclusive access when shared with multiple threads. +pub struct AdapterContext { + inner: Arc>, +} + +unsafe impl Sync for AdapterContext {} +unsafe impl Send for AdapterContext {} + +impl AdapterContext { + pub fn is_owned(&self) -> bool { + true + } + + pub fn raw_context(&self) -> *mut c_void { + match self.inner.lock().context { + Some(ref wgl) => wgl.context.0, + None => ptr::null_mut(), + } + } + + /// Obtain a lock to the WGL context and get handle to the [`glow::Context`] that can be used to + /// do rendering. + #[track_caller] + pub fn lock(&self) -> AdapterContextLock<'_> { + let inner = self + .inner + // Don't lock forever. If it takes longer than 1 second to get the lock we've got a + // deadlock and should panic to show where we got stuck + .try_lock_for(Duration::from_secs(CONTEXT_LOCK_TIMEOUT_SECS)) + .expect("Could not lock adapter context. This is most-likely a deadlock."); + + if let Some(wgl) = &inner.context { + wgl.make_current(inner.device.dc).unwrap() + }; + + AdapterContextLock { inner } + } + + /// Obtain a lock to the WGL context and get handle to the [`glow::Context`] that can be used to + /// do rendering. + /// + /// Unlike [`lock`](Self::lock), this accepts a device to pass to `make_current` and exposes the error + /// when `make_current` fails. + #[track_caller] + fn lock_with_dc(&self, device: Gdi::HDC) -> windows::core::Result> { + let inner = self + .inner + .try_lock_for(Duration::from_secs(CONTEXT_LOCK_TIMEOUT_SECS)) + .expect("Could not lock adapter context. This is most-likely a deadlock."); + + if let Some(wgl) = &inner.context { + wgl.make_current(device)?; + } + + Ok(AdapterContextLock { inner }) + } +} + +/// A guard containing a lock to an [`AdapterContext`], while the GL context is kept current. +pub struct AdapterContextLock<'a> { + inner: MutexGuard<'a, Inner>, +} + +impl<'a> core::ops::Deref for AdapterContextLock<'a> { + type Target = glow::Context; + + fn deref(&self) -> &Self::Target { + &self.inner.gl + } +} + +impl<'a> Drop for AdapterContextLock<'a> { + fn drop(&mut self) { + if let Some(wgl) = &self.inner.context { + wgl.unmake_current().unwrap() + } + } +} + +struct WglContext { + context: OpenGL::HGLRC, +} + +impl WglContext { + fn make_current(&self, device: Gdi::HDC) -> windows::core::Result<()> { + unsafe { OpenGL::wglMakeCurrent(device, self.context) } + } + + fn unmake_current(&self) -> windows::core::Result<()> { + if unsafe { OpenGL::wglGetCurrentContext() }.is_invalid() { + return Ok(()); + } + unsafe { OpenGL::wglMakeCurrent(Default::default(), Default::default()) } + } +} + +impl Drop for WglContext { + fn drop(&mut self) { + if let Err(e) = unsafe { OpenGL::wglDeleteContext(self.context) } { + log::error!("failed to delete WGL context: {e}"); + } + } +} + +unsafe impl Send for WglContext {} +unsafe impl Sync for WglContext {} + +struct Inner { + gl: ManuallyDrop, + device: InstanceDevice, + context: Option, +} + +impl Drop for Inner { + fn drop(&mut self) { + struct CurrentGuard<'a>(&'a WglContext); + impl Drop for CurrentGuard<'_> { + fn drop(&mut self) { + self.0.unmake_current().unwrap(); + } + } + + // Context must be current when dropped. See safety docs on + // `glow::HasContext`. + // + // NOTE: This is only set to `None` by `Adapter::new_external` which + // requires the context to be current when anything that may be holding + // the `Arc` is dropped. + let _guard = self.context.as_ref().map(|wgl| { + wgl.make_current(self.device.dc).unwrap(); + CurrentGuard(wgl) + }); + // SAFETY: Field not used after this. + unsafe { ManuallyDrop::drop(&mut self.gl) }; + } +} + +unsafe impl Send for Inner {} +unsafe impl Sync for Inner {} + +pub struct Instance { + srgb_capable: bool, + options: wgt::GlBackendOptions, + inner: Arc>, +} + +unsafe impl Send for Instance {} +unsafe impl Sync for Instance {} + +fn load_gl_func(name: &str, module: Option) -> *const c_void { + let addr = CString::new(name.as_bytes()).unwrap(); + let mut ptr = unsafe { OpenGL::wglGetProcAddress(PCSTR(addr.as_ptr().cast())) }; + if ptr.is_none() { + if let Some(module) = module { + ptr = unsafe { LibraryLoader::GetProcAddress(module, PCSTR(addr.as_ptr().cast())) }; + } + } + ptr.map_or_else(ptr::null_mut, |p| p as *mut c_void) +} + +fn get_extensions(extra: &Wgl, dc: Gdi::HDC) -> HashSet { + if extra.GetExtensionsStringARB.is_loaded() { + unsafe { CStr::from_ptr(extra.GetExtensionsStringARB(dc.0)) } + .to_str() + .unwrap_or("") + } else { + "" + } + .split(' ') + .map(|s| s.to_owned()) + .collect() +} + +unsafe fn setup_pixel_format(dc: Gdi::HDC) -> Result<(), crate::InstanceError> { + { + let format = OpenGL::PIXELFORMATDESCRIPTOR { + nVersion: 1, + nSize: size_of::() as u16, + dwFlags: OpenGL::PFD_DRAW_TO_WINDOW + | OpenGL::PFD_SUPPORT_OPENGL + | OpenGL::PFD_DOUBLEBUFFER, + iPixelType: OpenGL::PFD_TYPE_RGBA, + cColorBits: 8, + ..unsafe { mem::zeroed() } + }; + + let index = unsafe { OpenGL::ChoosePixelFormat(dc, &format) }; + if index == 0 { + return Err(crate::InstanceError::with_source( + String::from("unable to choose pixel format"), + Error::from_thread(), + )); + } + + let current = unsafe { OpenGL::GetPixelFormat(dc) }; + + if index != current { + unsafe { OpenGL::SetPixelFormat(dc, index, &format) }.map_err(|e| { + crate::InstanceError::with_source(String::from("unable to set pixel format"), e) + })?; + } + } + + { + let index = unsafe { OpenGL::GetPixelFormat(dc) }; + if index == 0 { + return Err(crate::InstanceError::with_source( + String::from("unable to get pixel format index"), + Error::from_thread(), + )); + } + let mut format = Default::default(); + if unsafe { + OpenGL::DescribePixelFormat(dc, index, size_of_val(&format) as u32, Some(&mut format)) + } == 0 + { + return Err(crate::InstanceError::with_source( + String::from("unable to read pixel format"), + Error::from_thread(), + )); + } + + if !format.dwFlags.contains(OpenGL::PFD_SUPPORT_OPENGL) + || format.iPixelType != OpenGL::PFD_TYPE_RGBA + { + return Err(crate::InstanceError::new(String::from( + "unsuitable pixel format", + ))); + } + } + Ok(()) +} + +fn create_global_window_class() -> Result { + let instance = unsafe { LibraryLoader::GetModuleHandleA(None) }.map_err(|e| { + crate::InstanceError::with_source(String::from("unable to get executable instance"), e) + })?; + + // Use the address of `UNIQUE` as part of the window class name to ensure different + // `wgpu` versions use different names. + static UNIQUE: Mutex = Mutex::new(0); + let class_addr: *const _ = &UNIQUE; + let name = format!("wgpu Device Class {:x}\0", class_addr as usize); + let name = CString::from_vec_with_nul(name.into_bytes()).unwrap(); + + // The window class may already be registered if we are a dynamic library that got + // unloaded & loaded back into the same process. If so, just skip creation. + let already_exists = unsafe { + let mut wc = mem::zeroed::(); + WindowsAndMessaging::GetClassInfoExA( + Some(instance.into()), + PCSTR(name.as_ptr().cast()), + &mut wc, + ) + .is_ok() + }; + if already_exists { + return Ok(name); + } + + // Use a wrapper function for compatibility with `windows-rs`. + unsafe extern "system" fn wnd_proc( + window: Foundation::HWND, + msg: u32, + wparam: Foundation::WPARAM, + lparam: Foundation::LPARAM, + ) -> Foundation::LRESULT { + unsafe { WindowsAndMessaging::DefWindowProcA(window, msg, wparam, lparam) } + } + + let window_class = WindowsAndMessaging::WNDCLASSEXA { + cbSize: size_of::() as u32, + style: WindowsAndMessaging::CS_OWNDC, + lpfnWndProc: Some(wnd_proc), + cbClsExtra: 0, + cbWndExtra: 0, + hInstance: instance.into(), + hIcon: WindowsAndMessaging::HICON::default(), + hCursor: WindowsAndMessaging::HCURSOR::default(), + hbrBackground: Gdi::HBRUSH::default(), + lpszMenuName: PCSTR::null(), + lpszClassName: PCSTR(name.as_ptr().cast()), + hIconSm: WindowsAndMessaging::HICON::default(), + }; + + let atom = unsafe { WindowsAndMessaging::RegisterClassExA(&window_class) }; + + if atom == 0 { + return Err(crate::InstanceError::with_source( + String::from("unable to register window class"), + Error::from_thread(), + )); + } + + // We intentionally leak the window class as we only need one per process. + + Ok(name) +} + +fn get_global_window_class() -> Result { + static GLOBAL: LazyLock> = + LazyLock::new(create_global_window_class); + GLOBAL.clone() +} + +struct InstanceDevice { + dc: Gdi::HDC, + + /// This is used to keep the thread owning `dc` alive until this struct is dropped. + _tx: SyncSender<()>, +} + +fn create_instance_device() -> Result { + #[derive(Clone, Copy)] + // TODO: We can get these SendSync definitions in the upstream metadata if this is the case + struct SendDc(Gdi::HDC); + unsafe impl Sync for SendDc {} + unsafe impl Send for SendDc {} + + struct Window { + window: Foundation::HWND, + } + impl Drop for Window { + fn drop(&mut self) { + if let Err(e) = unsafe { WindowsAndMessaging::DestroyWindow(self.window) } { + log::error!("failed to destroy window: {e}"); + } + } + } + + let window_class = get_global_window_class()?; + + let (drop_tx, drop_rx) = sync_channel(0); + let (setup_tx, setup_rx) = sync_channel(0); + + // We spawn a thread which owns the hidden window for this instance. + thread::Builder::new() + .stack_size(256 * 1024) + .name("wgpu-hal WGL Instance Thread".to_owned()) + .spawn(move || { + let setup = (|| { + let instance = unsafe { LibraryLoader::GetModuleHandleA(None) }.map_err(|e| { + crate::InstanceError::with_source( + String::from("unable to get executable instance"), + e, + ) + })?; + + // Create a hidden window since we don't pass `WS_VISIBLE`. + let window = unsafe { + WindowsAndMessaging::CreateWindowExA( + WindowsAndMessaging::WINDOW_EX_STYLE::default(), + PCSTR(window_class.as_ptr().cast()), + PCSTR(window_class.as_ptr().cast()), + WindowsAndMessaging::WINDOW_STYLE::default(), + 0, + 0, + 1, + 1, + None, + None, + Some(instance.into()), + None, + ) + } + .map_err(|e| { + crate::InstanceError::with_source( + String::from("unable to create hidden instance window"), + e, + ) + })?; + let window = Window { window }; + + let dc = unsafe { Gdi::GetDC(Some(window.window)) }; + if dc.is_invalid() { + return Err(crate::InstanceError::with_source( + String::from("unable to create memory device"), + Error::from_thread(), + )); + } + let dc = DeviceContextHandle { + device: dc, + window: window.window, + }; + unsafe { setup_pixel_format(dc.device)? }; + + Ok((window, dc)) + })(); + + match setup { + Ok((_window, dc)) => { + setup_tx.send(Ok(SendDc(dc.device))).unwrap(); + // Wait for the shutdown event to free the window and device context handle. + drop_rx.recv().ok(); + } + Err(err) => { + setup_tx.send(Err(err)).unwrap(); + } + } + }) + .map_err(|e| { + crate::InstanceError::with_source(String::from("unable to create instance thread"), e) + })?; + + let dc = setup_rx.recv().unwrap()?.0; + + Ok(InstanceDevice { dc, _tx: drop_tx }) +} + +impl crate::Instance for Instance { + type A = super::Api; + + unsafe fn init(desc: &crate::InstanceDescriptor<'_>) -> Result { + profiling::scope!("Init OpenGL (WGL) Backend"); + let opengl_module = + unsafe { LibraryLoader::LoadLibraryA(PCSTR(c"opengl32.dll".as_ptr().cast())) } + .map_err(|e| { + crate::InstanceError::with_source( + String::from("unable to load the OpenGL library"), + e, + ) + })?; + + let device = create_instance_device()?; + let dc = device.dc; + + let context = unsafe { OpenGL::wglCreateContext(dc) }.map_err(|e| { + crate::InstanceError::with_source( + String::from("unable to create initial OpenGL context"), + e, + ) + })?; + let context = WglContext { context }; + context.make_current(dc).map_err(|e| { + crate::InstanceError::with_source( + String::from("unable to set initial OpenGL context as current"), + e, + ) + })?; + + let extra = Wgl::load_with(|name| load_gl_func(name, None)); + let extensions = get_extensions(&extra, dc); + + let can_use_profile = extensions.contains("WGL_ARB_create_context_profile") + && extra.CreateContextAttribsARB.is_loaded(); + + let context = if can_use_profile { + let attributes = [ + CONTEXT_PROFILE_MASK_ARB as c_int, + CONTEXT_CORE_PROFILE_BIT_ARB as c_int, + CONTEXT_FLAGS_ARB as c_int, + if desc.flags.contains(InstanceFlags::DEBUG) { + CONTEXT_DEBUG_BIT_ARB as c_int + } else { + 0 + }, + 0, // End of list + ]; + let context = + unsafe { extra.CreateContextAttribsARB(dc.0, ptr::null(), attributes.as_ptr()) }; + if context.is_null() { + return Err(crate::InstanceError::with_source( + String::from("unable to create OpenGL context"), + Error::from_thread(), + )); + } + WglContext { + context: OpenGL::HGLRC(context.cast_mut()), + } + } else { + context + }; + + context.make_current(dc).map_err(|e| { + crate::InstanceError::with_source( + String::from("unable to set OpenGL context as current"), + e, + ) + })?; + + let mut gl = unsafe { + glow::Context::from_loader_function(|name| load_gl_func(name, Some(opengl_module))) + }; + + let extra = Wgl::load_with(|name| load_gl_func(name, None)); + let extensions = get_extensions(&extra, dc); + + let srgb_capable = extensions.contains("WGL_EXT_framebuffer_sRGB") + || extensions.contains("WGL_ARB_framebuffer_sRGB") + || gl + .supported_extensions() + .contains("GL_ARB_framebuffer_sRGB"); + + // In contrast to OpenGL ES, OpenGL requires explicitly enabling sRGB conversions, + // as otherwise the user has to do the sRGB conversion. + if srgb_capable { + unsafe { gl.enable(glow::FRAMEBUFFER_SRGB) }; + } + + if desc.flags.contains(InstanceFlags::VALIDATION) && gl.supports_debug() { + log::debug!("Enabling GL debug output"); + unsafe { gl.enable(glow::DEBUG_OUTPUT) }; + unsafe { gl.debug_message_callback(super::gl_debug_message_callback) }; + } + + // Wrap in ManuallyDrop to make it easier to "current" the GL context before dropping this + // GLOW context, which could also happen if a panic occurs after we uncurrent the context + // below but before Inner is constructed. + let gl = ManuallyDrop::new(gl); + context.unmake_current().map_err(|e| { + crate::InstanceError::with_source( + String::from("unable to unset the current WGL context"), + e, + ) + })?; + + Ok(Instance { + inner: Arc::new(Mutex::new(Inner { + device, + gl, + context: Some(context), + })), + options: desc.backend_options.gl.clone(), + srgb_capable, + }) + } + + unsafe fn create_surface( + &self, + display_handle: RawDisplayHandle, + window_handle: RawWindowHandle, + ) -> Result { + assert!(matches!(display_handle, RawDisplayHandle::Windows(_))); + let window = if let RawWindowHandle::Win32(handle) = window_handle { + handle + } else { + return Err(crate::InstanceError::new(format!( + "unsupported window: {window_handle:?}" + ))); + }; + Ok(Surface { + // This cast exists because of https://github.com/rust-windowing/raw-window-handle/issues/171 + window: Foundation::HWND(window.hwnd.get() as *mut _), + presentable: true, + swapchain: RwLock::new(None), + srgb_capable: self.srgb_capable, + }) + } + + unsafe fn enumerate_adapters( + &self, + _surface_hint: Option<&Surface>, + ) -> Vec> { + unsafe { + super::Adapter::expose( + AdapterContext { + inner: self.inner.clone(), + }, + self.options.clone(), + ) + } + .into_iter() + .collect() + } +} + +impl super::Adapter { + /// Creates a new external adapter using the specified loader function. + /// + /// # Safety + /// + /// - The underlying OpenGL ES context must be current. + /// - The underlying OpenGL ES context must be current when interfacing with any objects returned by + /// wgpu-hal from this adapter. + /// - The underlying OpenGL ES context must be current when dropping this adapter and when + /// dropping any objects returned from this adapter. + pub unsafe fn new_external( + fun: impl FnMut(&str) -> *const c_void, + options: wgt::GlBackendOptions, + ) -> Option> { + let context = unsafe { glow::Context::from_loader_function(fun) }; + unsafe { + Self::expose( + AdapterContext { + inner: Arc::new(Mutex::new(Inner { + gl: ManuallyDrop::new(context), + device: create_instance_device().ok()?, + context: None, + })), + }, + options, + ) + } + } + + pub fn adapter_context(&self) -> &AdapterContext { + &self.shared.context + } +} + +impl super::Device { + /// Returns the underlying WGL context. + pub fn context(&self) -> &AdapterContext { + &self.shared.context + } +} + +struct DeviceContextHandle { + device: Gdi::HDC, + window: Foundation::HWND, +} + +impl Drop for DeviceContextHandle { + fn drop(&mut self) { + unsafe { + Gdi::ReleaseDC(Some(self.window), self.device); + }; + } +} + +pub struct Swapchain { + framebuffer: glow::Framebuffer, + renderbuffer: glow::Renderbuffer, + + /// Extent because the window lies + extent: wgt::Extent3d, + + format: wgt::TextureFormat, + format_desc: super::TextureFormatDesc, + #[allow(unused)] + sample_type: wgt::TextureSampleType, +} + +pub struct Surface { + window: Foundation::HWND, + pub(super) presentable: bool, + swapchain: RwLock>, + srgb_capable: bool, +} + +unsafe impl Send for Surface {} +unsafe impl Sync for Surface {} + +impl Surface { + pub(super) unsafe fn present( + &self, + _suf_texture: super::Texture, + context: &AdapterContext, + ) -> Result<(), crate::SurfaceError> { + let swapchain = self.swapchain.read(); + let sc = swapchain.as_ref().unwrap(); + let dc = unsafe { Gdi::GetDC(Some(self.window)) }; + if dc.is_invalid() { + log::error!( + "unable to get the device context from window: {}", + Error::from_thread() + ); + return Err(crate::SurfaceError::Other( + "unable to get the device context from window", + )); + } + let dc = DeviceContextHandle { + device: dc, + window: self.window, + }; + + let gl = context.lock_with_dc(dc.device).map_err(|e| { + log::error!("unable to make the OpenGL context current for surface: {e}",); + crate::SurfaceError::Other("unable to make the OpenGL context current for surface") + })?; + + unsafe { gl.bind_framebuffer(glow::DRAW_FRAMEBUFFER, None) }; + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, Some(sc.framebuffer)) }; + + if self.srgb_capable { + // Disable sRGB conversions for `glBlitFramebuffer` as behavior does diverge between + // drivers and formats otherwise and we want to ensure no sRGB conversions happen. + unsafe { gl.disable(glow::FRAMEBUFFER_SRGB) }; + } + + // Note the Y-flipping here. GL's presentation is not flipped, + // but main rendering is. Therefore, we Y-flip the output positions + // in the shader, and also this blit. + unsafe { + gl.blit_framebuffer( + 0, + sc.extent.height as i32, + sc.extent.width as i32, + 0, + 0, + 0, + sc.extent.width as i32, + sc.extent.height as i32, + glow::COLOR_BUFFER_BIT, + glow::NEAREST, + ) + }; + + if self.srgb_capable { + unsafe { gl.enable(glow::FRAMEBUFFER_SRGB) }; + } + + unsafe { gl.bind_renderbuffer(glow::RENDERBUFFER, None) }; + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, None) }; + + if let Err(e) = unsafe { OpenGL::SwapBuffers(dc.device) } { + log::error!("unable to swap buffers: {e}"); + return Err(crate::SurfaceError::Other("unable to swap buffers")); + } + + Ok(()) + } + + pub fn supports_srgb(&self) -> bool { + self.srgb_capable + } +} + +impl crate::Surface for Surface { + type A = super::Api; + + unsafe fn configure( + &self, + device: &super::Device, + config: &crate::SurfaceConfiguration, + ) -> Result<(), crate::SurfaceError> { + // Remove the old configuration. + unsafe { self.unconfigure(device) }; + + let dc = unsafe { Gdi::GetDC(Some(self.window)) }; + if dc.is_invalid() { + log::error!( + "unable to get the device context from window: {}", + Error::from_thread() + ); + return Err(crate::SurfaceError::Other( + "unable to get the device context from window", + )); + } + let dc = DeviceContextHandle { + device: dc, + window: self.window, + }; + + if let Err(e) = unsafe { setup_pixel_format(dc.device) } { + log::error!("unable to setup surface pixel format: {e}",); + return Err(crate::SurfaceError::Other( + "unable to setup surface pixel format", + )); + } + + let format_desc = device.shared.describe_texture_format(config.format); + let gl = &device.shared.context.lock_with_dc(dc.device).map_err(|e| { + log::error!("unable to make the OpenGL context current for surface: {e}",); + crate::SurfaceError::Other("unable to make the OpenGL context current for surface") + })?; + + let renderbuffer = unsafe { gl.create_renderbuffer() }.map_err(|error| { + log::error!("Internal swapchain renderbuffer creation failed: {error}"); + crate::DeviceError::OutOfMemory + })?; + unsafe { gl.bind_renderbuffer(glow::RENDERBUFFER, Some(renderbuffer)) }; + unsafe { + gl.renderbuffer_storage( + glow::RENDERBUFFER, + format_desc.internal, + config.extent.width as _, + config.extent.height as _, + ) + }; + + let framebuffer = unsafe { gl.create_framebuffer() }.map_err(|error| { + log::error!("Internal swapchain framebuffer creation failed: {error}"); + crate::DeviceError::OutOfMemory + })?; + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, Some(framebuffer)) }; + unsafe { + gl.framebuffer_renderbuffer( + glow::READ_FRAMEBUFFER, + glow::COLOR_ATTACHMENT0, + glow::RENDERBUFFER, + Some(renderbuffer), + ) + }; + unsafe { gl.bind_renderbuffer(glow::RENDERBUFFER, None) }; + unsafe { gl.bind_framebuffer(glow::READ_FRAMEBUFFER, None) }; + + // Setup presentation mode + let extra = Wgl::load_with(|name| load_gl_func(name, None)); + let extensions = get_extensions(&extra, dc.device); + if !(extensions.contains("WGL_EXT_swap_control") && extra.SwapIntervalEXT.is_loaded()) { + log::error!("WGL_EXT_swap_control is unsupported"); + return Err(crate::SurfaceError::Other( + "WGL_EXT_swap_control is unsupported", + )); + } + + let vsync = match config.present_mode { + wgt::PresentMode::Immediate => false, + wgt::PresentMode::Fifo => true, + _ => { + log::error!("unsupported present mode: {:?}", config.present_mode); + return Err(crate::SurfaceError::Other("unsupported present mode")); + } + }; + + if unsafe { extra.SwapIntervalEXT(if vsync { 1 } else { 0 }) } == Foundation::FALSE.0 { + log::error!("unable to set swap interval: {}", Error::from_thread()); + return Err(crate::SurfaceError::Other("unable to set swap interval")); + } + + self.swapchain.write().replace(Swapchain { + renderbuffer, + framebuffer, + extent: config.extent, + format: config.format, + format_desc, + sample_type: wgt::TextureSampleType::Float { filterable: false }, + }); + + Ok(()) + } + + unsafe fn unconfigure(&self, device: &super::Device) { + let gl = &device.shared.context.lock(); + if let Some(sc) = self.swapchain.write().take() { + unsafe { + gl.delete_renderbuffer(sc.renderbuffer); + gl.delete_framebuffer(sc.framebuffer) + }; + } + } + + unsafe fn acquire_texture( + &self, + _timeout_ms: Option, + _fence: &super::Fence, + ) -> Result, crate::SurfaceError> { + let swapchain = self.swapchain.read(); + let sc = swapchain.as_ref().unwrap(); + let texture = super::Texture { + inner: super::TextureInner::Renderbuffer { + raw: sc.renderbuffer, + }, + drop_guard: None, + array_layer_count: 1, + mip_level_count: 1, + format: sc.format, + format_desc: sc.format_desc.clone(), + copy_size: crate::CopyExtent { + width: sc.extent.width, + height: sc.extent.height, + depth: 1, + }, + }; + Ok(crate::AcquiredSurfaceTexture { + texture, + suboptimal: false, + }) + } + unsafe fn discard_texture(&self, _texture: super::Texture) {} +} diff --git a/third_party/wgpu-hal-29.0.4-patched/src/lib.rs b/third_party/wgpu-hal-29.0.4-patched/src/lib.rs new file mode 100644 index 00000000..3f36e675 --- /dev/null +++ b/third_party/wgpu-hal-29.0.4-patched/src/lib.rs @@ -0,0 +1,2852 @@ +//! A cross-platform unsafe graphics abstraction. +//! +//! This crate defines a set of traits abstracting over modern graphics APIs, +//! with implementations ("backends") for Vulkan, Metal, Direct3D, and GL. +//! +//! `wgpu-hal` is a spiritual successor to +//! [gfx-hal](https://github.com/gfx-rs/gfx), but with reduced scope, and +//! oriented towards WebGPU implementation goals. It has no overhead for +//! validation or tracking, and the API translation overhead is kept to the bare +//! minimum by the design of WebGPU. This API can be used for resource-demanding +//! applications and engines. +//! +//! The `wgpu-hal` crate's main design choices: +//! +//! - Our traits are meant to be *portable*: proper use +//! should get equivalent results regardless of the backend. +//! +//! - Our traits' contracts are *unsafe*: implementations perform minimal +//! validation, if any, and incorrect use will often cause undefined behavior. +//! This allows us to minimize the overhead we impose over the underlying +//! graphics system. If you need safety, the [`wgpu-core`] crate provides a +//! safe API for driving `wgpu-hal`, implementing all necessary validation, +//! resource state tracking, and so on. (Note that `wgpu-core` is designed for +//! use via FFI; the [`wgpu`] crate provides more idiomatic Rust bindings for +//! `wgpu-core`.) Or, you can do your own validation. +//! +//! - In the same vein, returned errors *only cover cases the user can't +//! anticipate*, like running out of memory or losing the device. Any errors +//! that the user could reasonably anticipate are their responsibility to +//! avoid. For example, `wgpu-hal` returns no error for mapping a buffer that's +//! not mappable: as the buffer creator, the user should already know if they +//! can map it. +//! +//! - We use *static dispatch*. The traits are not +//! generally object-safe. You must select a specific backend type +//! like [`vulkan::Api`] or [`metal::Api`], and then use that +//! according to the main traits, or call backend-specific methods. +//! +//! - We use *idiomatic Rust parameter passing*, +//! taking objects by reference, returning them by value, and so on, +//! unlike `wgpu-core`, which refers to objects by ID. +//! +//! - We map buffer contents *persistently*. This means that the buffer can +//! remain mapped on the CPU while the GPU reads or writes to it. You must +//! explicitly indicate when data might need to be transferred between CPU and +//! GPU, if [`Device::map_buffer`] indicates that this is necessary. +//! +//! - You must record *explicit barriers* between different usages of a +//! resource. For example, if a buffer is written to by a compute +//! shader, and then used as and index buffer to a draw call, you +//! must use [`CommandEncoder::transition_buffers`] between those two +//! operations. +//! +//! - Pipeline layouts are *explicitly specified* when setting bind groups. +//! Incompatible layouts disturb groups bound at higher indices. +//! +//! - The API *accepts collections as iterators*, to avoid forcing the user to +//! store data in particular containers. The implementation doesn't guarantee +//! that any of the iterators are drained, unless stated otherwise by the +//! function documentation. For this reason, we recommend that iterators don't +//! do any mutating work. +//! +//! Unfortunately, `wgpu-hal`'s safety requirements are not fully documented. +//! Ideally, all trait methods would have doc comments setting out the +//! requirements users must meet to ensure correct and portable behavior. If you +//! are aware of a specific requirement that a backend imposes that is not +//! ensured by the traits' documented rules, please file an issue. Or, if you are +//! a capable technical writer, please file a pull request! +//! +//! [`wgpu-core`]: https://crates.io/crates/wgpu-core +//! [`wgpu`]: https://crates.io/crates/wgpu +//! [`vulkan::Api`]: vulkan/struct.Api.html +//! [`metal::Api`]: metal/struct.Api.html +//! +//! ## Primary backends +//! +//! The `wgpu-hal` crate has full-featured backends implemented on the following +//! platform graphics APIs: +//! +//! - Vulkan, available on Linux, Android, and Windows, using the [`ash`] crate's +//! Vulkan bindings. It's also available on macOS, if you install [MoltenVK]. +//! +//! - Metal on macOS, using the [`metal`] crate's bindings. +//! +//! - Direct3D 12 on Windows, using the [`windows`] crate's bindings. +//! +//! [`ash`]: https://crates.io/crates/ash +//! [MoltenVK]: https://github.com/KhronosGroup/MoltenVK +//! [`metal`]: https://crates.io/crates/metal +//! [`windows`]: https://crates.io/crates/windows +//! +//! ## Secondary backends +//! +//! The `wgpu-hal` crate has a partial implementation based on the following +//! platform graphics API: +//! +//! - The GL backend is available anywhere OpenGL, OpenGL ES, or WebGL are +//! available. See the [`gles`] module documentation for details. +//! +//! [`gles`]: gles/index.html +//! +//! You can see what capabilities an adapter is missing by checking the +//! [`DownlevelCapabilities`][tdc] in [`ExposedAdapter::capabilities`], available +//! from [`Instance::enumerate_adapters`]. +//! +//! The API is generally designed to fit the primary backends better than the +//! secondary backends, so the latter may impose more overhead. +//! +//! [tdc]: wgt::DownlevelCapabilities +//! +//! ## Traits +//! +//! The `wgpu-hal` crate defines a handful of traits that together +//! represent a cross-platform abstraction for modern GPU APIs. +//! +//! - The [`Api`] trait represents a `wgpu-hal` backend. It has no methods of its +//! own, only a collection of associated types. +//! +//! - [`Api::Instance`] implements the [`Instance`] trait. [`Instance::init`] +//! creates an instance value, which you can use to enumerate the adapters +//! available on the system. For example, [`vulkan::Api::Instance::init`][Ii] +//! returns an instance that can enumerate the Vulkan physical devices on your +//! system. +//! +//! - [`Api::Adapter`] implements the [`Adapter`] trait, representing a +//! particular device from a particular backend. For example, a Vulkan instance +//! might have a Lavapipe software adapter and a GPU-based adapter. +//! +//! - [`Api::Device`] implements the [`Device`] trait, representing an active +//! link to a device. You get a device value by calling [`Adapter::open`], and +//! then use it to create buffers, textures, shader modules, and so on. +//! +//! - [`Api::Queue`] implements the [`Queue`] trait, which you use to submit +//! command buffers to a given device. +//! +//! - [`Api::CommandEncoder`] implements the [`CommandEncoder`] trait, which you +//! use to build buffers of commands to submit to a queue. This has all the +//! methods for drawing and running compute shaders, which is presumably what +//! you're here for. +//! +//! - [`Api::Surface`] implements the [`Surface`] trait, which represents a +//! swapchain for presenting images on the screen, via interaction with the +//! system's window manager. +//! +//! The [`Api`] trait has various other associated types like [`Api::Buffer`] and +//! [`Api::Texture`] that represent resources the rest of the interface can +//! operate on, but these generally do not have their own traits. +//! +//! [Ii]: Instance::init +//! +//! ## Validation is the calling code's responsibility, not `wgpu-hal`'s +//! +//! As much as possible, `wgpu-hal` traits place the burden of validation, +//! resource tracking, and state tracking on the caller, not on the trait +//! implementations themselves. Anything which can reasonably be handled in +//! backend-independent code should be. A `wgpu_hal` backend's sole obligation is +//! to provide portable behavior, and report conditions that the calling code +//! can't reasonably anticipate, like device loss or running out of memory. +//! +//! The `wgpu` crate collection is intended for use in security-sensitive +//! applications, like web browsers, where the API is available to untrusted +//! code. This means that `wgpu-core`'s validation is not simply a service to +//! developers, to be provided opportunistically when the performance costs are +//! acceptable and the necessary data is ready at hand. Rather, `wgpu-core`'s +//! validation must be exhaustive, to ensure that even malicious content cannot +//! provoke and exploit undefined behavior in the platform's graphics API. +//! +//! Because graphics APIs' requirements are complex, the only practical way for +//! `wgpu` to provide exhaustive validation is to comprehensively track the +//! lifetime and state of all the resources in the system. Implementing this +//! separately for each backend is infeasible; effort would be better spent +//! making the cross-platform validation in `wgpu-core` legible and trustworthy. +//! Fortunately, the requirements are largely similar across the various +//! platforms, so cross-platform validation is practical. +//! +//! Some backends have specific requirements that aren't practical to foist off +//! on the `wgpu-hal` user. For example, properly managing macOS Objective-C or +//! Microsoft COM reference counts is best handled by using appropriate pointer +//! types within the backend. +//! +//! A desire for "defense in depth" may suggest performing additional validation +//! in `wgpu-hal` when the opportunity arises, but this must be done with +//! caution. Even experienced contributors infer the expectations their changes +//! must meet by considering not just requirements made explicit in types, tests, +//! assertions, and comments, but also those implicit in the surrounding code. +//! When one sees validation or state-tracking code in `wgpu-hal`, it is tempting +//! to conclude, "Oh, `wgpu-hal` checks for this, so `wgpu-core` needn't worry +//! about it - that would be redundant!" The responsibility for exhaustive +//! validation always rests with `wgpu-core`, regardless of what may or may not +//! be checked in `wgpu-hal`. +//! +//! To this end, any "defense in depth" validation that does appear in `wgpu-hal` +//! for requirements that `wgpu-core` should have enforced should report failure +//! via the `unreachable!` macro, because problems detected at this stage always +//! indicate a bug in `wgpu-core`. +//! +//! ## Debugging +//! +//! Most of the information on the wiki [Debugging wgpu Applications][wiki-debug] +//! page still applies to this API, with the exception of API tracing/replay +//! functionality, which is only available in `wgpu-core`. +//! +//! [wiki-debug]: https://github.com/gfx-rs/wgpu/wiki/Debugging-wgpu-Applications + +#![no_std] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![allow( + // this happens on the GL backend, where it is both thread safe and non-thread safe in the same code. + clippy::arc_with_non_send_sync, + // We don't use syntax sugar where it's not necessary. + clippy::match_like_matches_macro, + // Redundant matching is more explicit. + clippy::redundant_pattern_matching, + // Explicit lifetimes are often easier to reason about. + clippy::needless_lifetimes, + // No need for defaults in the internal types. + clippy::new_without_default, + // Matches are good and extendable, no need to make an exception here. + clippy::single_match, + // Push commands are more regular than macros. + clippy::vec_init_then_push, + // We unsafe impl `Send` for a reason. + clippy::non_send_fields_in_send_ty, + // TODO! + clippy::missing_safety_doc, + // It gets in the way a lot and does not prevent bugs in practice. + clippy::pattern_type_mismatch, + // We should investigate these. + clippy::large_enum_variant +)] +#![warn( + clippy::alloc_instead_of_core, + clippy::ptr_as_ptr, + clippy::std_instead_of_alloc, + clippy::std_instead_of_core, + trivial_casts, + trivial_numeric_casts, + unsafe_op_in_unsafe_fn, + unused_extern_crates, + unused_qualifications +)] + +extern crate alloc; +extern crate wgpu_types as wgt; +// Each of these backends needs `std` in some fashion; usually `std::thread` functions. +#[cfg(any(dx12, gles_with_std, metal, vulkan))] +#[macro_use] +extern crate std; + +/// DirectX12 API internals. +#[cfg(dx12)] +pub mod dx12; +/// GLES API internals. +#[cfg(gles)] +pub mod gles; +/// Metal API internals. +#[cfg(metal)] +pub mod metal; +/// A dummy API implementation. +// TODO(https://github.com/gfx-rs/wgpu/issues/7120): this should have a cfg +pub mod noop; +/// Vulkan API internals. +#[cfg(vulkan)] +pub mod vulkan; + +pub mod auxil; +pub mod api { + #[cfg(dx12)] + pub use super::dx12::Api as Dx12; + #[cfg(gles)] + pub use super::gles::Api as Gles; + #[cfg(metal)] + pub use super::metal::Api as Metal; + pub use super::noop::Api as Noop; + #[cfg(vulkan)] + pub use super::vulkan::Api as Vulkan; +} + +mod dynamic; +#[cfg(feature = "validation_canary")] +mod validation_canary; + +#[cfg(feature = "validation_canary")] +pub use validation_canary::{ValidationCanary, VALIDATION_CANARY}; + +pub(crate) use dynamic::impl_dyn_resource; +pub use dynamic::{ + DynAccelerationStructure, DynAcquiredSurfaceTexture, DynAdapter, DynBindGroup, + DynBindGroupLayout, DynBuffer, DynCommandBuffer, DynCommandEncoder, DynComputePipeline, + DynDevice, DynExposedAdapter, DynFence, DynInstance, DynOpenDevice, DynPipelineCache, + DynPipelineLayout, DynQuerySet, DynQueue, DynRenderPipeline, DynResource, DynSampler, + DynShaderModule, DynSurface, DynSurfaceTexture, DynTexture, DynTextureView, +}; + +#[allow(unused)] +use alloc::boxed::Box; +use alloc::{borrow::Cow, string::String, vec::Vec}; +use core::{ + borrow::Borrow, + error::Error, + fmt, + num::{NonZeroU32, NonZeroU64}, + ops::{Range, RangeInclusive}, + ptr::NonNull, +}; + +use bitflags::bitflags; +use raw_window_handle::DisplayHandle; +use thiserror::Error; +use wgt::WasmNotSendSync; + +cfg_if::cfg_if! { + if #[cfg(supports_ptr_atomics)] { + use alloc::sync::Arc; + } else if #[cfg(feature = "portable-atomic")] { + use portable_atomic_util::Arc; + } +} + +// - Vertex + Fragment +// - Compute +// Task + Mesh + Fragment +pub const MAX_CONCURRENT_SHADER_STAGES: usize = 3; +pub const MAX_ANISOTROPY: u8 = 16; +pub const MAX_BIND_GROUPS: usize = 8; +pub const MAX_VERTEX_BUFFERS: usize = 16; +pub const MAX_COLOR_ATTACHMENTS: usize = 8; +pub const MAX_MIP_LEVELS: u32 = 16; +/// Size of a single occlusion/timestamp query, when copied into a buffer, in bytes. +/// cbindgen:ignore +pub const QUERY_SIZE: wgt::BufferAddress = 8; + +pub type Label<'a> = Option<&'a str>; +pub type MemoryRange = Range; +pub type FenceValue = u64; +#[cfg(supports_64bit_atomics)] +pub type AtomicFenceValue = core::sync::atomic::AtomicU64; +#[cfg(not(supports_64bit_atomics))] +pub type AtomicFenceValue = portable_atomic::AtomicU64; + +/// A callback to signal that wgpu is no longer using a resource. +#[cfg(any(gles, vulkan))] +pub type DropCallback = Box; + +#[cfg(any(gles, vulkan))] +pub struct DropGuard { + callback: Option, +} + +#[cfg(all(any(gles, vulkan), any(native, Emscripten)))] +impl DropGuard { + fn from_option(callback: Option) -> Option { + callback.map(|callback| Self { + callback: Some(callback), + }) + } +} + +#[cfg(any(gles, vulkan))] +impl Drop for DropGuard { + fn drop(&mut self) { + if let Some(cb) = self.callback.take() { + (cb)(); + } + } +} + +#[cfg(any(gles, vulkan))] +impl fmt::Debug for DropGuard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DropGuard").finish() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum DeviceError { + #[error("Out of memory")] + OutOfMemory, + #[error("Device is lost")] + Lost, + #[error("Unexpected error variant (driver implementation is at fault)")] + Unexpected, +} + +#[cfg(any(dx12, vulkan))] +impl From for DeviceError { + fn from(result: gpu_allocator::AllocationError) -> Self { + match result { + gpu_allocator::AllocationError::OutOfMemory => Self::OutOfMemory, + gpu_allocator::AllocationError::FailedToMap(e) => { + log::error!("gpu-allocator: Failed to map: {e}"); + Self::Lost + } + gpu_allocator::AllocationError::NoCompatibleMemoryTypeFound => { + log::error!("gpu-allocator: No Compatible Memory Type Found"); + Self::Lost + } + gpu_allocator::AllocationError::InvalidAllocationCreateDesc => { + log::error!("gpu-allocator: Invalid Allocation Creation Description"); + Self::Lost + } + gpu_allocator::AllocationError::InvalidAllocatorCreateDesc(e) => { + log::error!("gpu-allocator: Invalid Allocator Creation Description: {e}"); + Self::Lost + } + + gpu_allocator::AllocationError::Internal(e) => { + log::error!("gpu-allocator: Internal Error: {e}"); + Self::Lost + } + gpu_allocator::AllocationError::BarrierLayoutNeedsDevice10 + | gpu_allocator::AllocationError::CastableFormatsRequiresEnhancedBarriers + | gpu_allocator::AllocationError::CastableFormatsRequiresAtLeastDevice12 => { + unreachable!() + } + } + } +} + +// A copy of gpu_allocator::AllocationSizes, allowing to read the configured value for +// the dx12 backend, we should instead add getters to gpu_allocator::AllocationSizes +// and remove this type. +// https://github.com/Traverse-Research/gpu-allocator/issues/295 +#[cfg_attr(not(any(dx12, vulkan)), expect(dead_code))] +pub(crate) struct AllocationSizes { + pub(crate) min_device_memblock_size: u64, + pub(crate) max_device_memblock_size: u64, + pub(crate) min_host_memblock_size: u64, + pub(crate) max_host_memblock_size: u64, +} + +impl AllocationSizes { + #[allow(dead_code)] // may be unused on some platforms + pub(crate) fn from_memory_hints(memory_hints: &wgt::MemoryHints) -> Self { + // TODO: the allocator's configuration should take hardware capability into + // account. + const MB: u64 = 1024 * 1024; + + match memory_hints { + wgt::MemoryHints::Performance => Self { + min_device_memblock_size: 128 * MB, + max_device_memblock_size: 256 * MB, + min_host_memblock_size: 64 * MB, + max_host_memblock_size: 128 * MB, + }, + wgt::MemoryHints::MemoryUsage => Self { + min_device_memblock_size: 8 * MB, + max_device_memblock_size: 64 * MB, + min_host_memblock_size: 4 * MB, + max_host_memblock_size: 32 * MB, + }, + wgt::MemoryHints::Manual { + suballocated_device_memory_block_size, + } => { + // TODO: https://github.com/gfx-rs/wgpu/issues/8625 + // Would it be useful to expose the host size in memory hints + // instead of always using half of the device size? + let device_size = suballocated_device_memory_block_size; + let host_size = device_size.start / 2..device_size.end / 2; + + // gpu_allocator clamps the sizes between 4MiB and 256MiB, but we clamp them ourselves since we use + // the sizes when detecting high memory pressure and there is no way to query the values otherwise. + Self { + min_device_memblock_size: device_size.start.clamp(4 * MB, 256 * MB), + max_device_memblock_size: device_size.end.clamp(4 * MB, 256 * MB), + min_host_memblock_size: host_size.start.clamp(4 * MB, 256 * MB), + max_host_memblock_size: host_size.end.clamp(4 * MB, 256 * MB), + } + } + } + } +} + +#[cfg(any(dx12, vulkan))] +impl From for gpu_allocator::AllocationSizes { + fn from(value: AllocationSizes) -> gpu_allocator::AllocationSizes { + gpu_allocator::AllocationSizes::new( + value.min_device_memblock_size, + value.min_host_memblock_size, + ) + .with_max_device_memblock_size(value.max_device_memblock_size) + .with_max_host_memblock_size(value.max_host_memblock_size) + } +} + +#[allow(dead_code)] // may be unused on some platforms +#[cold] +fn hal_usage_error(txt: T) -> ! { + panic!("wgpu-hal invariant was violated (usage error): {txt}") +} + +#[allow(dead_code)] // may be unused on some platforms +#[cold] +fn hal_internal_error(txt: T) -> ! { + panic!("wgpu-hal ran into a preventable internal error: {txt}") +} + +#[derive(Clone, Debug, Eq, PartialEq, Error)] +pub enum ShaderError { + #[error("Compilation failed: {0:?}")] + Compilation(String), + #[error(transparent)] + Device(#[from] DeviceError), +} + +#[derive(Clone, Debug, Eq, PartialEq, Error)] +pub enum PipelineError { + #[error("Linkage failed for stage {0:?}: {1}")] + Linkage(wgt::ShaderStages, String), + #[error("Entry point for stage {0:?} is invalid")] + EntryPoint(naga::ShaderStage), + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Pipeline constant error for stage {0:?}: {1}")] + PipelineConstants(wgt::ShaderStages, String), +} + +#[derive(Clone, Debug, Eq, PartialEq, Error)] +pub enum PipelineCacheError { + #[error(transparent)] + Device(#[from] DeviceError), +} + +#[derive(Clone, Debug, Eq, PartialEq, Error)] +pub enum SurfaceError { + #[error("Surface is lost")] + Lost, + #[error("Surface is outdated, needs to be re-created")] + Outdated, + #[error("Timed out waiting for a surface texture")] + Timeout, + #[error("The window is occluded (e.g. minimized or behind another window). Try again once the window is no longer occluded.")] + Occluded, + #[error(transparent)] + Device(#[from] DeviceError), + #[error("Other reason: {0}")] + Other(&'static str), +} + +/// Error occurring while trying to create an instance, or create a surface from an instance; +/// typically relating to the state of the underlying graphics API or hardware. +#[derive(Clone, Debug, Error)] +#[error("{message}")] +pub struct InstanceError { + /// These errors are very platform specific, so do not attempt to encode them as an enum. + /// + /// This message should describe the problem in sufficient detail to be useful for a + /// user-to-developer “why won't this work on my machine” bug report, and otherwise follow + /// . + message: String, + + /// Underlying error value, if any is available. + #[source] + source: Option>, +} + +impl InstanceError { + #[allow(dead_code)] // may be unused on some platforms + pub(crate) fn new(message: String) -> Self { + Self { + message, + source: None, + } + } + #[allow(dead_code)] // may be unused on some platforms + pub(crate) fn with_source(message: String, source: impl Error + Send + Sync + 'static) -> Self { + cfg_if::cfg_if! { + if #[cfg(supports_ptr_atomics)] { + let source = Arc::new(source); + } else { + // TODO(https://github.com/rust-lang/rust/issues/18598): avoid indirection via Box once arbitrary types support unsized coercion + let source: Box = Box::new(source); + let source = Arc::from(source); + } + } + Self { + message, + source: Some(source), + } + } +} + +/// All the types and methods that make up a implementation on top of a backend. +/// +/// Only the types that have non-dyn trait bounds have methods on them. Most methods +/// are either on [`CommandEncoder`] or [`Device`]. +/// +/// The api can either be used through generics (through use of this trait and associated +/// types) or dynamically through using the `Dyn*` traits. +pub trait Api: Clone + fmt::Debug + Sized + WasmNotSendSync + 'static { + const VARIANT: wgt::Backend; + + type Instance: DynInstance + Instance; + type Surface: DynSurface + Surface; + type Adapter: DynAdapter + Adapter; + type Device: DynDevice + Device; + + type Queue: DynQueue + Queue; + type CommandEncoder: DynCommandEncoder + CommandEncoder; + + /// This API's command buffer type. + /// + /// The only thing you can do with `CommandBuffer`s is build them + /// with a [`CommandEncoder`] and then pass them to + /// [`Queue::submit`] for execution, or destroy them by passing + /// them to [`CommandEncoder::reset_all`]. + /// + /// [`CommandEncoder`]: Api::CommandEncoder + type CommandBuffer: DynCommandBuffer; + + type Buffer: DynBuffer; + type Texture: DynTexture; + type SurfaceTexture: DynSurfaceTexture + Borrow; + type TextureView: DynTextureView; + type Sampler: DynSampler; + type QuerySet: DynQuerySet; + + /// A value you can block on to wait for something to finish. + /// + /// A `Fence` holds a monotonically increasing [`FenceValue`]. You can call + /// [`Device::wait`] to block until a fence reaches or passes a value you + /// choose. [`Queue::submit`] can take a `Fence` and a [`FenceValue`] to + /// store in it when the submitted work is complete. + /// + /// Attempting to set a fence to a value less than its current value has no + /// effect. + /// + /// Waiting on a fence returns as soon as the fence reaches *or passes* the + /// requested value. This implies that, in order to reliably determine when + /// an operation has completed, operations must finish in order of + /// increasing fence values: if a higher-valued operation were to finish + /// before a lower-valued operation, then waiting for the fence to reach the + /// lower value could return before the lower-valued operation has actually + /// finished. + type Fence: DynFence; + + type BindGroupLayout: DynBindGroupLayout; + type BindGroup: DynBindGroup; + type PipelineLayout: DynPipelineLayout; + type ShaderModule: DynShaderModule; + type RenderPipeline: DynRenderPipeline; + type ComputePipeline: DynComputePipeline; + type PipelineCache: DynPipelineCache; + + type AccelerationStructure: DynAccelerationStructure + 'static; +} + +pub trait Instance: Sized + WasmNotSendSync { + type A: Api; + + unsafe fn init(desc: &InstanceDescriptor<'_>) -> Result; + unsafe fn create_surface( + &self, + display_handle: raw_window_handle::RawDisplayHandle, + window_handle: raw_window_handle::RawWindowHandle, + ) -> Result<::Surface, InstanceError>; + /// `surface_hint` is only used by the GLES backend targeting WebGL2 + unsafe fn enumerate_adapters( + &self, + surface_hint: Option<&::Surface>, + ) -> Vec>; +} + +pub trait Surface: WasmNotSendSync { + type A: Api; + + /// Configure `self` to use `device`. + /// + /// # Safety + /// + /// - All GPU work using `self` must have been completed. + /// - All [`AcquiredSurfaceTexture`]s must have been destroyed. + /// - All [`Api::TextureView`]s derived from the [`AcquiredSurfaceTexture`]s must have been destroyed. + /// - The surface `self` must not currently be configured to use any other [`Device`]. + unsafe fn configure( + &self, + device: &::Device, + config: &SurfaceConfiguration, + ) -> Result<(), SurfaceError>; + + /// Unconfigure `self` on `device`. + /// + /// # Safety + /// + /// - All GPU work that uses `surface` must have been completed. + /// - All [`AcquiredSurfaceTexture`]s must have been destroyed. + /// - All [`Api::TextureView`]s derived from the [`AcquiredSurfaceTexture`]s must have been destroyed. + /// - The surface `self` must have been configured on `device`. + unsafe fn unconfigure(&self, device: &::Device); + + /// Return the next texture to be presented by `self`, for the caller to draw on. + /// + /// On success, return an [`AcquiredSurfaceTexture`] representing the + /// texture into which the caller should draw the image to be displayed on + /// `self`. + /// + /// If `timeout` elapses before `self` has a texture ready to be acquired, + /// return `Err(SurfaceError::Timeout)`. If `timeout` is `None`, wait + /// indefinitely, with no timeout. + /// + /// # Using an [`AcquiredSurfaceTexture`] + /// + /// On success, this function returns an [`AcquiredSurfaceTexture`] whose + /// [`texture`] field is a [`SurfaceTexture`] from which the caller can + /// [`borrow`] a [`Texture`] to draw on. The [`AcquiredSurfaceTexture`] also + /// carries some metadata about that [`SurfaceTexture`]. + /// + /// All calls to [`Queue::submit`] that draw on that [`Texture`] must also + /// include the [`SurfaceTexture`] in the `surface_textures` argument. + /// + /// When you are done drawing on the texture, you can display it on `self` + /// by passing the [`SurfaceTexture`] and `self` to [`Queue::present`]. + /// + /// If you do not wish to display the texture, you must pass the + /// [`SurfaceTexture`] to [`self.discard_texture`], so that it can be reused + /// by future acquisitions. + /// + /// # Portability + /// + /// Some backends can't support a timeout when acquiring a texture. On these + /// backends, `timeout` is ignored. + /// + /// On macOS, this returns `Err(SurfaceError::Timeout)` when the window is + /// not visible (minimized, fully occluded, or on another virtual desktop) + /// to avoid blocking in `CAMetalLayer.nextDrawable()`. + /// + /// # Safety + /// + /// - The surface `self` must currently be configured on some [`Device`]. + /// + /// - The `fence` argument must be the same [`Fence`] passed to all calls to + /// [`Queue::submit`] that used [`Texture`]s acquired from this surface. + /// + /// - You may only have one texture acquired from `self` at a time. When + /// `acquire_texture` returns `Ok(ast)`, you must pass the returned + /// [`SurfaceTexture`] `ast.texture` to either [`Queue::present`] or + /// [`Surface::discard_texture`] before calling `acquire_texture` again. + /// + /// [`texture`]: AcquiredSurfaceTexture::texture + /// [`SurfaceTexture`]: Api::SurfaceTexture + /// [`borrow`]: alloc::borrow::Borrow::borrow + /// [`Texture`]: Api::Texture + /// [`Fence`]: Api::Fence + /// [`self.discard_texture`]: Surface::discard_texture + unsafe fn acquire_texture( + &self, + timeout: Option, + fence: &::Fence, + ) -> Result, SurfaceError>; + + /// Relinquish an acquired texture without presenting it. + /// + /// After this call, the texture underlying [`SurfaceTexture`] may be + /// returned by subsequent calls to [`self.acquire_texture`]. + /// + /// # Safety + /// + /// - The surface `self` must currently be configured on some [`Device`]. + /// + /// - `texture` must be a [`SurfaceTexture`] returned by a call to + /// [`self.acquire_texture`] that has not yet been passed to + /// [`Queue::present`]. + /// + /// [`SurfaceTexture`]: Api::SurfaceTexture + /// [`self.acquire_texture`]: Surface::acquire_texture + unsafe fn discard_texture(&self, texture: ::SurfaceTexture); +} + +pub trait Adapter: WasmNotSendSync { + type A: Api; + + unsafe fn open( + &self, + features: wgt::Features, + limits: &wgt::Limits, + memory_hints: &wgt::MemoryHints, + ) -> Result, DeviceError>; + + /// Return the set of supported capabilities for a texture format. + unsafe fn texture_format_capabilities( + &self, + format: wgt::TextureFormat, + ) -> TextureFormatCapabilities; + + /// Returns the capabilities of working with a specified surface. + /// + /// `None` means presentation is not supported for it. + unsafe fn surface_capabilities( + &self, + surface: &::Surface, + ) -> Option; + + /// Creates a [`PresentationTimestamp`] using the adapter's WSI. + /// + /// [`PresentationTimestamp`]: wgt::PresentationTimestamp + unsafe fn get_presentation_timestamp(&self) -> wgt::PresentationTimestamp; + + /// The combination of all usages that the are guaranteed to be be ordered by the hardware. + /// If a usage is ordered, then if the buffer state doesn't change between draw calls, + /// there are no barriers needed for synchronization. + fn get_ordered_buffer_usages(&self) -> wgt::BufferUses; + + /// The combination of all usages that the are guaranteed to be be ordered by the hardware. + /// If a usage is ordered, then if the buffer state doesn't change between draw calls, + /// there are no barriers needed for synchronization. + fn get_ordered_texture_usages(&self) -> wgt::TextureUses; +} + +/// A connection to a GPU and a pool of resources to use with it. +/// +/// A `wgpu-hal` `Device` represents an open connection to a specific graphics +/// processor, controlled via the backend [`Device::A`]. A `Device` is mostly +/// used for creating resources. Each `Device` has an associated [`Queue`] used +/// for command submission. +/// +/// On Vulkan a `Device` corresponds to a logical device ([`VkDevice`]). Other +/// backends don't have an exact analog: for example, [`ID3D12Device`]s and +/// [`MTLDevice`]s are owned by the backends' [`wgpu_hal::Adapter`] +/// implementations, and shared by all [`wgpu_hal::Device`]s created from that +/// `Adapter`. +/// +/// A `Device`'s life cycle is generally: +/// +/// 1) Obtain a `Device` and its associated [`Queue`] by calling +/// [`Adapter::open`]. +/// +/// Alternatively, the backend-specific types that implement [`Adapter`] often +/// have methods for creating a `wgpu-hal` `Device` from a platform-specific +/// handle. For example, [`vulkan::Adapter::device_from_raw`] can create a +/// [`vulkan::Device`] from an [`ash::Device`]. +/// +/// 1) Create resources to use on the device by calling methods like +/// [`Device::create_texture`] or [`Device::create_shader_module`]. +/// +/// 1) Call [`Device::create_command_encoder`] to obtain a [`CommandEncoder`], +/// which you can use to build [`CommandBuffer`]s holding commands to be +/// executed on the GPU. +/// +/// 1) Call [`Queue::submit`] on the `Device`'s associated [`Queue`] to submit +/// [`CommandBuffer`]s for execution on the GPU. If needed, call +/// [`Device::wait`] to wait for them to finish execution. +/// +/// 1) Free resources with methods like [`Device::destroy_texture`] or +/// [`Device::destroy_shader_module`]. +/// +/// 1) Drop the device. +/// +/// [`vkDevice`]: https://registry.khronos.org/vulkan/specs/1.3-extensions/html/vkspec.html#VkDevice +/// [`ID3D12Device`]: https://learn.microsoft.com/en-us/windows/win32/api/d3d12/nn-d3d12-id3d12device +/// [`MTLDevice`]: https://developer.apple.com/documentation/metal/mtldevice +/// [`wgpu_hal::Adapter`]: Adapter +/// [`wgpu_hal::Device`]: Device +/// [`vulkan::Adapter::device_from_raw`]: vulkan/struct.Adapter.html#method.device_from_raw +/// [`vulkan::Device`]: vulkan/struct.Device.html +/// [`ash::Device`]: https://docs.rs/ash/latest/ash/struct.Device.html +/// [`CommandBuffer`]: Api::CommandBuffer +/// +/// # Safety +/// +/// As with other `wgpu-hal` APIs, [validation] is the caller's +/// responsibility. Here are the general requirements for all `Device` +/// methods: +/// +/// - Any resource passed to a `Device` method must have been created by that +/// `Device`. For example, a [`Texture`] passed to [`Device::destroy_texture`] must +/// have been created with the `Device` passed as `self`. +/// +/// - Resources may not be destroyed if they are used by any submitted command +/// buffers that have not yet finished execution. +/// +/// [validation]: index.html#validation-is-the-calling-codes-responsibility-not-wgpu-hals +/// [`Texture`]: Api::Texture +pub trait Device: WasmNotSendSync { + type A: Api; + + /// Creates a new buffer. + /// + /// The initial usage is `wgt::BufferUses::empty()`. + unsafe fn create_buffer( + &self, + desc: &BufferDescriptor, + ) -> Result<::Buffer, DeviceError>; + + /// Free `buffer` and any GPU resources it owns. + /// + /// Note that backends are allowed to allocate GPU memory for buffers from + /// allocation pools, and this call is permitted to simply return `buffer`'s + /// storage to that pool, without making it available to other applications. + /// + /// # Safety + /// + /// - The given `buffer` must not currently be mapped. + unsafe fn destroy_buffer(&self, buffer: ::Buffer); + + /// A hook for when a wgpu-core buffer is created from a raw wgpu-hal buffer. + unsafe fn add_raw_buffer(&self, buffer: &::Buffer); + + /// Return a pointer to CPU memory mapping the contents of `buffer`. + /// + /// Buffer mappings are persistent: the buffer may remain mapped on the CPU + /// while the GPU reads or writes to it. (Note that `wgpu_core` does not use + /// this feature: when a `wgpu_core::Buffer` is unmapped, the underlying + /// `wgpu_hal` buffer is also unmapped.) + /// + /// If this function returns `Ok(mapping)`, then: + /// + /// - `mapping.ptr` is the CPU address of the start of the mapped memory. + /// + /// - If `mapping.is_coherent` is `true`, then CPU writes to the mapped + /// memory are immediately visible on the GPU, and vice versa. + /// + /// # Safety + /// + /// - The given `buffer` must have been created with the [`MAP_READ`] or + /// [`MAP_WRITE`] flags set in [`BufferDescriptor::usage`]. + /// + /// - The given `range` must fall within the size of `buffer`. + /// + /// - The caller must avoid data races between the CPU and the GPU. A data + /// race is any pair of accesses to a particular byte, one of which is a + /// write, that are not ordered with respect to each other by some sort of + /// synchronization operation. + /// + /// - If this function returns `Ok(mapping)` and `mapping.is_coherent` is + /// `false`, then: + /// + /// - Every CPU write to a mapped byte followed by a GPU read of that byte + /// must have at least one call to [`Device::flush_mapped_ranges`] + /// covering that byte that occurs between those two accesses. + /// + /// - Every GPU write to a mapped byte followed by a CPU read of that byte + /// must have at least one call to [`Device::invalidate_mapped_ranges`] + /// covering that byte that occurs between those two accesses. + /// + /// Note that the data race rule above requires that all such access pairs + /// be ordered, so it is meaningful to talk about what must occur + /// "between" them. + /// + /// - Zero-sized mappings are not allowed. + /// + /// - The returned [`BufferMapping::ptr`] must not be used after a call to + /// [`Device::unmap_buffer`]. + /// + /// [`MAP_READ`]: wgt::BufferUses::MAP_READ + /// [`MAP_WRITE`]: wgt::BufferUses::MAP_WRITE + unsafe fn map_buffer( + &self, + buffer: &::Buffer, + range: MemoryRange, + ) -> Result; + + /// Remove the mapping established by the last call to [`Device::map_buffer`]. + /// + /// # Safety + /// + /// - The given `buffer` must be currently mapped. + unsafe fn unmap_buffer(&self, buffer: &::Buffer); + + /// Indicate that CPU writes to mapped buffer memory should be made visible to the GPU. + /// + /// # Safety + /// + /// - The given `buffer` must be currently mapped. + /// + /// - All ranges produced by `ranges` must fall within `buffer`'s size. + unsafe fn flush_mapped_ranges(&self, buffer: &::Buffer, ranges: I) + where + I: Iterator; + + /// Indicate that GPU writes to mapped buffer memory should be made visible to the CPU. + /// + /// # Safety + /// + /// - The given `buffer` must be currently mapped. + /// + /// - All ranges produced by `ranges` must fall within `buffer`'s size. + unsafe fn invalidate_mapped_ranges(&self, buffer: &::Buffer, ranges: I) + where + I: Iterator; + + /// Creates a new texture. + /// + /// The initial usage for all subresources is `wgt::TextureUses::UNINITIALIZED`. + unsafe fn create_texture( + &self, + desc: &TextureDescriptor, + ) -> Result<::Texture, DeviceError>; + unsafe fn destroy_texture(&self, texture: ::Texture); + + /// A hook for when a wgpu-core texture is created from a raw wgpu-hal texture. + unsafe fn add_raw_texture(&self, texture: &::Texture); + + unsafe fn create_texture_view( + &self, + texture: &::Texture, + desc: &TextureViewDescriptor, + ) -> Result<::TextureView, DeviceError>; + unsafe fn destroy_texture_view(&self, view: ::TextureView); + unsafe fn create_sampler( + &self, + desc: &SamplerDescriptor, + ) -> Result<::Sampler, DeviceError>; + unsafe fn destroy_sampler(&self, sampler: ::Sampler); + + /// Create a fresh [`CommandEncoder`]. + /// + /// The new `CommandEncoder` is in the "closed" state. + unsafe fn create_command_encoder( + &self, + desc: &CommandEncoderDescriptor<::Queue>, + ) -> Result<::CommandEncoder, DeviceError>; + + /// Creates a bind group layout. + unsafe fn create_bind_group_layout( + &self, + desc: &BindGroupLayoutDescriptor, + ) -> Result<::BindGroupLayout, DeviceError>; + unsafe fn destroy_bind_group_layout(&self, bg_layout: ::BindGroupLayout); + unsafe fn create_pipeline_layout( + &self, + desc: &PipelineLayoutDescriptor<::BindGroupLayout>, + ) -> Result<::PipelineLayout, DeviceError>; + unsafe fn destroy_pipeline_layout(&self, pipeline_layout: ::PipelineLayout); + + #[allow(clippy::type_complexity)] + unsafe fn create_bind_group( + &self, + desc: &BindGroupDescriptor< + ::BindGroupLayout, + ::Buffer, + ::Sampler, + ::TextureView, + ::AccelerationStructure, + >, + ) -> Result<::BindGroup, DeviceError>; + unsafe fn destroy_bind_group(&self, group: ::BindGroup); + + unsafe fn create_shader_module( + &self, + desc: &ShaderModuleDescriptor, + shader: ShaderInput, + ) -> Result<::ShaderModule, ShaderError>; + unsafe fn destroy_shader_module(&self, module: ::ShaderModule); + + #[allow(clippy::type_complexity)] + unsafe fn create_render_pipeline( + &self, + desc: &RenderPipelineDescriptor< + ::PipelineLayout, + ::ShaderModule, + ::PipelineCache, + >, + ) -> Result<::RenderPipeline, PipelineError>; + unsafe fn destroy_render_pipeline(&self, pipeline: ::RenderPipeline); + + #[allow(clippy::type_complexity)] + unsafe fn create_compute_pipeline( + &self, + desc: &ComputePipelineDescriptor< + ::PipelineLayout, + ::ShaderModule, + ::PipelineCache, + >, + ) -> Result<::ComputePipeline, PipelineError>; + unsafe fn destroy_compute_pipeline(&self, pipeline: ::ComputePipeline); + + unsafe fn create_pipeline_cache( + &self, + desc: &PipelineCacheDescriptor<'_>, + ) -> Result<::PipelineCache, PipelineCacheError>; + fn pipeline_cache_validation_key(&self) -> Option<[u8; 16]> { + None + } + unsafe fn destroy_pipeline_cache(&self, cache: ::PipelineCache); + + unsafe fn create_query_set( + &self, + desc: &wgt::QuerySetDescriptor